mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_azure_create_no_blind_retry
# Conflicts: # litellm/router_python_core.py
This commit is contained in:
commit
4de8bfea56
188 changed files with 14933 additions and 3532 deletions
1
.github/workflows/osv-scan.yml
vendored
1
.github/workflows/osv-scan.yml
vendored
|
|
@ -5,6 +5,7 @@ on:
|
|||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
- "rc/**"
|
||||
schedule:
|
||||
- cron: "23 6 * * *"
|
||||
workflow_dispatch:
|
||||
|
|
|
|||
|
|
@ -151,6 +151,8 @@ pub enum Uplift {
|
|||
Residency(Residency),
|
||||
/// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
|
||||
NonGlobalEndpoint,
|
||||
/// OpenAI FedRAMP endpoint (gov.api.openai.com), from `fedramp_uplift_multiplier`
|
||||
FedrampEndpoint,
|
||||
/// Anthropic `speed: fast`, from `provider_specific_entry.fast`
|
||||
Speed(Speed),
|
||||
/// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ impl From<&EntryWire> for Adjustments {
|
|||
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
|
||||
(Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
|
||||
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
|
||||
(Uplift::FedrampEndpoint, wire.fedramp_uplift),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
|
||||
|
|
@ -746,7 +747,8 @@ mod tests {
|
|||
fn uplifts_are_adjustments_not_prices() {
|
||||
let wire = wire(json!({
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
"regional_processing_uplift_multiplier_eu": "1.2"
|
||||
"regional_processing_uplift_multiplier_eu": "1.2",
|
||||
"fedramp_uplift_multiplier": 1.1
|
||||
}));
|
||||
let adjustments = Adjustments::from(&wire);
|
||||
|
||||
|
|
@ -764,6 +766,12 @@ mod tests {
|
|||
Some(1.2)
|
||||
);
|
||||
assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
|
||||
assert_eq!(
|
||||
adjustments
|
||||
.get(&Uplift::FedrampEndpoint)
|
||||
.map(Multiplier::get),
|
||||
Some(1.1)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ pub struct ModelInfoFields {
|
|||
pub output_cost_per_token_ultrafast: Option<f64>,
|
||||
pub regional_processing_uplift_multiplier_eu: Option<f64>,
|
||||
pub regional_processing_uplift_multiplier_us: Option<f64>,
|
||||
pub fedramp_uplift_multiplier: Option<f64>,
|
||||
pub regional_endpoint_uplift_multiplier: Option<f64>,
|
||||
pub output_cost_per_audio_token: Option<f64>,
|
||||
pub output_cost_per_character: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ struct PriceBlocks {
|
|||
regional_processing_uplift_multiplier_eu: Option<Multiplier>,
|
||||
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
|
||||
regional_processing_uplift_multiplier_us: Option<Multiplier>,
|
||||
/// Multiplier applied to all token costs when served from the OpenAI FedRAMP endpoint (e.g. 1.10 = +10%).
|
||||
fedramp_uplift_multiplier: Option<Multiplier>,
|
||||
/// USD cost per web search query, keyed by search context size.
|
||||
search_context_cost_per_query: Option<SearchContextCostPerQuery>,
|
||||
/// Context-length or result-count tiered rates; each tier's costs apply within its range.
|
||||
|
|
|
|||
|
|
@ -95,6 +95,8 @@ attributes! {
|
|||
source: String,
|
||||
/// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
|
||||
successor_model: String,
|
||||
/// URL of the provider page or feed `deprecation_date` was read from.
|
||||
deprecation_source: String,
|
||||
/// Audio container formats the model can return.
|
||||
supported_audio_formats: Vec<AudioFormat>,
|
||||
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
|
||||
|
|
@ -307,6 +309,8 @@ mod tests {
|
|||
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
|
||||
#[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
|
||||
#[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
|
||||
#[case::deprecation_source("deprecation_source", json!("https://developers.openai.com/api/docs/deprecations"), true)]
|
||||
#[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)]
|
||||
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
|
||||
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
|
||||
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ pub struct EntryWire {
|
|||
pub regional_endpoint_uplift: Option<Multiplier>,
|
||||
pub residency_uplift_eu: Option<Multiplier>,
|
||||
pub residency_uplift_us: Option<Multiplier>,
|
||||
pub fedramp_uplift: Option<Multiplier>,
|
||||
pub unknown: Fields,
|
||||
pub problems: Vec<Error>,
|
||||
}
|
||||
|
|
@ -57,6 +58,7 @@ impl EntryWire {
|
|||
"regional_processing_uplift_multiplier_us" => {
|
||||
self.residency_uplift_us = reader.value(path(), value)
|
||||
}
|
||||
"fedramp_uplift_multiplier" => self.fedramp_uplift = reader.value(path(), value),
|
||||
_ => self.read_open_key(reader, key, value),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -688,6 +688,7 @@ darkbloom_models: Set = set()
|
|||
openinfer_models: Final[Set[str]] = set()
|
||||
kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
streamlake_models: Final[Set[str]] = set()
|
||||
wally_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
|
|
@ -984,6 +985,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
kyma_models.add(key)
|
||||
elif value.get("litellm_provider") == "streamlake":
|
||||
streamlake_models.add(key)
|
||||
elif value.get("litellm_provider") == "wally":
|
||||
wally_models.add(key)
|
||||
elif value.get("litellm_provider") == "zerogpu":
|
||||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
|
|
@ -1071,7 +1074,6 @@ _populate_provider_model_sets(model_cost)
|
|||
# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting
|
||||
azure_llms = {
|
||||
"gpt-35-turbo": "azure/gpt-35-turbo",
|
||||
"gpt-35-turbo-16k": "azure/gpt-35-turbo-16k",
|
||||
"gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct",
|
||||
"azure/gpt-41": "gpt-4.1",
|
||||
"azure/gpt-41-mini": "gpt-4.1-mini",
|
||||
|
|
@ -1166,6 +1168,7 @@ model_list = list(
|
|||
| openinfer_models
|
||||
| kyma_models
|
||||
| streamlake_models
|
||||
| wally_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
|
|
@ -1299,6 +1302,7 @@ def _build_models_by_provider() -> dict:
|
|||
"openinfer": openinfer_models,
|
||||
"kyma": kyma_models,
|
||||
"streamlake": streamlake_models,
|
||||
"wally": wally_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,588 @@
|
|||
{
|
||||
"posts": [
|
||||
{
|
||||
"title": "Realtime WebRTC HTTP Endpoints",
|
||||
"description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.",
|
||||
"date": "2026-03-12",
|
||||
"url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints"
|
||||
"title": "Release cadence changes and an LTS version pilot",
|
||||
"description": "Starting the week of November 2, 2026, release candidates and stable releases ship on Tuesday mornings, and LiteLLM is piloting a long-term support line on 1.108 for one month.",
|
||||
"date": "2026-11-02",
|
||||
"url": "https://docs.litellm.ai/blog/release-cadence-and-lts-pilot",
|
||||
"tags": [
|
||||
"release",
|
||||
"support"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How to build an agent with grounding on LiteLLM",
|
||||
"description": "Build a grounded chat agent in four steps with LiteLLM's AI Gateway. Connect your evidence, call a grounding guardrail, and show its verdict in chat.",
|
||||
"date": "2026-10-31",
|
||||
"url": "https://docs.litellm.ai/blog/grounded-agent",
|
||||
"tags": [
|
||||
"engineering",
|
||||
"guardrails",
|
||||
"agents"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing LiteLLM’s Unified Cost Accounting Engine",
|
||||
"description": "Consistent cost reporting, independently scalable accounting, and spend analysis by team, user, model, or custom dimensions.",
|
||||
"date": "2026-10-31",
|
||||
"url": "https://docs.litellm.ai/blog/cost-infrastructure-makeover",
|
||||
"tags": [
|
||||
"engineering",
|
||||
"reliability"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "What we are changing after October 21",
|
||||
"description": "Why the controls built after March did not stop the v1.107.0 release, and the identity, credential and detection changes we are making in response.",
|
||||
"date": "2026-10-30",
|
||||
"url": "https://docs.litellm.ai/blog/security-changes-after-oct-21-2026",
|
||||
"tags": [
|
||||
"security",
|
||||
"incident-report"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Use Hex to analyze LiteLLM costs, then set budgets and routing",
|
||||
"description": "Connect Hex to LiteLLM’s Postgres data, import a cost dashboard, then use the LiteLLM UI to set a budget or change model routing.",
|
||||
"date": "2026-10-30",
|
||||
"url": "https://docs.litellm.ai/blog/hex-litellm-roi",
|
||||
"tags": [
|
||||
"observability",
|
||||
"cost tracking",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing the LiteLLM Private Registry",
|
||||
"description": "LiteLLM is committing to a private registry to provide early access to fixes for the most severe security vulnerabilities, ahead of public release.",
|
||||
"date": "2026-10-27",
|
||||
"url": "https://docs.litellm.ai/blog/litellm-private-registry",
|
||||
"tags": [
|
||||
"security"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Morph open-weight models on LiteLLM",
|
||||
"description": "Use Morph's Kimi, GLM, and DeepSeek models through LiteLLM with tool calling, structured output, streaming, and prompt-cache cost tracking.",
|
||||
"date": "2026-10-27",
|
||||
"url": "https://docs.litellm.ai/blog/morph-provider",
|
||||
"tags": []
|
||||
},
|
||||
{
|
||||
"title": "Abliterated models on LiteLLM",
|
||||
"description": "Call abliterated models from Refuseless and Abliteration through LiteLLM, with gateway access controls, streaming, tool calls, and cached-token pricing.",
|
||||
"date": "2026-10-26",
|
||||
"url": "https://docs.litellm.ai/blog/abliterated-models",
|
||||
"tags": [
|
||||
"engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "GPT-6.1 Terra API Pricing and Day 0 Support on LiteLLM",
|
||||
"description": "GPT-6.1 Terra costs $1 per 1M input tokens and $5 per 1M output tokens, half the price of GPT-6.1 Sol. LiteLLM supports it on day 0.",
|
||||
"date": "2026-10-26",
|
||||
"url": "https://docs.litellm.ai/blog/gpt-6-1-terra",
|
||||
"tags": [
|
||||
"openai",
|
||||
"gpt-6.1",
|
||||
"gpt-6.1-terra",
|
||||
"pricing",
|
||||
"day 0 support",
|
||||
"card_title:GPT-6.1 Terra",
|
||||
"card_summary:Half the price of GPT-6.1 Sol at $1 / $5 per 1M tokens, 50.3% on Terminal-Bench 4.0, 1.05M context."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Rust Migration: /v1/messages end-to-end in Rust by Dec 1",
|
||||
"description": "By December 1st, the proxy will handle Anthropic messages requests 20x faster",
|
||||
"date": "2026-10-26",
|
||||
"url": "https://docs.litellm.ai/blog/rust-migration-rerouted",
|
||||
"tags": [
|
||||
"rust",
|
||||
"rust-migration",
|
||||
"ai-gateway",
|
||||
"engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "OpenCode Go is now a LiteLLM provider",
|
||||
"description": "Use your OpenCode Go subscription through LiteLLM. Connect once, route models to the right API, preserve sessions, and track estimated usage.",
|
||||
"date": "2026-10-24",
|
||||
"url": "https://docs.litellm.ai/blog/opencode-go",
|
||||
"tags": [
|
||||
"providers",
|
||||
"tutorial"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "The attack chain behind the v1.107.0 LiteLLM release",
|
||||
"description": "How an internal key committed to a public repository in May led to three stolen GitHub tokens and the compromised litellm v1.107.0 release.",
|
||||
"date": "2026-10-23",
|
||||
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain",
|
||||
"tags": [
|
||||
"security",
|
||||
"incident-report"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Tokenize text across vLLM and Watsonx",
|
||||
"description": "Choose token counting for prompt budgets or /tokenize to inspect tokens. Use one request format across vLLM and Watsonx through LiteLLM.",
|
||||
"date": "2026-10-23",
|
||||
"url": "https://docs.litellm.ai/blog/unified-tokenize",
|
||||
"tags": [
|
||||
"engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "October Townhall Updates: 426 Bug Fixes, 88.4% Coverage, and the Decisions API",
|
||||
"description": "A recap of the October LiteLLM town hall: security changes that need action before you upgrade, 426 bug fixes, test coverage at 88.4%, the Decisions API, Lens, Moyai, and LiteLLM Relay.",
|
||||
"date": "2026-10-23",
|
||||
"url": "https://docs.litellm.ai/blog/october-townhall-updates",
|
||||
"tags": [
|
||||
"townhall",
|
||||
"security",
|
||||
"reliability",
|
||||
"product"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Contained: Oct 21, 2026 LiteLLM release v1.107.0 Compromised",
|
||||
"description": "LiteLLM release v1.107.0 was compromised by a malicious actor.",
|
||||
"date": "2026-10-21",
|
||||
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026",
|
||||
"tags": [
|
||||
"security",
|
||||
"incident-report"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "JevRank: decision models through LiteLLM’s rerank API",
|
||||
"description": "Preview JevRank for low-cost reranking with Perplexity and TypeSafe decision models. Explore SciFact results, reproducible data and LiteLLM setup.",
|
||||
"date": "2026-10-20",
|
||||
"url": "https://docs.litellm.ai/blog/jevrank",
|
||||
"tags": [
|
||||
"engineering",
|
||||
"reranking",
|
||||
"decisions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Cache-Aware Routing for Self-Hosted Models",
|
||||
"description": "Route each agent session to the same vLLM pod on Kubernetes to reuse cached prompt prefixes.",
|
||||
"date": "2026-10-20",
|
||||
"url": "https://docs.litellm.ai/blog/cache-aware-routing",
|
||||
"tags": [
|
||||
"routing",
|
||||
"kubernetes",
|
||||
"vllm",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: TypeSafe Jev 1.14: Adds images, box questions, and long context",
|
||||
"description": "TypeSafe Jev 1.14 reads images, answers box questions with coordinates and takes long-context requests. LiteLLM serves all three on /v1/systemone and /v1/decisions behind TypeSafe's opt-in headers.",
|
||||
"date": "2026-10-15",
|
||||
"url": "https://docs.litellm.ai/blog/typesafe_jev_images",
|
||||
"tags": [
|
||||
"typesafe",
|
||||
"jev",
|
||||
"decisions",
|
||||
"day 0 support",
|
||||
"card_title:Jev 1.14",
|
||||
"card_summary:Jev 1.14 reads PNG and JPEG images, answers box questions with coordinates, and takes long-context requests on LiteLLM."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Decision models on LiteLLM: /v1/decisions and /v1/systemone",
|
||||
"description": "Decision models return probabilities instead of text, which makes routing, triage and guardrail checks cheap and deterministic. LiteLLM serves nine providers of them behind one request shape.",
|
||||
"date": "2026-10-14",
|
||||
"url": "https://docs.litellm.ai/blog/decision-models",
|
||||
"tags": [
|
||||
"openai",
|
||||
"typesafe",
|
||||
"decisions",
|
||||
"day 0 support",
|
||||
"card_title:Decision Models",
|
||||
"card_summary:Typed yes/no, choice and score answers as probabilities, from 9 providers on /v1/decisions and /v1/systemone."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Workshop: Build a self-hosted cloud agent.",
|
||||
"description": "Build a self-hosted cloud agent with the LiteLLM team. A two-week remote workshop, October 19–30, to build a working v0 and deploy an internal beta on your infrastructure.",
|
||||
"date": "2026-10-13",
|
||||
"url": "https://docs.litellm.ai/blog/cloud-agent-workshop",
|
||||
"tags": [
|
||||
"agents",
|
||||
"infrastructure",
|
||||
"announcement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Gemini 4 Argon API Pricing and Day 0 Support on LiteLLM",
|
||||
"description": "Gemini 4 Argon costs $2.00 per 1M input tokens and $10.00 per 1M output tokens until Nov 30. LiteLLM supports it on day 0 on AI Studio and Google Cloud.",
|
||||
"date": "2026-10-13",
|
||||
"url": "https://docs.litellm.ai/blog/gemini-4-argon",
|
||||
"tags": [
|
||||
"gemini",
|
||||
"gemini 4",
|
||||
"gemini-4-argon",
|
||||
"pricing",
|
||||
"day 0 support",
|
||||
"card_title:Gemini 4 Argon",
|
||||
"card_summary:Google's new flagship: 77.9% on DeepSWE v1.1, 1M context, thinking levels, $2 / $10 per 1M tokens until Nov 30."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "October Townhall: Product + Roadmap Updates",
|
||||
"description": "Join the LiteLLM October townhall on Friday, 23 October at 7:30 AM PT to learn about LiteLLM's product updates and roadmap.",
|
||||
"date": "2026-10-10",
|
||||
"url": "https://docs.litellm.ai/blog/october-townhall-announcement",
|
||||
"tags": [
|
||||
"announcement",
|
||||
"townhall"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing Microsoft 365 Copilot in LiteLLM",
|
||||
"description": "Use Microsoft 365 Copilot from your applications through LiteLLM, with access to your Microsoft 365 work data.",
|
||||
"date": "2026-10-09",
|
||||
"url": "https://docs.litellm.ai/blog/microsoft-365-copilot",
|
||||
"tags": [
|
||||
"announcement",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing litellm-core: the first step toward a leaner SDK",
|
||||
"description": "Ongoing work to reduce LiteLLM's dependencies and installation footprint, improve runtime loading, and make imports faster",
|
||||
"date": "2026-10-08",
|
||||
"url": "https://docs.litellm.ai/blog/introducing-litellm-core",
|
||||
"tags": []
|
||||
},
|
||||
{
|
||||
"title": "Claude Haiku 5.5 API Pricing and Day 0 Support on LiteLLM",
|
||||
"description": "Claude Haiku 5.5 costs $0.10 per 1M input tokens and $0.50 per 1M output tokens. LiteLLM supports it on day 0 on Anthropic, AWS, Google Cloud and Azure.",
|
||||
"date": "2026-10-07",
|
||||
"url": "https://docs.litellm.ai/blog/claude-haiku-5-5",
|
||||
"tags": [
|
||||
"anthropic",
|
||||
"claude",
|
||||
"claude haiku",
|
||||
"haiku 5.5",
|
||||
"pricing",
|
||||
"day 0 support",
|
||||
"card_title:Claude Haiku 5.5",
|
||||
"card_summary:90% cheaper than Haiku 4.5 on short prompts, 1M context, 72.4% on OSWorld 2.1, $0.10 / $0.50 per 1M tokens."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Open Sourcing Moyai: Self-Hosted Cloud Coding Agent",
|
||||
"description": "Run cloud coding agents on your own infrastructure with Moyai. Choose your agent and models through LiteLLM, track costs, and follow the setup guide.",
|
||||
"date": "2026-10-07",
|
||||
"url": "https://docs.litellm.ai/blog/moyai-open-source",
|
||||
"tags": [
|
||||
"agents",
|
||||
"open-source",
|
||||
"infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: Nano Banana 2.1",
|
||||
"description": "Day 0 support for Gemini Nano Banana 2.1 on LiteLLM, on Google AI Studio and Gemini Enterprise Agent Platform, with image output at half Nano Banana 2's price.",
|
||||
"date": "2026-10-06",
|
||||
"url": "https://docs.litellm.ai/blog/nano_banana_2_1",
|
||||
"tags": [
|
||||
"gemini",
|
||||
"gemini enterprise agent platform",
|
||||
"nano banana",
|
||||
"image generation",
|
||||
"day 0 support",
|
||||
"card_title:Nano Banana 2.1",
|
||||
"card_summary:Gemini image model with image output at half Nano Banana 2's price: $30 / MTok image out, $1.50 text in."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How LiteLLM Lens finds repeated failures across 1,000s of agent traces",
|
||||
"description": "Inside Lens: parallel trace review, Python tools for large traces, and investigations that connect repeated failures to source evidence.",
|
||||
"date": "2026-10-05",
|
||||
"url": "https://docs.litellm.ai/blog/lens-failure-patterns",
|
||||
"tags": [
|
||||
"lens",
|
||||
"agent-tracing",
|
||||
"engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How we cut time to first byte by 94% for long prompts",
|
||||
"description": "A 440k-token benchmark went from 553 ms to 35 ms median time to first byte. We removed unnecessary prompt-cache routing work before the model call.",
|
||||
"date": "2026-10-03",
|
||||
"url": "https://docs.litellm.ai/blog/long-prompt-latency",
|
||||
"tags": [
|
||||
"performance",
|
||||
"proxy",
|
||||
"engineering",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How we built our own internal Devin in 2 days",
|
||||
"description": "How we built Moyai Devin with Render, Modal, Hermes, Temporal, and LiteLLM: durable sessions, parallel agents, Slack, and shared organization connections.",
|
||||
"date": "2026-10-03",
|
||||
"url": "https://docs.litellm.ai/blog/internal-devin-two-days",
|
||||
"tags": [
|
||||
"engineering",
|
||||
"agents",
|
||||
"infrastructure",
|
||||
"slack"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Adding Self-hosted Auto Router Classifiers: Laya & Nimble",
|
||||
"description": "Use Laya or Bespoke Nimble to classify Auto Router requests on your own infrastructure. Control where classification runs, which model you serve, and how you provision it.",
|
||||
"date": "2026-10-02",
|
||||
"url": "https://docs.litellm.ai/blog/laya-nimble-classifiers",
|
||||
"tags": [
|
||||
"auto-router",
|
||||
"product",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Launching LiteLLM Lens",
|
||||
"description": "LiteLLM Lens turns the traces flowing through your gateway into findings your agents can act on. Built for agent swarms generating 200K+ traces.",
|
||||
"date": "2026-10-01",
|
||||
"url": "https://docs.litellm.ai/blog/litellm-lens-launch",
|
||||
"tags": [
|
||||
"lens",
|
||||
"agent-tracing"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How we cut LiteLLM's Redis round trips per request by 64%",
|
||||
"description": "A governed request to the LiteLLM AI Gateway waited on Redis 22 times. It now waits 8 times: one pipeline per Redis backend before the model call, one after.",
|
||||
"date": "2026-10-01",
|
||||
"url": "https://docs.litellm.ai/blog/redis-request-pipelines",
|
||||
"tags": [
|
||||
"performance",
|
||||
"redis",
|
||||
"proxy",
|
||||
"engineering",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "How we made the LiteLLM Usage page 120x faster",
|
||||
"description": "From six minutes to 3.2 seconds: how moving aggregation into Postgres made LiteLLM's Usage page roughly 120x faster in our benchmark.",
|
||||
"date": "2026-09-30",
|
||||
"url": "https://docs.litellm.ai/blog/usage-page-bounded-loading",
|
||||
"tags": [
|
||||
"performance",
|
||||
"admin-ui",
|
||||
"postgres",
|
||||
"engineering",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: GPT-6.1 Sol",
|
||||
"description": "Day 0 support for GPT-6.1 Sol on LiteLLM, with cached input at half GPT-6 Sol's price.",
|
||||
"date": "2026-09-29",
|
||||
"url": "https://docs.litellm.ai/blog/gpt_6_1_sol",
|
||||
"tags": [
|
||||
"openai",
|
||||
"gpt-6",
|
||||
"gpt-6.1-sol",
|
||||
"completion",
|
||||
"day 0 support",
|
||||
"card_title:GPT-6.1 Sol",
|
||||
"card_summary:GPT-6 Sol successor with cached input at half GPT-6 Sol's price, same request family and params."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Learn the LiteLLM gateway with a guided course",
|
||||
"description": "Follow a request through the LiteLLM gateway, Router, and SDK. A guided course for teams deploying the gateway and contributors making code changes.",
|
||||
"date": "2026-09-28",
|
||||
"url": "https://docs.litellm.ai/blog/litellm-gateway-course",
|
||||
"tags": [
|
||||
"proxy",
|
||||
"product"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: Claude Sonnet 5.5",
|
||||
"description": "Day 0 support for Claude Sonnet 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
|
||||
"date": "2026-09-28",
|
||||
"url": "https://docs.litellm.ai/blog/claude_sonnet_5_5",
|
||||
"tags": [
|
||||
"anthropic",
|
||||
"claude",
|
||||
"sonnet 5.5",
|
||||
"day 0 support",
|
||||
"card_title:Claude Sonnet 5.5",
|
||||
"card_summary:Same $2 / $10 price as Sonnet 5, 30%+ faster, 70.6% on Terminal-Bench 4.0 (Sonnet 5: 10.3%), 1M context."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing the LiteLLM ROI Calculator",
|
||||
"description": "Compare each engineer's LiteLLM gateway spend with the estimated effort of their merged pull requests. Open source and self-hosted.",
|
||||
"date": "2026-09-26",
|
||||
"url": "https://docs.litellm.ai/blog/litellm-roi-calculator",
|
||||
"tags": [
|
||||
"cost",
|
||||
"product",
|
||||
"announcement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing LiteAgents",
|
||||
"description": "Switch agent harnesses without rewriting your agent. Keep your tools and model configuration, use native harness controls, and add Temporal when you need durable runs.",
|
||||
"date": "2026-09-25",
|
||||
"url": "https://docs.litellm.ai/blog/liteagents-sdk",
|
||||
"tags": [
|
||||
"agents",
|
||||
"sdk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "September Townhall Updates: 583 Bug Fixes, OCR on Rust, and 83.5% Coverage",
|
||||
"description": "A recap of the September LiteLLM town hall: security and stability updates, OCR running on Rust by default, test coverage past the 80% target, and new Fusion and liteagents launches.",
|
||||
"date": "2026-09-24",
|
||||
"url": "https://docs.litellm.ai/blog/september-townhall-updates",
|
||||
"tags": [
|
||||
"townhall",
|
||||
"reliability",
|
||||
"product"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 support: Gemini 3.8 Flash TTS and Flash-Lite TTS",
|
||||
"description": "Day 0 support for Gemini 3.8 Flash TTS and Flash-Lite TTS on LiteLLM's /v1/audio/speech, with launch pricing tracked through 2026.",
|
||||
"date": "2026-09-23",
|
||||
"url": "https://docs.litellm.ai/blog/gemini_3_8_flash_tts",
|
||||
"tags": [
|
||||
"gemini",
|
||||
"gemini-3.8-flash-tts",
|
||||
"text to speech",
|
||||
"day 0 support",
|
||||
"card_title:Gemini 3.8 Flash TTS and Flash-Lite TTS",
|
||||
"card_summary:Gemini speech models on /v1/audio/speech, Flash and Flash-Lite, with launch pricing tracked through 2026."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Introducing LiteAdmin MCP",
|
||||
"description": "Give your agent tools to create keys, add models, and manage budgets with LiteAdmin MCP. Use the same connector through LiteAdmin, our Slack admin agent.",
|
||||
"date": "2026-09-23",
|
||||
"url": "https://docs.litellm.ai/blog/litellm-admin-mcp",
|
||||
"tags": [
|
||||
"mcp",
|
||||
"agents",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: GPT-6 Sol and GPT-6 Luna",
|
||||
"description": "Day 0 support for GPT-6 Sol and GPT-6 Luna on LiteLLM, at half the price of their GPT-5.6 counterparts.",
|
||||
"date": "2026-09-22",
|
||||
"url": "https://docs.litellm.ai/blog/gpt_6_sol_luna",
|
||||
"tags": [
|
||||
"openai",
|
||||
"gpt-6",
|
||||
"gpt-6-sol",
|
||||
"gpt-6-luna",
|
||||
"completion",
|
||||
"day 0 support",
|
||||
"card_title:GPT-6 Sol and GPT-6 Luna",
|
||||
"card_summary:Two GPT-6 variants at half the price of their GPT-5.6 counterparts, routed as the GPT-5 request family."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: Claude Opus 5.5",
|
||||
"description": "Day 0 support for Claude Opus 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
|
||||
"date": "2026-09-22",
|
||||
"url": "https://docs.litellm.ai/blog/claude_opus_5_5",
|
||||
"tags": [
|
||||
"anthropic",
|
||||
"claude",
|
||||
"opus 5.5",
|
||||
"day 0 support",
|
||||
"card_title:Claude Opus 5.5",
|
||||
"card_summary:20% cheaper than Opus 5 at $4 / $20 per 1M, 30% faster output, Terminal-Bench 4.0 66.4%, thinking always on."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Getting Started with Fireworks AI on LiteLLM",
|
||||
"description": "A beginner tutorial: run an open model on Fireworks AI behind a LiteLLM gateway, call it with the OpenAI SDK you already use, then add a second model and a fallback.",
|
||||
"date": "2026-09-22",
|
||||
"url": "https://docs.litellm.ai/blog/fireworks-getting-started",
|
||||
"tags": [
|
||||
"tutorial",
|
||||
"providers",
|
||||
"fireworks",
|
||||
"getting-started"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: Xiaomi MiMo V2.6",
|
||||
"description": "Day 0 support for Xiaomi MiMo V2.6 Pro and Flash on LiteLLM, priced on the native route for the first time.",
|
||||
"date": "2026-09-22",
|
||||
"url": "https://docs.litellm.ai/blog/mimo_v2_6",
|
||||
"tags": [
|
||||
"xiaomi",
|
||||
"mimo",
|
||||
"mimo-v2.6",
|
||||
"day 0 support",
|
||||
"card_title:Xiaomi MiMo V2.6",
|
||||
"card_summary:Xiaomi's Pro ($0.435 / $0.87 per 1M) and Flash ($0.14 / $0.28), same rates as V2.5, priced on the native route."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Claude Code server-side auto mode through LiteLLM",
|
||||
"description": "Anthropic is moving Claude Code auto mode's safety classifier server-side. LiteLLM's native /v1/messages route now forwards the safeguards contract to the Anthropic API, Bedrock InvokeModel, Bedrock Mantle and Vertex AI. Here is the contract, what changed, which releases carry it, and how to verify it.",
|
||||
"date": "2026-09-21",
|
||||
"url": "https://docs.litellm.ai/blog/claude-code-server-side-auto-mode",
|
||||
"tags": [
|
||||
"announcement",
|
||||
"claude-code",
|
||||
"anthropic",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Day 0 Support: Grok 4.7",
|
||||
"description": "Day 0 support for Grok 4.7 on LiteLLM, at the same price as Grok 4.6.",
|
||||
"date": "2026-09-21",
|
||||
"url": "https://docs.litellm.ai/blog/grok_4_7",
|
||||
"tags": [
|
||||
"xai",
|
||||
"grok",
|
||||
"grok-4.7",
|
||||
"day 0 support",
|
||||
"card_title:Grok 4.7",
|
||||
"card_summary:SpaceXAI's update to Grok 4.6 at the same price and limits, better at checking its own work and long context."
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "JEV Classifier: 5.43x as Fast as Haiku, 96% Lower Cost",
|
||||
"description": "JEV classified requests 5.43x as fast as Haiku by median latency in our AI Gateway benchmark. Explore the setup, cost savings and methodology.",
|
||||
"date": "2026-09-20",
|
||||
"url": "https://docs.litellm.ai/blog/jev-auto-router-benchmark",
|
||||
"tags": [
|
||||
"routing",
|
||||
"complexity-router",
|
||||
"engineering",
|
||||
"ai-gateway"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "TypeSafe Jev on LiteLLM",
|
||||
"description": "TypeSafe AI's Jev lands in LiteLLM v1.103.0-rc: call it through the proxy with logging and cost tracking.",
|
||||
"date": "2026-09-20",
|
||||
"url": "https://docs.litellm.ai/blog/typesafe_jev",
|
||||
"tags": [
|
||||
"typesafe",
|
||||
"jev",
|
||||
"product"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
|
|||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
|
||||
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
|
||||
DEFAULT_MAX_REQUEST_HEADER_SIZE: Final = 16 * 1024
|
||||
budget_reservation_disabled_info_emitted = False
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)
|
||||
DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512)
|
||||
|
|
@ -1092,6 +1093,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"https://api.scx.ai/v1",
|
||||
"https://api.prisminference.com/v1",
|
||||
"https://api.reka.ai/v1",
|
||||
"https://inference.runanywhere.ai/v1",
|
||||
"https://gigachat.devices.sberbank.ru/api/v1",
|
||||
"https://api.openinfer.ai/v1",
|
||||
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
|
||||
|
|
@ -1179,6 +1181,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"sail",
|
||||
"reka",
|
||||
"streamlake",
|
||||
"wally",
|
||||
]
|
||||
|
||||
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
|
||||
|
|
@ -2067,6 +2070,7 @@ SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
|
|||
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30
|
||||
SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000
|
||||
SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000
|
||||
SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS: Final = 30
|
||||
SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE: Final = 100
|
||||
# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
|
||||
# callers from forcing a DB query per request for unknown names, while bounding
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
get_billable_input_tokens,
|
||||
get_cost_per_unit,
|
||||
get_modality_token_cost_key,
|
||||
get_regional_uplift_multiplier,
|
||||
get_service_tier_cost_key,
|
||||
get_token_type_cost_breakdown,
|
||||
|
|
@ -442,6 +444,14 @@ def _per_second_pricing_cost(
|
|||
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
|
||||
return None
|
||||
return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds)
|
||||
|
||||
|
||||
def _per_second_pricing_cost_from_info(
|
||||
model_info: ModelInfoBase,
|
||||
response_time_ms: float | None,
|
||||
audio_seconds: float = 0.0,
|
||||
) -> tuple[float, float] | None:
|
||||
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
|
||||
return None
|
||||
cost_per_second: Final = model_info.get("cost_per_second")
|
||||
|
|
@ -464,7 +474,7 @@ def _per_second_pricing_cost(
|
|||
)
|
||||
verbose_logger.debug(
|
||||
"For model=%s - cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
model_info.get("key"),
|
||||
resolved_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
|
|
@ -487,6 +497,7 @@ def cost_per_token(
|
|||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
### NUMBER OF QUERIES ###
|
||||
number_of_queries: int | None = None,
|
||||
### USAGE OBJECT ###
|
||||
|
|
@ -598,6 +609,20 @@ def cost_per_token(
|
|||
if response_cost is not None:
|
||||
return response_cost[0], response_cost[1]
|
||||
|
||||
if requested_model_info_override is not None:
|
||||
per_second: Final = _per_second_pricing_cost_from_info(
|
||||
requested_model_info_override, response_time_ms, audio_transcription_file_duration
|
||||
)
|
||||
if per_second is not None:
|
||||
return per_second
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider or "",
|
||||
service_tier=service_tier,
|
||||
model_info=requested_model_info_override,
|
||||
)
|
||||
|
||||
model_cost_ref: Final = litellm.model_cost
|
||||
# Only callers that explicitly pass `custom_llm_provider` get the
|
||||
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
|
||||
|
|
@ -1747,6 +1772,7 @@ def completion_cost(
|
|||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
optional_params: dict | None = None,
|
||||
custom_pricing: bool | None = None,
|
||||
base_model: str | None = None,
|
||||
|
|
@ -1800,6 +1826,7 @@ def completion_cost(
|
|||
quality=quality,
|
||||
n=n,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
|
|
@ -2274,6 +2301,7 @@ def completion_cost(
|
|||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
|
|
@ -2508,6 +2536,7 @@ def response_cost_calculator(
|
|||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2564,6 +2593,7 @@ def response_cost_calculator(
|
|||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
|
|
@ -3158,8 +3188,12 @@ def _batch_rate(
|
|||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
fallback: float,
|
||||
*,
|
||||
prompt_tokens: int = 0,
|
||||
) -> float:
|
||||
rate: Final = model_info.get(key)
|
||||
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
|
||||
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
|
||||
rate: Final = model_info.get(key) if context_rate is None else context_rate
|
||||
return fallback if rate is None else rate
|
||||
|
||||
|
||||
|
|
@ -3309,7 +3343,8 @@ def _batch_prompt_cost(
|
|||
)
|
||||
return (
|
||||
text_tokens * input_rate
|
||||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ audio_tokens
|
||||
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
|
|
|
|||
|
|
@ -867,6 +867,7 @@ class BadGatewayError(openai.APIStatusError):
|
|||
class InternalServerError(openai.InternalServerError):
|
||||
response: httpx.Response # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX response
|
||||
request: httpx.Request # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX request
|
||||
provider_status_code: int | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -878,8 +879,10 @@ class InternalServerError(openai.InternalServerError):
|
|||
max_retries: int | None = None,
|
||||
num_retries: int | None = None,
|
||||
body: object | None = None,
|
||||
provider_status_code: int | None = None,
|
||||
):
|
||||
self.status_code = 500
|
||||
self.provider_status_code = provider_status_code
|
||||
self.message = f"litellm.InternalServerError: {message}"
|
||||
self.llm_provider = llm_provider
|
||||
self.model = model
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import json
|
|||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
|
@ -73,7 +74,7 @@ class CodeInterpreterLogOutput(TypedDict):
|
|||
class CodeInterpreterCall(TypedDict):
|
||||
id: str
|
||||
type: Literal["code_interpreter_call"]
|
||||
status: Literal["completed"]
|
||||
status: Literal["completed", "failed"]
|
||||
code: str
|
||||
container_id: str | None
|
||||
outputs: list[CodeInterpreterLogOutput]
|
||||
|
|
@ -141,6 +142,39 @@ class _SupportsOutput(Protocol):
|
|||
_CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ReadySandbox:
|
||||
container: ContainerHandle
|
||||
params: SandboxToolParams | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SandboxUnavailable:
|
||||
output: str
|
||||
|
||||
|
||||
_Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ToolRun:
|
||||
output: str
|
||||
status: Literal["completed", "failed"]
|
||||
|
||||
|
||||
_SANDBOX_UNAVAILABLE_OUTPUT: Final = "The code did not run because the code execution sandbox is unavailable."
|
||||
_SANDBOX_LOST_RESULT_OUTPUT: Final = (
|
||||
"The code execution sandbox failed while running the code, so the code may have run but its result is lost."
|
||||
)
|
||||
|
||||
|
||||
def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNAVAILABLE_OUTPUT) -> _SandboxUnavailable:
|
||||
verbose_logger.warning(
|
||||
"CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error
|
||||
)
|
||||
return _SandboxUnavailable(output=output)
|
||||
|
||||
|
||||
def _output_item_type(item: object) -> object:
|
||||
if isinstance(item, dict):
|
||||
item_mapping: Final[dict[str, object]] = item
|
||||
|
|
@ -189,6 +223,14 @@ def _extract_identity(kwargs: Mapping[str, object]) -> str:
|
|||
return identity if isinstance(identity, str) else ""
|
||||
|
||||
|
||||
def _bridged_from_responses(kwargs: Mapping[str, object], call_type: CallTypes | None) -> bool:
|
||||
route: Final = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
|
||||
return call_type in (CallTypes.completion, CallTypes.acompletion) and route in (
|
||||
CallTypes.responses,
|
||||
CallTypes.aresponses,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None:
|
||||
if sandbox_tool_name is None:
|
||||
return None
|
||||
|
|
@ -261,7 +303,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: dict[str, object], call_type: CallTypes | None
|
||||
) -> dict | None:
|
||||
if not kwargs.get("_agentic_loop_depth"):
|
||||
if not kwargs.get("_agentic_loop_depth") and not _bridged_from_responses(kwargs, call_type):
|
||||
kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
|
||||
kwargs.pop(_SANDBOX_KEY, None)
|
||||
self._strip_interception_metadata(kwargs)
|
||||
|
|
@ -451,16 +493,16 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
sandbox_key: Final = self._extract_sandbox_key(kwargs)
|
||||
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
|
||||
identity: Final = _extract_identity(kwargs) if is_session else None
|
||||
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
|
||||
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
|
||||
|
||||
try:
|
||||
container_id: Final = self._container_id(container)
|
||||
container_id: Final = self._sandbox_container_id(sandbox)
|
||||
input_list: Final = self._normalize_messages(messages)
|
||||
code_interpreter_calls: Final[list[CodeInterpreterCall]] = []
|
||||
for tool_call in tool_calls:
|
||||
arguments = tool_call.get("arguments", "")
|
||||
code = self._parse_code(arguments)
|
||||
stdout = await self._run_tool_call(container=container, params=params, arguments=arguments)
|
||||
tool_run = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
|
||||
input_list.append(
|
||||
{
|
||||
"type": "function_call",
|
||||
|
|
@ -473,22 +515,23 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": tool_call.get("call_id"),
|
||||
"output": stdout,
|
||||
"output": tool_run.output,
|
||||
}
|
||||
)
|
||||
code_interpreter_calls.append(
|
||||
{
|
||||
"id": f"ci_{uuid.uuid4().hex}",
|
||||
"type": "code_interpreter_call",
|
||||
"status": "completed",
|
||||
"status": tool_run.status,
|
||||
"code": code,
|
||||
"container_id": container_id,
|
||||
"outputs": ([{"type": "logs", "logs": stdout}] if stdout else []),
|
||||
"outputs": ([{"type": "logs", "logs": tool_run.output}] if tool_run.output else []),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
await self._delete_container_for_cache_key(sandbox_key)
|
||||
raise
|
||||
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
|
||||
|
||||
optional_params: Final = anthropic_messages_optional_request_params
|
||||
request_patch: Final = AgenticLoopRequestPatch(
|
||||
|
|
@ -526,14 +569,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
sandbox_key: Final = self._extract_sandbox_key(kwargs)
|
||||
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
|
||||
identity: Final = _extract_identity(kwargs) if is_session else None
|
||||
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
|
||||
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
|
||||
|
||||
try:
|
||||
container_id: Final = self._container_id(container)
|
||||
container_id: Final = self._sandbox_container_id(sandbox)
|
||||
tool_results: Final = [
|
||||
await self._build_chat_completion_tool_result(
|
||||
container=container,
|
||||
params=params,
|
||||
sandbox=sandbox,
|
||||
tool_call=tool_call,
|
||||
container_id=container_id,
|
||||
)
|
||||
|
|
@ -544,6 +586,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
raise
|
||||
tool_messages: Final = [result[0] for result in tool_results]
|
||||
code_interpreter_calls: Final = [result[1] for result in tool_results]
|
||||
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
|
||||
|
||||
request_patch: Final = AgenticLoopRequestPatch(
|
||||
model=model,
|
||||
|
|
@ -569,9 +612,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _container_id(container: ContainerHandle) -> str | None:
|
||||
container_id: Final[object] = getattr(container, "id", None)
|
||||
return container_id if isinstance(container_id, str) else None
|
||||
def _sandbox_container_id(sandbox: _Sandbox) -> str | None:
|
||||
match sandbox:
|
||||
case _ReadySandbox(container=container):
|
||||
container_id: Final[object] = getattr(container, "id", None)
|
||||
return container_id if isinstance(container_id, str) else None
|
||||
case _SandboxUnavailable():
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]:
|
||||
|
|
@ -588,28 +635,27 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
|
||||
async def _build_chat_completion_tool_result(
|
||||
self,
|
||||
container: ContainerHandle,
|
||||
params: SandboxToolParams | None,
|
||||
sandbox: _Sandbox,
|
||||
tool_call: CodeExecutionToolCall,
|
||||
container_id: str | None,
|
||||
) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
|
||||
arguments: Final = tool_call.get("arguments", "")
|
||||
code: Final = self._parse_code(arguments)
|
||||
stdout: Final = await self._run_tool_call(container=container, params=params, arguments=arguments)
|
||||
tool_run: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
|
||||
tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex
|
||||
return (
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call_id,
|
||||
"content": stdout,
|
||||
"content": tool_run.output,
|
||||
},
|
||||
{
|
||||
"id": f"ci_{uuid.uuid4().hex}",
|
||||
"type": "code_interpreter_call",
|
||||
"status": "completed",
|
||||
"status": tool_run.status,
|
||||
"code": code,
|
||||
"container_id": container_id,
|
||||
"outputs": [{"type": "logs", "logs": stdout}] if stdout else [],
|
||||
"outputs": [{"type": "logs", "logs": tool_run.output}] if tool_run.output else [],
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -694,19 +740,55 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
|
|||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return ""
|
||||
|
||||
async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str:
|
||||
async def _evict_sandbox_if_a_run_failed(
|
||||
self, sandbox_key: str | None, sandbox: _Sandbox, calls: Sequence[CodeInterpreterCall]
|
||||
) -> None:
|
||||
if not sandbox_key or not isinstance(sandbox, _ReadySandbox):
|
||||
return
|
||||
if not any(call["status"] == "failed" for call in calls):
|
||||
return
|
||||
cached: Final = self._container_cache.get(sandbox_key)
|
||||
if cached is None or cached[0] is not sandbox.container:
|
||||
return
|
||||
self._container_cache.pop(sandbox_key)
|
||||
await self._delete_container(container=cached[0], params=cached[1])
|
||||
|
||||
async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun:
|
||||
match sandbox:
|
||||
case _SandboxUnavailable(output=output):
|
||||
return _ToolRun(output=output, status="failed")
|
||||
case _ReadySandbox(container=container, params=params):
|
||||
match await self._run_tool_call(container=container, params=params, arguments=arguments):
|
||||
case _SandboxUnavailable(output=output):
|
||||
return _ToolRun(output=output, status="failed")
|
||||
case str() as output:
|
||||
return _ToolRun(output=output, status="completed")
|
||||
|
||||
async def _run_tool_call(
|
||||
self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str
|
||||
) -> str | _SandboxUnavailable:
|
||||
try:
|
||||
code: Final = json.loads(arguments).get("code", "") if arguments else ""
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return "[invalid tool arguments: could not parse code]"
|
||||
|
||||
result: Final = await self._run_code(container=container, params=params, code=code)
|
||||
try:
|
||||
result: Final = await self._run_code(container=container, params=params, code=code)
|
||||
except Exception as e:
|
||||
return _sandbox_unavailable("running the code", e, _SANDBOX_LOST_RESULT_OUTPUT)
|
||||
if getattr(result, "error", None):
|
||||
error: Final = result.error
|
||||
message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error)
|
||||
return f"[execution error] {message}"
|
||||
return getattr(result, "stdout", "") or ""
|
||||
|
||||
async def _acquire_sandbox(self, cache_key: str | None, identity: str | None) -> _Sandbox:
|
||||
try:
|
||||
container, params = await self._get_or_create_container(cache_key=cache_key, identity=identity)
|
||||
except Exception as e:
|
||||
return _sandbox_unavailable("creating the sandbox", e)
|
||||
return _ReadySandbox(container=container, params=params)
|
||||
|
||||
async def _get_or_create_container(
|
||||
self,
|
||||
cache_key: str | None,
|
||||
|
|
|
|||
36
litellm/litellm_core_utils/agentic_followup_failure.py
Normal file
36
litellm/litellm_core_utils/agentic_followup_failure.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
ServedRound: TypeAlias = tuple[Usage, float]
|
||||
|
||||
_MARK_ATTRIBUTE: Final = "_litellm_agentic_followup_mark"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FollowUpMark:
|
||||
served_rounds: tuple[ServedRound, ...]
|
||||
|
||||
|
||||
def _mark_of(error: BaseException) -> _FollowUpMark | None:
|
||||
mark: Final[object] = getattr(error, _MARK_ATTRIBUTE, None)
|
||||
return mark if isinstance(mark, _FollowUpMark) else None
|
||||
|
||||
|
||||
def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None:
|
||||
setattr(error, _MARK_ATTRIBUTE, _FollowUpMark(served_rounds))
|
||||
|
||||
|
||||
def is_agentic_followup_failure(error: BaseException) -> bool:
|
||||
return _mark_of(error) is not None
|
||||
|
||||
|
||||
def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]:
|
||||
mark: Final = _mark_of(error)
|
||||
return () if mark is None else mark.served_rounds
|
||||
|
||||
|
||||
def carry_agentic_followup_mark(source: BaseException, mapped: BaseException) -> None:
|
||||
if mapped is not source and is_agentic_followup_failure(source):
|
||||
mark_agentic_followup_failure(mapped, rounds_served_before(source))
|
||||
107
litellm/litellm_core_utils/agentic_loop_refusal.py
Normal file
107
litellm/litellm_core_utils/agentic_loop_refusal.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Iterable, Mapping
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageCustomToolCall,
|
||||
ChatCompletionMessageToolCall,
|
||||
Choices,
|
||||
ModelResponse,
|
||||
)
|
||||
|
||||
|
||||
def _strings(values: Iterable[object]) -> frozenset[str]:
|
||||
return frozenset(value for value in values if isinstance(value, str))
|
||||
|
||||
|
||||
def _claimed_calls(tool_calls: object) -> tuple[object, ...]:
|
||||
wrapped: Final = message_field(tool_calls, "tool_calls")
|
||||
return parts_of(tool_calls if wrapped is None else wrapped)
|
||||
|
||||
|
||||
async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[object, ...]:
|
||||
try:
|
||||
should_run, tool_calls = await gate
|
||||
except Exception as e:
|
||||
verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed with %s", type(e).__name__)
|
||||
return ()
|
||||
return _claimed_calls(tool_calls) if should_run else ()
|
||||
|
||||
|
||||
async def every_claimed_tool_call(
|
||||
refused_tool_calls: object, other_gates: Iterable[Awaitable[tuple[bool, object]]]
|
||||
) -> Mapping[str, tuple[object, ...]]:
|
||||
claims: Final = await asyncio.gather(*(_calls_a_gate_claims(gate) for gate in other_gates))
|
||||
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
|
||||
|
||||
|
||||
def _call_ids(call: object) -> frozenset[str]:
|
||||
return _strings((message_field(call, "id"), message_field(call, "call_id")))
|
||||
|
||||
|
||||
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
calls: Final = _claimed_calls(tool_calls)
|
||||
ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls))
|
||||
names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call))
|
||||
return ids, names
|
||||
|
||||
|
||||
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names
|
||||
|
||||
|
||||
def is_refused_item(
|
||||
item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str]
|
||||
) -> bool:
|
||||
if message_field(item, "type") != item_type:
|
||||
return False
|
||||
return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names)
|
||||
|
||||
|
||||
def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse:
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
kept: Final = [
|
||||
item
|
||||
for item in parts_of(message_field(response, "output"))
|
||||
if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names)
|
||||
]
|
||||
return response.model_copy(update={"output": kept})
|
||||
|
||||
|
||||
def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
name: Final = message_field(message_field(call, "function"), "name")
|
||||
return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names)
|
||||
|
||||
|
||||
def _at_position(
|
||||
call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, position: int
|
||||
) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall:
|
||||
return call if message_field(call, "index") is None else call.model_copy(update={"index": position})
|
||||
|
||||
|
||||
def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object:
|
||||
if not isinstance(choice, Choices) or not choice.message.tool_calls:
|
||||
return choice
|
||||
surviving: Final = (
|
||||
call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names)
|
||||
)
|
||||
kept: Final = [_at_position(call, position) for position, call in enumerate(surviving)]
|
||||
finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason
|
||||
return choice.model_copy(
|
||||
update={
|
||||
"message": choice.message.model_copy(update={"tool_calls": kept or None}),
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse:
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
return response.model_copy(
|
||||
update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]}
|
||||
)
|
||||
|
|
@ -1,29 +1,35 @@
|
|||
# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Iterator, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._internal_context import internal_sub_call
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
AgenticLoopLimitError,
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
converted_stream_requested,
|
||||
is_interception_internal_key,
|
||||
stashed_stream_options,
|
||||
|
|
@ -31,6 +37,8 @@ from litellm.types.integrations.custom_logger import (
|
|||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
_DispatchResult: Final = TypeVar("_DispatchResult")
|
||||
|
||||
_FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
|
||||
(
|
||||
"acompletion",
|
||||
|
|
@ -44,22 +52,41 @@ _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
async def bill_first_round_on_failure(
|
||||
dispatch: Awaitable[_DispatchResult], *, logging_obj: object, response: object, stream: bool
|
||||
) -> _DispatchResult:
|
||||
try:
|
||||
return await dispatch
|
||||
except Exception as error:
|
||||
if stream or not isinstance(logging_obj, LiteLLMLoggingObject):
|
||||
mark_agentic_followup_failure(error, rounds_served_before(error))
|
||||
raise
|
||||
this_round: Final = logging_obj.served_round(response)
|
||||
served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,)))
|
||||
mark_agentic_followup_failure(error, served_rounds)
|
||||
logging_obj.record_served_rounds_for_failure(served_rounds)
|
||||
raise
|
||||
|
||||
|
||||
def _gate_overridden(callback: CustomLogger) -> bool:
|
||||
base: Final = CustomLogger.async_should_run_agentic_loop
|
||||
func: Final = type(callback).async_should_run_agentic_loop
|
||||
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
|
||||
return type(callback).async_should_run_agentic_loop is not CustomLogger.async_should_run_agentic_loop
|
||||
|
||||
|
||||
def _build_plan_overridden(callback: CustomLogger) -> bool:
|
||||
base: Final = CustomLogger.async_build_agentic_loop_plan
|
||||
func: Final = type(callback).async_build_agentic_loop_plan
|
||||
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
|
||||
return type(callback).async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
|
||||
|
||||
|
||||
def _chat_build_plan_overridden(callback: CustomLogger) -> bool:
|
||||
return (
|
||||
type(callback).async_build_chat_completion_agentic_loop_plan
|
||||
is not CustomLogger.async_build_chat_completion_agentic_loop_plan
|
||||
)
|
||||
|
||||
|
||||
def _post_hook_overridden(callback: CustomLogger) -> bool:
|
||||
base: Final = CustomLogger.async_post_agentic_loop_response_hook
|
||||
func: Final = type(callback).async_post_agentic_loop_response_hook
|
||||
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
|
||||
return (
|
||||
type(callback).async_post_agentic_loop_response_hook is not CustomLogger.async_post_agentic_loop_response_hook
|
||||
)
|
||||
|
||||
|
||||
_STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
|
|
@ -76,14 +103,14 @@ def _coerce_int(value: object, default: int) -> int:
|
|||
return int(value) if isinstance(value, (int, str)) else default
|
||||
|
||||
|
||||
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
|
||||
def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, tuple[str, ...]]:
|
||||
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
|
||||
configured: Final = validated_max_agentic_loops(
|
||||
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
|
||||
)
|
||||
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
|
||||
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
|
||||
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
|
||||
fingerprints: Final = tuple(str(fp) for fp in raw_fingerprints) if isinstance(raw_fingerprints, list) else ()
|
||||
return depth, max_loops, fingerprints
|
||||
|
||||
|
||||
|
|
@ -96,16 +123,16 @@ def _fingerprint_tools(tool_calls: object) -> str:
|
|||
|
||||
def _check_agentic_loop_safety(
|
||||
tool_calls: object,
|
||||
fingerprints: list[str],
|
||||
fingerprints: Sequence[str],
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
model: str,
|
||||
) -> str:
|
||||
fingerprint: Final = _fingerprint_tools(tool_calls)
|
||||
if fingerprint in fingerprints:
|
||||
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
if depth >= max_loops:
|
||||
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
return fingerprint
|
||||
|
||||
|
||||
|
|
@ -134,6 +161,83 @@ def _converted_stream_replay(
|
|||
)
|
||||
|
||||
|
||||
def _end_refused_turn(
|
||||
finalized: ModelResponse,
|
||||
*,
|
||||
kwargs: Mapping[str, object],
|
||||
depth: int,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: object,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
replay: Final = _converted_stream_replay(
|
||||
finalized,
|
||||
kwargs=kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return finalized if replay is None else replay
|
||||
|
||||
|
||||
def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]:
|
||||
return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)]
|
||||
|
||||
|
||||
def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]:
|
||||
import litellm
|
||||
|
||||
dynamic: Final = getattr(logging_obj, "dynamic_success_callbacks", None) or []
|
||||
return tuple(callback for callback in (*litellm.callbacks, *dynamic) if isinstance(callback, CustomLogger))
|
||||
|
||||
|
||||
async def _refused_chat_turn(
|
||||
callbacks: Sequence[CustomLogger],
|
||||
refusing: CustomLogger,
|
||||
refused_tool_calls: object,
|
||||
*,
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
messages: list[dict[str, object]],
|
||||
tools: list[dict[str, object]] | None,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> ModelResponse:
|
||||
surface_kwargs: Final = {
|
||||
**kwargs,
|
||||
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
}
|
||||
|
||||
def other_gates() -> Iterator[Awaitable[tuple[bool, object]]]:
|
||||
for callback in callbacks:
|
||||
if callback is refusing:
|
||||
continue
|
||||
yield callback.async_should_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=False,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=dict(kwargs),
|
||||
)
|
||||
if _gate_overridden(callback):
|
||||
yield callback.async_should_run_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=False,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=dict(surface_kwargs),
|
||||
)
|
||||
|
||||
claimed: Final = await every_claimed_tool_call(refused_tool_calls, other_gates())
|
||||
return finalize_refused_chat_turn(response, claimed)
|
||||
|
||||
|
||||
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
|
||||
metadata: Final = kwargs_for_followup.get("litellm_metadata")
|
||||
return MappingProxyType(
|
||||
|
|
@ -164,6 +268,12 @@ def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]:
|
|||
}
|
||||
|
||||
|
||||
def _leave_pricing_to_the_request(response: object) -> None:
|
||||
hidden_params: Final = get_hidden_params(response)
|
||||
if hidden_params is not None and not served_from_cache(response):
|
||||
hidden_params.pop("response_cost", None)
|
||||
|
||||
|
||||
async def _execute_chat_completion_agentic_plan(
|
||||
*,
|
||||
plan: AgenticLoopPlan,
|
||||
|
|
@ -175,7 +285,7 @@ async def _execute_chat_completion_agentic_plan(
|
|||
custom_llm_provider: str,
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
fingerprints: list[str],
|
||||
fingerprints: Sequence[str],
|
||||
fingerprint: str,
|
||||
) -> object:
|
||||
import litellm
|
||||
|
|
@ -226,6 +336,7 @@ async def _execute_chat_completion_agentic_plan(
|
|||
model,
|
||||
str(e),
|
||||
)
|
||||
_leave_pricing_to_the_request(response_followup)
|
||||
replay: Final = _converted_stream_replay(
|
||||
response_followup,
|
||||
kwargs=kwargs,
|
||||
|
|
@ -252,23 +363,18 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
*,
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
messages: list,
|
||||
optional_params: dict,
|
||||
messages: list[dict[str, object]],
|
||||
optional_params: dict[str, object],
|
||||
kwargs: dict[str, object],
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str,
|
||||
stream: bool,
|
||||
) -> ModelResponse | CustomStreamWrapper | None:
|
||||
import litellm
|
||||
|
||||
callbacks: Final = litellm.callbacks + (getattr(logging_obj, "dynamic_success_callbacks", None) or [])
|
||||
callbacks: Final = _custom_loggers(logging_obj)
|
||||
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
|
||||
tools: Final = optional_params.get("tools", [])
|
||||
tools: Final = _request_tools(optional_params)
|
||||
|
||||
for callback in callbacks:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
continue
|
||||
|
||||
if not _gate_overridden(callback):
|
||||
continue
|
||||
|
||||
|
|
@ -297,29 +403,39 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
if not should_run:
|
||||
continue
|
||||
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
|
||||
try:
|
||||
if not _build_plan_overridden(callback):
|
||||
return await callback.async_run_agentic_loop(
|
||||
tools=tool_calls,
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
return _end_refused_turn(
|
||||
await _refused_chat_turn(
|
||||
callbacks,
|
||||
callback,
|
||||
tool_calls,
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
anthropic_messages_provider_config=None,
|
||||
anthropic_messages_optional_request_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=hook_kwargs,
|
||||
)
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
),
|
||||
kwargs=kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
plan = await callback.async_build_agentic_loop_plan(
|
||||
if not _build_plan_overridden(callback):
|
||||
return await callback.async_run_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -331,31 +447,38 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
kwargs=hook_kwargs,
|
||||
)
|
||||
|
||||
if plan.response_override is not None:
|
||||
return plan.response_override
|
||||
if plan.terminate:
|
||||
return response
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
plan = await callback.async_build_agentic_loop_plan(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
anthropic_messages_provider_config=None,
|
||||
anthropic_messages_optional_request_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=hook_kwargs,
|
||||
)
|
||||
|
||||
return await _execute_chat_completion_agentic_plan(
|
||||
plan=plan,
|
||||
callback=callback,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s",
|
||||
str(e),
|
||||
)
|
||||
if plan.response_override is not None:
|
||||
return plan.response_override
|
||||
if plan.terminate:
|
||||
return response
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
|
||||
return await _execute_chat_completion_agentic_plan(
|
||||
plan=plan,
|
||||
callback=callback,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
return _converted_stream_replay(
|
||||
response,
|
||||
|
|
@ -365,3 +488,113 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
|
||||
async def run_chat_completion_agentic_hooks(
|
||||
*,
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
messages: list[dict[str, object]],
|
||||
optional_params: dict[str, object],
|
||||
kwargs: dict[str, object],
|
||||
logging_obj: LiteLLMLoggingObject,
|
||||
custom_llm_provider: str,
|
||||
stream: bool,
|
||||
) -> object | None:
|
||||
callbacks: Final = _custom_loggers(logging_obj)
|
||||
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
|
||||
tools: Final = _request_tools(optional_params)
|
||||
hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider}
|
||||
|
||||
for callback in callbacks:
|
||||
try:
|
||||
should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=stream,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
except Exception:
|
||||
verbose_logger.exception(
|
||||
"LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop"
|
||||
)
|
||||
continue
|
||||
|
||||
if not should_run:
|
||||
continue
|
||||
|
||||
try:
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
return await _refused_chat_turn(
|
||||
callbacks,
|
||||
callback,
|
||||
tool_calls,
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
if not _chat_build_plan_overridden(callback):
|
||||
return await callback.async_run_chat_completion_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=hook_kwargs,
|
||||
)
|
||||
|
||||
plan = await callback.async_build_chat_completion_agentic_loop_plan(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=hook_kwargs,
|
||||
)
|
||||
|
||||
if plan.response_override is not None:
|
||||
return plan.response_override
|
||||
if plan.terminate:
|
||||
verbose_logger.debug(
|
||||
"Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason
|
||||
)
|
||||
return response
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
|
||||
return await _execute_chat_completion_agentic_plan(
|
||||
plan=plan,
|
||||
callback=callback,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
kwargs=hook_kwargs,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -25,6 +25,14 @@ def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
|
|||
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
|
||||
|
||||
|
||||
def deployment_backend_model(deployment: object) -> str:
|
||||
return next(iter(deployment_pricing_models(deployment)), "")
|
||||
|
||||
|
||||
def deployment_backend_models(deployments: Iterable[object]) -> tuple[str, ...]:
|
||||
return tuple(map(deployment_backend_model, deployments))
|
||||
|
||||
|
||||
def _named_models(*candidates: object) -> Iterator[str]:
|
||||
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)
|
||||
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ _STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType(
|
|||
}
|
||||
)
|
||||
|
||||
_OVERLOADED_STATUS_CODE: Final = "529"
|
||||
|
||||
_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = (
|
||||
TypeError,
|
||||
KeyError,
|
||||
|
|
@ -201,6 +203,8 @@ def normalize_error(exc: Exception | None, status_code: str, message: str) -> st
|
|||
)
|
||||
if by_message is not None:
|
||||
return by_message
|
||||
if status_code == _OVERLOADED_STATUS_CODE:
|
||||
return PROVIDER_OVERLOADED
|
||||
by_class: Final = _classify_by_class(exc)
|
||||
if by_class is not None:
|
||||
return by_class
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm._logging import _ENABLE_SECRET_REDACTION, redact_string, verbose_logger
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import carry_agentic_followup_mark
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
|
|
@ -605,6 +606,11 @@ def _map_openai_exception(
|
|||
)
|
||||
|
||||
|
||||
def _anthropic_overload_status(original_exception: _ProviderHTTPException) -> int | None:
|
||||
upstream_status: Final = getattr(original_exception, "status_code", None)
|
||||
return 529 if isinstance(upstream_status, int) and upstream_status >= 500 else None
|
||||
|
||||
|
||||
def _map_anthropic_exception(
|
||||
*,
|
||||
model: str,
|
||||
|
|
@ -630,6 +636,7 @@ def _map_anthropic_exception(
|
|||
message=f"AnthropicError - {error_str}",
|
||||
model=model,
|
||||
llm_provider="anthropic",
|
||||
provider_status_code=_anthropic_overload_status(original_exception),
|
||||
)
|
||||
if "Invalid API Key" in error_str:
|
||||
raise AuthenticationError(
|
||||
|
|
@ -694,6 +701,7 @@ def _map_anthropic_exception(
|
|||
llm_provider="anthropic",
|
||||
model=model,
|
||||
response=getattr(original_exception, "response", None),
|
||||
provider_status_code=original_exception.status_code,
|
||||
)
|
||||
elif original_exception.status_code == 502:
|
||||
raise BadGatewayError(
|
||||
|
|
@ -2895,6 +2903,7 @@ def exception_type(
|
|||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request
|
||||
)
|
||||
except Exception as e:
|
||||
carry_agentic_followup_mark(source=original_exception, mapped=e)
|
||||
# LOGGING
|
||||
exception_logging(
|
||||
logger_fn=None,
|
||||
|
|
@ -2920,6 +2929,7 @@ def exception_type(
|
|||
model="",
|
||||
)
|
||||
setattr(raised_exc, "litellm_response_headers", litellm_response_headers)
|
||||
carry_agentic_followup_mark(source=original_exception, mapped=raised_exc)
|
||||
raise raised_exc
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
|
|||
)
|
||||
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header"
|
||||
AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset(
|
||||
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"}
|
||||
)
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset(
|
||||
{
|
||||
"token_exchange_endpoint",
|
||||
|
|
@ -94,6 +97,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
| ANTHROPIC_WIF_KWARGS_KEYS
|
||||
| OPENAI_WIF_KWARGS_KEYS
|
||||
| OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS
|
||||
| AGENTIC_LOOP_CONTROL_KWARGS_KEYS
|
||||
| frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
BaseTokenUsageProcessor,
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
ResponsesWebSocketTokenUsageProcessor,
|
||||
declared_cost_map_pricing,
|
||||
|
|
@ -77,6 +78,7 @@ from litellm.integrations.langtrace import langtrace_trace_endpoint
|
|||
from litellm.integrations.mlflow import MlflowLogger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
|
||||
from litellm.litellm_core_utils.classifier_logging import (
|
||||
classifier_audit_fields,
|
||||
classifier_input_snapshot,
|
||||
|
|
@ -722,6 +724,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
custom_model_info: ModelInfo | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
|
@ -943,6 +946,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._sync_success_scheduled: bool = False
|
||||
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
|
||||
self._stream_failure_claimed: bool = False
|
||||
self._stash_owed_on_success: bool = False
|
||||
self._stash_billed_by_continuation: bool = False
|
||||
self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
|
||||
self._failed_attempt_ids_owed_on_success: tuple[str, ...] = ()
|
||||
self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = ()
|
||||
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
|
||||
|
||||
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
|
||||
|
|
@ -2227,6 +2235,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"custom_pricing": custom_pricing,
|
||||
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
|
||||
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
|
||||
"requested_model_info_override": self.custom_model_info if custom_pricing else None,
|
||||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
|
|
@ -2318,12 +2327,19 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return 0.0
|
||||
|
||||
def uncached_price(self, result: object) -> UncachedPrice:
|
||||
return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True)
|
||||
|
||||
def _price_without_recording(
|
||||
self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool
|
||||
) -> UncachedPrice:
|
||||
previous_cost_breakdown: Final = self.cost_breakdown
|
||||
previous_billed_token_rates: Final = self.billed_token_rates
|
||||
debug_key: Final = "response_cost_failure_debug_information"
|
||||
previous_failure_debug_info: Final = self.model_call_details.get(debug_key)
|
||||
try:
|
||||
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
|
||||
cost: Final = self.response_cost_calculator(
|
||||
result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost
|
||||
)
|
||||
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
|
||||
finally:
|
||||
self.cost_breakdown = previous_cost_breakdown
|
||||
|
|
@ -2423,6 +2439,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
additional_headers=additional_headers,
|
||||
)
|
||||
|
||||
def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None:
|
||||
"""Price this call from its own full price entry instead of ``litellm.model_cost``."""
|
||||
self.custom_model_info = custom_model_info
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
if self.spilled_over_ptu_for(result):
|
||||
return False
|
||||
|
|
@ -2771,27 +2791,43 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
own usage while the request's failure log still bills every attempt."""
|
||||
usage: Final = self._model_call_details_view.get("combined_usage_object")
|
||||
response_cost: Final = self._model_call_details_view.get("response_cost")
|
||||
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
|
||||
billed_by_continuation: Final = self._stash_billed_by_continuation
|
||||
owed_on_success: Final = self._stash_owed_on_success
|
||||
self._stash_billed_by_continuation = False
|
||||
self._stash_owed_on_success = False
|
||||
captured_attempt_ids: Final = (
|
||||
self.capture_failed_attempt_usage(usage)
|
||||
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None
|
||||
else ()
|
||||
)
|
||||
self.model_call_details.pop("_accounting_capture", None)
|
||||
self.model_call_details.pop("_accounting_pricing_snapshot", None)
|
||||
self.model_call_details.pop("_accounting_usage_observed", None)
|
||||
self.model_call_details.pop("combined_usage_object", None)
|
||||
self.model_call_details.pop("response_cost", None)
|
||||
if isinstance(usage, Usage):
|
||||
self.model_call_details["current_attempt_api_call_start_time"] = None
|
||||
self.model_call_details["failed_attempts_usage"] = (
|
||||
*self.model_call_details.get("failed_attempts_usage", ()),
|
||||
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
|
||||
)
|
||||
if not isinstance(usage, Usage):
|
||||
return
|
||||
attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0)
|
||||
self.model_call_details["current_attempt_api_call_start_time"] = None
|
||||
self.model_call_details["failed_attempts_usage"] = (
|
||||
*self.model_call_details.get("failed_attempts_usage", ()),
|
||||
attempt,
|
||||
)
|
||||
if not owed_on_success or billed_by_continuation:
|
||||
return
|
||||
self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt)
|
||||
self._failed_attempt_ids_owed_on_success = (*self._failed_attempt_ids_owed_on_success, *captured_attempt_ids)
|
||||
|
||||
def capture_failed_attempt_usage(self, usage: Usage) -> None:
|
||||
def mark_stash_billed_by_continuation(self) -> None:
|
||||
self._stash_billed_by_continuation = True
|
||||
|
||||
def capture_failed_attempt_usage(self, usage: Usage) -> tuple[str, ...]:
|
||||
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
|
||||
|
||||
capture: Final = type(self).usage_capture_callback
|
||||
if capture is None:
|
||||
return
|
||||
return ()
|
||||
attempt: Final = self.attempt_scoped_copy()
|
||||
attempt.prepared_cost = None
|
||||
attempt.finalized_cost = None
|
||||
|
|
@ -2826,11 +2862,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
datetime.datetime.now(datetime.timezone.utc),
|
||||
False,
|
||||
)
|
||||
if isinstance(captured, CapturedUsage):
|
||||
self.model_call_details["_accounting_prior_captures"] = (
|
||||
*self.model_call_details.get("_accounting_prior_captures", ()),
|
||||
captured,
|
||||
)
|
||||
if not isinstance(captured, CapturedUsage):
|
||||
return ()
|
||||
self.model_call_details["_accounting_prior_captures"] = (
|
||||
*self.model_call_details.get("_accounting_prior_captures", ()),
|
||||
captured,
|
||||
)
|
||||
return tuple(write.event.event_id for write in captured.writes)
|
||||
|
||||
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
|
||||
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
|
||||
|
|
@ -2838,20 +2876,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["continued_partial_cost"] = response_cost
|
||||
self.model_call_details["continued_response_id"] = response_id
|
||||
|
||||
def _add_continued_partial_spend(self, response: object) -> None:
|
||||
def _add_continued_partial_spend(self, response: object) -> Usage | None:
|
||||
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
|
||||
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
|
||||
own_usage: Final = getattr(response, "usage", None)
|
||||
own_cost: Final = self.model_call_details.get("response_cost")
|
||||
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
|
||||
return
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor
|
||||
|
||||
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
|
||||
usage_objects=[own_usage, partial_usage]
|
||||
)
|
||||
return own_usage if isinstance(own_usage, Usage) else None
|
||||
combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage])
|
||||
self.model_call_details["combined_usage_object"] = combined_usage
|
||||
if not isinstance(own_cost, (int, float)):
|
||||
return
|
||||
return combined_usage
|
||||
self.model_call_details["response_cost"] = own_cost + partial_cost
|
||||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
|
||||
|
|
@ -2861,6 +2896,27 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"mid_stream_partial_cost": partial_cost,
|
||||
},
|
||||
)
|
||||
return combined_usage
|
||||
|
||||
def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None:
|
||||
own_cost: Final = self.model_call_details.get("response_cost")
|
||||
if own_usage is None or not isinstance(own_cost, (int, float)):
|
||||
return
|
||||
failed_attempts: Final = self._failed_attempts_owed_on_success
|
||||
if not failed_attempts:
|
||||
return
|
||||
failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts)
|
||||
self.model_call_details.pop("failed_attempts_usage", None)
|
||||
self.failed_attempts_billed_on_success = failed_attempts
|
||||
self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
|
||||
if self._failed_attempt_ids_owed_on_success:
|
||||
self.model_call_details["_accounting_included_attempt_ids"] = self._failed_attempt_ids_owed_on_success
|
||||
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
|
||||
usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
|
||||
)
|
||||
self.model_call_details["response_cost"] = own_cost + failed_attempts_cost
|
||||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost)
|
||||
|
||||
def claim_stream_failure_report(self) -> bool:
|
||||
if self._stream_failure_claimed:
|
||||
|
|
@ -2868,6 +2924,24 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._stream_failure_claimed = True
|
||||
return True
|
||||
|
||||
def served_round(self, response: object) -> ServedRound | None:
|
||||
usage: Final = get_usage_object(completion_response=response)
|
||||
if usage is None:
|
||||
return None
|
||||
price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False)
|
||||
return usage, price.cost or 0.0
|
||||
|
||||
def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None:
|
||||
if not served_rounds:
|
||||
return
|
||||
self._stash_owed_on_success = True
|
||||
self.record_partial_usage_for_failure(
|
||||
served_rounds[0][0]
|
||||
if len(served_rounds) == 1
|
||||
else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]),
|
||||
sum(cost for _, cost in served_rounds),
|
||||
)
|
||||
|
||||
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
|
||||
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
|
||||
usage: Final = getattr(assembled, "usage", None)
|
||||
|
|
@ -3172,9 +3246,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
elif self.model_call_details.get("cache_hit") is True:
|
||||
self.model_call_details["response_cost"] = 0.0
|
||||
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
|
||||
elif "response_cost" in hidden_params:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
|
||||
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
):
|
||||
|
|
@ -3184,6 +3260,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
pass
|
||||
else:
|
||||
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
|
||||
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
|
||||
|
||||
if not build_logging_payload:
|
||||
return None
|
||||
|
|
@ -3747,7 +3824,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["response_cost"] = self.response_cost_calculator(
|
||||
result=complete_streaming_response if provider_transcription is None else provider_transcription
|
||||
)
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
logged_streaming_response: Final = self._client_facing_streaming_response(
|
||||
self._as_served_chat_response(complete_streaming_response)
|
||||
|
|
@ -4343,7 +4420,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model,
|
||||
)
|
||||
self.model_call_details["response_cost"] = None
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
|
||||
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
|
||||
|
|
@ -7836,6 +7913,20 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
|
|||
)
|
||||
|
||||
|
||||
def _with_failed_attempts_cost(breakdown: CostBreakdown, own_cost: float, failed_attempts_cost: float) -> CostBreakdown:
|
||||
return breakdown | CostBreakdown(
|
||||
total_cost=own_cost + failed_attempts_cost,
|
||||
additional_costs={**breakdown.get("additional_costs", {}), "failed_attempts_cost": failed_attempts_cost},
|
||||
)
|
||||
|
||||
|
||||
def _cached_cost_breakdown(served: CostBreakdown | None, failed_attempts: tuple[ServedRound, ...]) -> CostBreakdown:
|
||||
cached: Final = _cache_hit_cost_breakdown(served)
|
||||
if not failed_attempts:
|
||||
return cached
|
||||
return _with_failed_attempts_cost(cached, 0.0, sum(cost for _, cost in failed_attempts))
|
||||
|
||||
|
||||
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
|
||||
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
|
||||
finalized: Final = getattr(logging_obj, "finalized_cost", None)
|
||||
|
|
@ -8029,7 +8120,7 @@ def get_standard_logging_object_payload(
|
|||
if finalized_cost is not None
|
||||
else cost_breakdown_with_guardrail(
|
||||
(
|
||||
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
|
||||
_cached_cost_breakdown(uncached_price.cost_breakdown, logging_obj.failed_attempts_billed_on_success)
|
||||
if uncached_price is not None
|
||||
else logging_obj.cost_breakdown
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1203,21 +1203,55 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
|
|||
)
|
||||
|
||||
|
||||
def get_modality_token_cost_key(
|
||||
model_info: ModelInfo,
|
||||
cost_key: str,
|
||||
prompt_tokens: int,
|
||||
*,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> str:
|
||||
suffix: Final = next((suffix for suffix in _NON_STANDARD_THRESHOLD_SUFFIXES if cost_key.endswith(suffix)), "")
|
||||
base_key: Final = cost_key.removesuffix(suffix)
|
||||
threshold_key: Final = max(
|
||||
(
|
||||
key
|
||||
for key in model_info
|
||||
if key.startswith(f"{base_key}_above_")
|
||||
and key.endswith(("_tokens", f"_tokens{suffix}"))
|
||||
and model_info.get(key) is not None
|
||||
and _prompt_exceeds_threshold(prompt_tokens, _parse_above_token_threshold(key), threshold_is_inclusive)
|
||||
),
|
||||
key=_parse_above_token_threshold,
|
||||
default=None,
|
||||
)
|
||||
if threshold_key is None:
|
||||
return cost_key
|
||||
return f"{threshold_key.removesuffix(suffix)}{suffix}"
|
||||
|
||||
|
||||
def _modality_token_cost(
|
||||
model_info: ModelInfo,
|
||||
modality_cost_key: str,
|
||||
tokens: int,
|
||||
prompt_base_cost: float,
|
||||
service_tier: str | None,
|
||||
*,
|
||||
prompt_tokens: int = 0,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> float:
|
||||
"""
|
||||
Audio, image and video prompt tokens bill at their modality rate when the model has one.
|
||||
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
|
||||
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
|
||||
"""
|
||||
tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
|
||||
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
|
||||
if model_info.get(cost_key) is None:
|
||||
cost_key: Final = get_modality_token_cost_key(
|
||||
model_info,
|
||||
get_service_tier_cost_key(modality_cost_key, service_tier),
|
||||
prompt_tokens,
|
||||
threshold_is_inclusive=threshold_is_inclusive,
|
||||
)
|
||||
standard_key: Final = cost_key.removesuffix(get_service_tier_cost_key("", service_tier))
|
||||
if model_info.get(cost_key) is None and model_info.get(standard_key) is None:
|
||||
return float(tokens) * prompt_base_cost
|
||||
return calculate_cost_component(model_info, cost_key, tokens)
|
||||
|
||||
|
|
@ -1230,6 +1264,9 @@ def _calculate_input_cost(
|
|||
cache_creation_cost: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
service_tier: str | None = None,
|
||||
*,
|
||||
prompt_tokens: int = 0,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> float:
|
||||
"""
|
||||
Calculates the input cost for a given model, prompt tokens, and completion tokens.
|
||||
|
|
@ -1260,6 +1297,8 @@ def _calculate_input_cost(
|
|||
prompt_tokens_details["audio_tokens"],
|
||||
prompt_base_cost,
|
||||
service_tier,
|
||||
prompt_tokens=prompt_tokens,
|
||||
threshold_is_inclusive=threshold_is_inclusive,
|
||||
)
|
||||
|
||||
### IMAGE TOKEN COST
|
||||
|
|
@ -1618,6 +1657,8 @@ def generic_cost_per_token(
|
|||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
|
|
@ -1675,11 +1716,9 @@ def generic_cost_per_token(
|
|||
|
||||
## IMAGE COST
|
||||
if not is_text_tokens_total and image_tokens and image_tokens > 0:
|
||||
_output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None)
|
||||
_output_cost_per_image_token = (
|
||||
_output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost
|
||||
)
|
||||
completion_cost += float(image_tokens) * _output_cost_per_image_token
|
||||
image_cost_key: Final = get_service_tier_cost_key("output_cost_per_image_token", service_tier)
|
||||
image_rate: Final = get_cost_per_unit(resolved_model_info, image_cost_key, None)
|
||||
completion_cost += float(image_tokens) * (completion_base_cost if image_rate is None else image_rate)
|
||||
|
||||
## VIDEO COST
|
||||
if not is_text_tokens_total and video_tokens and video_tokens > 0:
|
||||
|
|
|
|||
104
litellm/litellm_core_utils/per_call_pricing.py
Normal file
104
litellm/litellm_core_utils/per_call_pricing.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""The price entry for a single call that sets its own custom pricing, built without writing ``litellm.model_cost``."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfo
|
||||
|
||||
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO: Final = TypeAdapter(ModelInfo)
|
||||
|
||||
|
||||
def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
"""A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that
|
||||
deployment's full custom pricing under that id when the deployment is added."""
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key)
|
||||
if not isinstance(metadata, Mapping):
|
||||
continue
|
||||
model_info = _STR_MAPPING.validate_python(metadata).get("model_info")
|
||||
if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str):
|
||||
return _STR_MAPPING.validate_python(model_info)
|
||||
return None
|
||||
|
||||
|
||||
def per_call_model_info(
|
||||
model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"]
|
||||
) -> ModelInfo | None:
|
||||
"""The full price entry for one call, or None when the cost map (or the Router's registered deployment entry)
|
||||
already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with
|
||||
the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices.
|
||||
The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the
|
||||
same model name is priced from the cost map again."""
|
||||
deployment: Final = _router_deployment_model_info(kwargs)
|
||||
registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None
|
||||
custom: Final = {
|
||||
field: kwargs[field]
|
||||
for field in CustomPricingLiteLLMParams.model_fields
|
||||
if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field])
|
||||
}
|
||||
if not custom:
|
||||
return None
|
||||
unmapped: Final[Mapping[str, object]] = {
|
||||
"key": model,
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
"litellm_provider": custom_llm_provider,
|
||||
"mode": mode,
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
row: Final = _catalog_row(model, custom_llm_provider) or unmapped
|
||||
catalog: Final = (
|
||||
{**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row
|
||||
)
|
||||
zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None
|
||||
return _MODEL_INFO.validate_python(
|
||||
{
|
||||
**catalog,
|
||||
**_token_rates_cleared_for_per_second_overrides(custom),
|
||||
**custom,
|
||||
**_tiers_with_flat_overrides(catalog, custom),
|
||||
**(zeroed or {}),
|
||||
"mode": catalog.get("mode") or mode,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_FLAT_TOKEN_RATE_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token")
|
||||
_PER_SECOND_RATE_FIELDS: Final = ("cost_per_second", "input_cost_per_second", "output_cost_per_second")
|
||||
_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def _token_rates_cleared_for_per_second_overrides(custom: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Token rates in the entry make the cost calculator skip per-second pricing, so a caller who sets only a
|
||||
per-second rate gets the catalog's token rates and tier table cleared for this call."""
|
||||
if not any(key in custom for key in _PER_SECOND_RATE_FIELDS) or any(
|
||||
key in custom for key in _FLAT_TOKEN_RATE_FIELDS
|
||||
):
|
||||
return {}
|
||||
return {"input_cost_per_token": None, "output_cost_per_token": None, "tiered_pricing": None}
|
||||
|
||||
|
||||
def _tiers_with_flat_overrides(catalog: Mapping[str, object], custom: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""A ``tiered_pricing`` table wins over flat rates in the cost calculator, so a flat rate the caller set is written
|
||||
into every tier of the table in force (the caller's own, else the inherited one) for that direction, and the other
|
||||
direction keeps each tier's own rate."""
|
||||
overrides: Final = {key: custom[key] for key in _FLAT_TOKEN_RATE_FIELDS if key in custom}
|
||||
tiers: Final = custom.get("tiered_pricing", catalog.get("tiered_pricing"))
|
||||
if not overrides or not isinstance(tiers, (list, tuple)) or not tiers:
|
||||
return {}
|
||||
return {"tiered_pricing": [{**tier, **overrides} for tier in _TIERS.validate_python(tiers)]}
|
||||
|
||||
|
||||
def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None:
|
||||
try:
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
|
||||
return None
|
||||
|
|
@ -399,6 +399,74 @@ def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, objec
|
|||
return projection.content, projection.media
|
||||
|
||||
|
||||
_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/")
|
||||
|
||||
|
||||
def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
|
||||
|
||||
mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower()
|
||||
if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES):
|
||||
return ()
|
||||
return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},)
|
||||
|
||||
|
||||
_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}")
|
||||
|
||||
|
||||
def _is_base64_data_uri(value: str) -> bool:
|
||||
header_end: Final = value.find(",", 0, 256)
|
||||
if header_end < 0 or not value[:header_end].endswith(";base64"):
|
||||
return False
|
||||
payload_start: Final = header_end + 1
|
||||
if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None:
|
||||
return False
|
||||
last_quantum: Final = value[max(payload_start, len(value) - 4) :]
|
||||
return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum
|
||||
|
||||
|
||||
def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection:
|
||||
projection: Final = _tpm_file_media(part)
|
||||
fields: Final = as_str_mapping(part.get("file"))
|
||||
data: Final = fields.get("file_data") if fields is not None else None
|
||||
if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data):
|
||||
return projection
|
||||
media: Final = _tpm_embedding_file(data)
|
||||
return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media)
|
||||
|
||||
|
||||
def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
|
||||
part: Final = as_str_mapping(value)
|
||||
if part is not None and part.get("type") == "file":
|
||||
return _tpm_embedding_file_block(part)
|
||||
if not isinstance(value, str) or not value.startswith("data:"):
|
||||
return _tpm_media_leaf(value)
|
||||
if not _is_base64_data_uri(value):
|
||||
return _TPMMediaProjection(value)
|
||||
media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value)
|
||||
return _TPMMediaProjection("" if media else value, media)
|
||||
|
||||
|
||||
def _is_json_array(value: object) -> bool:
|
||||
return isinstance(value, (list, tuple))
|
||||
|
||||
|
||||
def _tpm_embedding_item(value: object) -> _TPMMediaProjection:
|
||||
return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value)
|
||||
|
||||
|
||||
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
|
||||
projection: Final = (
|
||||
_tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value)
|
||||
)
|
||||
return projection.content, projection.media
|
||||
|
||||
|
||||
def embedding_input_is_all_media(value: object) -> bool:
|
||||
items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array
|
||||
return all(_tpm_embedding_leaf(item).media for item in items)
|
||||
|
||||
|
||||
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
|
||||
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
|
||||
recursion; an Anthropic tool_result nests its image parts exactly one level down."""
|
||||
|
|
|
|||
|
|
@ -610,6 +610,127 @@ def estimate_image_tokens(
|
|||
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
|
||||
|
||||
|
||||
_MEDIA_FILE_CHARS_PER_TOKEN: Final = 4
|
||||
_EmbeddingInputT: Final = TypeVar("_EmbeddingInputT")
|
||||
|
||||
|
||||
def _max_input_tokens(model: str) -> int | None:
|
||||
try:
|
||||
limit: Final = litellm.get_model_info(model).get("max_input_tokens")
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
|
||||
return None
|
||||
return limit if isinstance(limit, int) and limit > 0 else None
|
||||
|
||||
|
||||
def _media_file_tokens(data: str, models: Sequence[str]) -> int:
|
||||
text_estimate: Final = len(data) // _MEDIA_FILE_CHARS_PER_TOKEN
|
||||
ceilings: Final = tuple(map(_max_input_tokens, models))
|
||||
known_ceilings: Final = tuple(ceiling for ceiling in ceilings if ceiling is not None)
|
||||
if not known_ceilings or len(known_ceilings) != len(ceilings):
|
||||
return text_estimate
|
||||
return min(text_estimate, max(known_ceilings))
|
||||
|
||||
|
||||
def _block_str_field(block: Mapping[str, object], container: str, field: str) -> str | None:
|
||||
inner: Final = as_str_mapping(block.get(container))
|
||||
value: Final = inner.get(field) if inner is not None else None
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def estimate_media_block_tokens(block: Mapping[str, object], models: Sequence[str], fallback_model: str | None) -> int:
|
||||
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
||||
|
||||
match block.get("type"):
|
||||
case "input_audio":
|
||||
return estimate_audio_tokens(
|
||||
_block_str_field(block, "input_audio", "data"), models, fallback_model=fallback_model
|
||||
)
|
||||
case "file":
|
||||
return _media_file_tokens(_block_str_field(block, "file", "file_data") or "", models)
|
||||
case _:
|
||||
detail: Final = _block_str_field(block, "image_url", "detail")
|
||||
return estimate_image_tokens(
|
||||
models,
|
||||
_block_str_field(block, "image_url", "url") or "",
|
||||
"auto" if detail is None else detail,
|
||||
fallback_model=fallback_model,
|
||||
)
|
||||
|
||||
|
||||
_PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS: Final = frozenset({"vertex_ai", "gemini", "azure_ai"})
|
||||
|
||||
_DataUriMediaScope: TypeAlias = Literal["per_item", "whole_batch"]
|
||||
|
||||
|
||||
def _is_pdf_block(block: Mapping[str, object]) -> bool:
|
||||
return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf")
|
||||
|
||||
|
||||
def _bedrock_data_uri_media_scope(
|
||||
resolved_model: str, media: Sequence[Mapping[str, object]]
|
||||
) -> _DataUriMediaScope | None:
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.utils import supports_embedding_image_input
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/"))
|
||||
if not supports_embedding_image_input(f"bedrock/{base_model}"):
|
||||
return None
|
||||
match BaseAWSLLM.get_bedrock_embedding_provider(base_model):
|
||||
case "cohere":
|
||||
return "whole_batch"
|
||||
case "twelvelabs":
|
||||
return None
|
||||
case "nova" if any(map(_is_pdf_block, media)):
|
||||
return None
|
||||
case _:
|
||||
return "per_item"
|
||||
|
||||
|
||||
def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> _DataUriMediaScope | None:
|
||||
from litellm.utils import supports_embedding_image_input
|
||||
|
||||
try:
|
||||
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return None
|
||||
if provider == "bedrock":
|
||||
return _bedrock_data_uri_media_scope(resolved_model, media)
|
||||
if not supports_embedding_image_input(model):
|
||||
return None
|
||||
if provider == "cohere":
|
||||
return "whole_batch"
|
||||
return "per_item" if provider in _PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS else None
|
||||
|
||||
|
||||
def split_embedding_input_tokens(
|
||||
value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None
|
||||
) -> tuple[_EmbeddingInputT, int]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
embedding_input_is_all_media,
|
||||
split_embedding_media_for_tpm,
|
||||
)
|
||||
|
||||
text, media = split_embedding_media_for_tpm(value)
|
||||
if not media:
|
||||
return value, 0
|
||||
models: Final = tuple(media_models())
|
||||
scopes: Final = frozenset(_data_uri_media_scope(model, media) for model in models)
|
||||
if not models or None in scopes or ("whole_batch" in scopes and not embedding_input_is_all_media(value)):
|
||||
return value, 0
|
||||
media_tokens: Final = sum(estimate_media_block_tokens(block, models, fallback_model) for block in media)
|
||||
return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape
|
||||
|
||||
|
||||
def embedding_aware_token_count(
|
||||
messages: Sequence[Mapping[str, str]] | None,
|
||||
text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str]
|
||||
media_models: Callable[[], Sequence[str]],
|
||||
) -> int:
|
||||
embedding_text, media_tokens = split_embedding_input_tokens(text, media_models)
|
||||
return litellm.token_counter(messages=messages, text=embedding_text) + media_tokens
|
||||
|
||||
|
||||
TokenCounterFunction = Callable[[str], int]
|
||||
"""
|
||||
Type for a function that counts tokens in a string.
|
||||
|
|
|
|||
|
|
@ -28,7 +28,11 @@ from litellm.constants import (
|
|||
RESPONSE_FORMAT_TOOL_NAME,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import is_event_loop_running
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
drops_unsupported_params,
|
||||
map_finish_reason,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
DEFS_MAX_INLINED_BYTES,
|
||||
invalid_type_message,
|
||||
|
|
@ -581,12 +585,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None):
|
||||
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
|
||||
config: Final = super().get_config()
|
||||
|
||||
# anthropic requires a default value for max_tokens
|
||||
if config.get("max_tokens") is None:
|
||||
config["max_tokens"] = cls.get_max_tokens_for_model(model)
|
||||
config["max_tokens"] = (
|
||||
configured_max_output_tokens
|
||||
if configured_max_output_tokens is not None
|
||||
else cls.get_max_tokens_for_model(model)
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
|
|
@ -2539,7 +2547,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
optional_params["tools"] = tools
|
||||
|
||||
## Load Config
|
||||
config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model))
|
||||
model_info: Final[object] = cast( # cast-ok: request parameters are a Mapping
|
||||
Mapping[str, object], litellm_params
|
||||
).get("model_info")
|
||||
configured_max_output_tokens: Final = (
|
||||
coerce_token_limit(
|
||||
cast(Mapping[str, object], model_info).get("max_output_tokens") # cast-ok: runtime Mapping check
|
||||
)
|
||||
if isinstance(model_info, Mapping)
|
||||
else None
|
||||
)
|
||||
config: Final[Mapping[str, object]] = MappingProxyType(
|
||||
litellm.AnthropicConfig.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
|
||||
)
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in optional_params
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
|
||||
from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -35,10 +36,16 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
|
|||
b'data: {"type": "error", "error": {"type": "api_error", "message": '
|
||||
b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
|
||||
)
|
||||
AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = (
|
||||
b"event: error\n"
|
||||
b'data: {"type": "error", "error": {"type": "api_error", "message": '
|
||||
b'"The agentic loop reached its limit before this turn finished. '
|
||||
b'Raise max_agentic_loops to allow more loops."}}\n\n'
|
||||
)
|
||||
|
||||
|
||||
def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
|
||||
return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
|
||||
return chunk in (SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES)
|
||||
|
||||
|
||||
async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
|
||||
|
|
@ -216,6 +223,7 @@ class AgenticAnthropicStreamingIterator:
|
|||
self._follow_up_chunk_task: asyncio.Task | None = None
|
||||
self._replay_index = 0
|
||||
self._error_emitted = False
|
||||
self._loop_limit_reached = False
|
||||
|
||||
@property
|
||||
def has_buffered_provider_output(self) -> bool:
|
||||
|
|
@ -288,6 +296,8 @@ class AgenticAnthropicStreamingIterator:
|
|||
"tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
|
||||
self._model,
|
||||
)
|
||||
if self._loop_limit_reached:
|
||||
return AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES
|
||||
return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
|
||||
|
||||
if self._replay_index < len(self._collected_bytes):
|
||||
|
|
@ -393,14 +403,9 @@ class AgenticAnthropicStreamingIterator:
|
|||
"AgenticStreamingIterator: Unexpected result type from hooks: %s",
|
||||
type(result).__name__,
|
||||
)
|
||||
except Exception as e:
|
||||
_call_id: Final = getattr(self._logging_obj, "litellm_call_id", "unknown")
|
||||
verbose_logger.exception(
|
||||
"AgenticStreamingIterator: Error in agentic hook processing [call_id=%s model=%s]: %s",
|
||||
_call_id,
|
||||
self._model,
|
||||
str(e),
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
self._loop_limit_reached = isinstance(e, AgenticLoopLimitError)
|
||||
verbose_logger.warning("AgenticStreamingIterator: agentic loop refused [rail=%s]", type(e).__name__)
|
||||
|
||||
@staticmethod
|
||||
def _rebuild_anthropic_response_from_sse(
|
||||
|
|
|
|||
|
|
@ -111,10 +111,4 @@ class AzureAIRerankConfig(CohereRerankConfig):
|
|||
return rerank_response
|
||||
|
||||
def _get_base_model(self, azure_model_group: str | None) -> str | None:
|
||||
if azure_model_group is None:
|
||||
return None
|
||||
if azure_model_group == "offer-cohere-rerank-mul-paygo":
|
||||
return "azure_ai/cohere-rerank-v3-multilingual"
|
||||
if azure_model_group == "offer-cohere-rerank-eng-paygo":
|
||||
return "azure_ai/cohere-rerank-v3-english"
|
||||
return azure_model_group
|
||||
|
|
|
|||
|
|
@ -965,8 +965,6 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
self,
|
||||
aws_role_name: str,
|
||||
ssl_verify: bool | str | None = None,
|
||||
aws_region_name: str | None = None,
|
||||
aws_sts_endpoint: str | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the current environment is already running as the target IAM role.
|
||||
|
|
@ -1000,14 +998,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
import boto3
|
||||
|
||||
with tracer.trace("boto3.client(sts).get_caller_identity"):
|
||||
sts_client: Final = boto3.client(
|
||||
"sts",
|
||||
**self._build_sts_client_kwargs(
|
||||
aws_sts_endpoint=aws_sts_endpoint,
|
||||
ssl_verify=ssl_verify,
|
||||
aws_region_name=aws_region_name,
|
||||
),
|
||||
)
|
||||
sts_client: Final = boto3.client("sts", verify=self._get_ssl_verify(ssl_verify))
|
||||
identity: Final = sts_client.get_caller_identity()
|
||||
caller_arn: Final = identity.get("Arn", "")
|
||||
|
||||
|
|
@ -1298,12 +1289,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
caller's key skips both. ``aws_session_name`` defaults inside this fetcher rather than in
|
||||
``get_credentials`` so the cache key stays stable when the caller does not supply one.
|
||||
"""
|
||||
if self._is_already_running_as_role(
|
||||
aws_role_name,
|
||||
ssl_verify=ssl_verify,
|
||||
aws_region_name=aws_region_name,
|
||||
aws_sts_endpoint=aws_sts_endpoint,
|
||||
):
|
||||
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
|
||||
verbose_logger.debug(
|
||||
"Already running as target role %s, using ambient credentials",
|
||||
aws_role_name,
|
||||
|
|
@ -1429,12 +1415,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
# partition, and role name). This avoids silently using the
|
||||
# wrong identity when there is a genuine trust-policy or
|
||||
# permission misconfiguration.
|
||||
if self._is_already_running_as_role(
|
||||
aws_role_name,
|
||||
ssl_verify=ssl_verify,
|
||||
aws_region_name=aws_region_name,
|
||||
aws_sts_endpoint=aws_sts_endpoint,
|
||||
):
|
||||
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
|
||||
verbose_logger.warning(
|
||||
"AssumeRole failed for %s (%s). "
|
||||
"Caller is already running as this role; "
|
||||
|
|
|
|||
|
|
@ -109,8 +109,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
|||
self._read_configured_max_tokens: Final = read_configured_max_tokens
|
||||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict
|
||||
return {key: value for key, value in super().get_config(model=model).items() if key != "max_tokens"}
|
||||
def get_config(
|
||||
cls, *, model: str | None = None, configured_max_output_tokens: int | None = None
|
||||
) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict
|
||||
return {
|
||||
key: value
|
||||
for key, value in super()
|
||||
.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
|
||||
.items()
|
||||
if key != "max_tokens"
|
||||
}
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import httpx
|
|||
|
||||
from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.custom_httpx.http_handler import header_value
|
||||
from litellm.types.llms.bedrock import (
|
||||
CohereEmbeddingRequest,
|
||||
CohereEmbeddingRequestWithModel,
|
||||
|
|
@ -23,6 +24,11 @@ class _SupportsEncode(Protocol):
|
|||
def encode(self, text: str, /) -> Sized: ...
|
||||
|
||||
|
||||
def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None:
|
||||
count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count")
|
||||
return int(count) if count is not None and count.isdigit() else None
|
||||
|
||||
|
||||
class CohereEmbeddingConfig:
|
||||
"""
|
||||
Reference: https://docs.cohere.com/v2/reference/embed
|
||||
|
|
@ -76,32 +82,30 @@ class CohereEmbeddingConfig:
|
|||
) -> CohereEmbeddingRequestWithModel:
|
||||
return self._transform_request(model, input, inference_params)
|
||||
|
||||
def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage:
|
||||
input_tokens = 0
|
||||
|
||||
def _calculate_usage(
|
||||
self, input: list[str], encoding: _SupportsEncode, meta: dict, billed_input_tokens: int | None = None
|
||||
) -> Usage:
|
||||
text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens")
|
||||
|
||||
image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images")
|
||||
|
||||
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
|
||||
if image_tokens is None and text_tokens is None:
|
||||
for text in input:
|
||||
input_tokens += len(encoding.encode(text))
|
||||
else:
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
image_tokens=image_tokens,
|
||||
text_tokens=text_tokens,
|
||||
counted_tokens: Final = (
|
||||
billed_input_tokens
|
||||
if billed_input_tokens is not None
|
||||
else sum(len(encoding.encode(text)) for text in input)
|
||||
)
|
||||
if image_tokens:
|
||||
input_tokens += image_tokens
|
||||
if text_tokens:
|
||||
input_tokens += text_tokens
|
||||
return Usage(prompt_tokens=counted_tokens, completion_tokens=0, total_tokens=counted_tokens)
|
||||
|
||||
input_tokens: Final = (image_tokens or 0) + (text_tokens or 0)
|
||||
return Usage(
|
||||
prompt_tokens=input_tokens,
|
||||
completion_tokens=0,
|
||||
total_tokens=input_tokens,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
image_tokens=image_tokens,
|
||||
text_tokens=text_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
def _transform_response(
|
||||
|
|
@ -129,6 +133,7 @@ class CohereEmbeddingConfig:
|
|||
model=model,
|
||||
encoding=encoding,
|
||||
input=input,
|
||||
billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None,
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
|
|
@ -151,6 +156,7 @@ class CohereEmbeddingConfig:
|
|||
model: str,
|
||||
encoding: _SupportsEncode,
|
||||
input: list,
|
||||
billed_input_tokens: int | None = None,
|
||||
) -> EmbeddingResponse:
|
||||
"""
|
||||
Parse a Cohere embed response body into an OpenAI-style EmbeddingResponse.
|
||||
|
|
@ -195,7 +201,7 @@ class CohereEmbeddingConfig:
|
|||
setattr(
|
||||
model_response,
|
||||
"usage",
|
||||
self._calculate_usage(input, encoding, response_json.get("meta", {})),
|
||||
self._calculate_usage(input, encoding, response_json.get("meta", {}), billed_input_tokens),
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
|
|
|||
|
|
@ -47,6 +47,12 @@ from litellm.anthropic_beta_headers_manager import (
|
|||
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import (
|
||||
every_claimed_tool_call,
|
||||
finalize_refused_responses_turn,
|
||||
is_refused_item,
|
||||
refused_tool_call_identifiers,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
|
|
@ -56,11 +62,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
|
|||
SUBTITLE_RESPONSE_FORMATS,
|
||||
synthesize_subtitle_document,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
qualify_agentic_followup_model,
|
||||
set_provider_response_headers_in_hidden_params,
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
|
||||
bill_first_round_on_failure,
|
||||
run_chat_completion_agentic_hooks,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
|
|
@ -150,6 +156,7 @@ from litellm.types.extraction import ExtractionResponse
|
|||
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
AgenticLoopLimitError,
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
|
|
@ -6005,7 +6012,7 @@ class BaseLLMHTTPHandler:
|
|||
@staticmethod
|
||||
def _check_agentic_loop_safety(
|
||||
tool_calls: object,
|
||||
fingerprints: list[str],
|
||||
fingerprints: Sequence[str],
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
model: str,
|
||||
|
|
@ -6024,7 +6031,7 @@ class BaseLLMHTTPHandler:
|
|||
if fingerprint in fingerprints:
|
||||
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
if depth >= max_loops:
|
||||
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
return fingerprint
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -6034,26 +6041,6 @@ class BaseLLMHTTPHandler:
|
|||
except Exception:
|
||||
return str(tools)
|
||||
|
||||
@staticmethod
|
||||
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
"""
|
||||
Collect the ids and names of the tool calls a safety rail just refused.
|
||||
|
||||
Callbacks hand back either a bare list of tool calls or a dict wrapping
|
||||
that list under ``tool_calls``, and both the anthropic and responses
|
||||
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
|
||||
"""
|
||||
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
|
||||
if not isinstance(calls, list):
|
||||
return frozenset(), frozenset()
|
||||
dict_calls: Final = (call for call in calls if isinstance(call, dict))
|
||||
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
|
||||
ids: Final = frozenset(
|
||||
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
|
||||
)
|
||||
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
|
||||
return ids, names
|
||||
|
||||
@staticmethod
|
||||
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
"""
|
||||
|
|
@ -6064,22 +6051,13 @@ class BaseLLMHTTPHandler:
|
|||
share a name with a refused one. The name is only consulted for tool call
|
||||
shapes that arrive without an id.
|
||||
"""
|
||||
if not isinstance(block, dict) or block.get("type") != "tool_use":
|
||||
return False
|
||||
block_id: Final = block.get("id")
|
||||
if isinstance(block_id, str) and refused_ids:
|
||||
return block_id in refused_ids
|
||||
return block.get("name") in refused_names
|
||||
return is_refused_item(block, "tool_use", "id", refused_ids, refused_names)
|
||||
|
||||
@staticmethod
|
||||
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
|
||||
"""
|
||||
Whether a refused rerun can still be answered with a finalized turn.
|
||||
|
||||
Only the anthropic messages surface can. The responses surface carries a
|
||||
pydantic model the finalizer does not rewrite, so it keeps raising, which
|
||||
is what every surface did before this path learned to end the turn.
|
||||
|
||||
The messages and responses call sites pass ``stream=False``, because
|
||||
interception converts an intercepted stream to non-streaming before the
|
||||
loop runs and rebuilds the SSE stream from the finalized turn
|
||||
|
|
@ -6088,7 +6066,7 @@ class BaseLLMHTTPHandler:
|
|||
finalized turn would reach the client as a second message rather than
|
||||
as a replacement.
|
||||
"""
|
||||
return not stream and api_surface == "anthropic_messages"
|
||||
return not stream and api_surface in ("anthropic_messages", "responses")
|
||||
|
||||
@staticmethod
|
||||
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
|
||||
|
|
@ -6105,10 +6083,12 @@ class BaseLLMHTTPHandler:
|
|||
one is still in the response the stop reason stays ``tool_use`` so the
|
||||
client knows to answer it.
|
||||
"""
|
||||
if isinstance(response, ResponsesAPIResponse):
|
||||
return finalize_refused_responses_turn(response, tool_calls)
|
||||
if not isinstance(response, dict):
|
||||
return response
|
||||
|
||||
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
finalized: Final = dict(response)
|
||||
content: Final = finalized.get("content")
|
||||
if isinstance(content, list):
|
||||
|
|
@ -6135,7 +6115,7 @@ class BaseLLMHTTPHandler:
|
|||
kwargs: dict,
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
fingerprints: list[str],
|
||||
fingerprints: Sequence[str],
|
||||
fingerprint: str,
|
||||
stream: bool = False,
|
||||
callback: Optional["CustomLogger"] = None,
|
||||
|
|
@ -6176,7 +6156,7 @@ class BaseLLMHTTPHandler:
|
|||
kwargs_for_followup.update(patch.kwargs)
|
||||
kwargs_for_followup["_agentic_loop_depth"] = depth + 1
|
||||
kwargs_for_followup["max_agentic_loops"] = max_loops
|
||||
kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
|
||||
kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint]
|
||||
|
||||
response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate(
|
||||
**{
|
||||
|
|
@ -6215,7 +6195,7 @@ class BaseLLMHTTPHandler:
|
|||
kwargs: dict,
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
fingerprints: list[str],
|
||||
fingerprints: Sequence[str],
|
||||
fingerprint: str,
|
||||
callback: Optional["CustomLogger"] = None,
|
||||
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
|
||||
|
|
@ -6336,67 +6316,6 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
async def _execute_chat_completion_agentic_plan(
|
||||
self,
|
||||
plan: AgenticLoopPlan,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
optional_params: dict,
|
||||
kwargs: dict,
|
||||
custom_llm_provider: str,
|
||||
depth: int,
|
||||
max_loops: int,
|
||||
fingerprints: list[str],
|
||||
fingerprint: str,
|
||||
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
patch: Final = plan.request_patch or AgenticLoopRequestPatch()
|
||||
if patch.messages is None:
|
||||
raise ValueError("Agentic loop plan missing patched messages")
|
||||
|
||||
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
|
||||
|
||||
optional_params_for_followup: Final = dict(optional_params)
|
||||
optional_params_for_followup.update(patch.optional_params)
|
||||
if patch.tools is not None:
|
||||
optional_params_for_followup["tools"] = patch.tools
|
||||
|
||||
internal_params: Final = {
|
||||
"_websearch_interception",
|
||||
"acompletion",
|
||||
"litellm_logging_obj",
|
||||
"custom_llm_provider",
|
||||
"model_alias_map",
|
||||
"stream_response",
|
||||
"custom_prompt_dict",
|
||||
}
|
||||
kwargs_for_followup: Final = build_agentic_followup_kwargs(
|
||||
request_kwargs=MappingProxyType(
|
||||
{
|
||||
k: v
|
||||
for k, v in kwargs.items()
|
||||
if not k.startswith("_websearch_interception")
|
||||
and not k.startswith("_compression_interception")
|
||||
and k not in internal_params
|
||||
}
|
||||
),
|
||||
patch_kwargs=patch.kwargs,
|
||||
request_params=frozenset((*optional_params_for_followup, "model", "messages")),
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
with internal_sub_call(logging_obj.litellm_call_id if logging_obj is not None else None):
|
||||
response: Final = await litellm.acompletion(
|
||||
model=full_model_name,
|
||||
messages=patch.messages,
|
||||
**optional_params_for_followup,
|
||||
**kwargs_for_followup,
|
||||
)
|
||||
return response
|
||||
|
||||
def _maybe_wrap_in_fake_stream(
|
||||
self,
|
||||
response: _ResponseT,
|
||||
|
|
@ -6513,26 +6432,38 @@ class BaseLLMHTTPHandler:
|
|||
except AgenticLoopSafetyError as e:
|
||||
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
|
||||
raise
|
||||
verbose_logger.warning(
|
||||
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
|
||||
logging_obj.litellm_call_id,
|
||||
model,
|
||||
str(e),
|
||||
)
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
|
||||
self._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls=await every_claimed_tool_call(
|
||||
tool_calls,
|
||||
(
|
||||
other.async_should_run_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=stream,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=hook_kwargs,
|
||||
)
|
||||
for other in callbacks
|
||||
if isinstance(other, CustomLogger) and other is not callback
|
||||
),
|
||||
),
|
||||
),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
|
||||
try:
|
||||
kwargs_with_provider = hook_kwargs.copy()
|
||||
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
|
||||
build_plan_overridden = (
|
||||
callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
|
||||
)
|
||||
if not build_plan_overridden:
|
||||
agentic_result: object = await callback.async_run_agentic_loop(
|
||||
kwargs_with_provider = {**hook_kwargs, "custom_llm_provider": custom_llm_provider}
|
||||
build_plan_overridden = (
|
||||
callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
|
||||
)
|
||||
if not build_plan_overridden:
|
||||
agentic_result: object = await bill_first_round_on_failure(
|
||||
callback.async_run_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -6542,10 +6473,15 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=kwargs_with_provider,
|
||||
)
|
||||
return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface)
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
)
|
||||
return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface)
|
||||
|
||||
plan = await callback.async_build_agentic_loop_plan(
|
||||
plan = await bill_first_round_on_failure(
|
||||
callback.async_build_agentic_loop_plan(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -6555,30 +6491,35 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=kwargs_with_provider,
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
if plan.response_override is not None:
|
||||
return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface)
|
||||
if plan.terminate:
|
||||
verbose_logger.debug(
|
||||
"Agentic loop terminated by callback=%s reason=%s",
|
||||
callback.__class__.__name__,
|
||||
plan.stop_reason,
|
||||
)
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
await callback.async_post_agentic_loop_response_hook(
|
||||
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
|
||||
plan=plan,
|
||||
kwargs=kwargs_with_provider,
|
||||
),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
|
||||
if plan.response_override is not None:
|
||||
return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface)
|
||||
if plan.terminate:
|
||||
verbose_logger.debug(
|
||||
"Agentic loop terminated by callback=%s reason=%s",
|
||||
callback.__class__.__name__,
|
||||
plan.stop_reason,
|
||||
)
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
await callback.async_post_agentic_loop_response_hook(
|
||||
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
|
||||
plan=plan,
|
||||
kwargs=kwargs_with_provider,
|
||||
),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
|
||||
if api_surface == "responses":
|
||||
return await self._execute_responses_agentic_plan(
|
||||
if api_surface == "responses":
|
||||
return await bill_first_round_on_failure(
|
||||
self._execute_responses_agentic_plan(
|
||||
plan=plan,
|
||||
model=model,
|
||||
response_api_optional_request_params=anthropic_messages_optional_request_params,
|
||||
|
|
@ -6589,10 +6530,15 @@ class BaseLLMHTTPHandler:
|
|||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
callback=callback,
|
||||
)
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
await self._execute_anthropic_agentic_plan(
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
await bill_first_round_on_failure(
|
||||
self._execute_anthropic_agentic_plan(
|
||||
plan=plan,
|
||||
model=model,
|
||||
messages=messages,
|
||||
|
|
@ -6606,17 +6552,13 @@ class BaseLLMHTTPHandler:
|
|||
stream=stream,
|
||||
callback=callback,
|
||||
),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
except Exception as e:
|
||||
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
|
||||
verbose_logger.exception(
|
||||
"LiteLLM.AgenticHookError: Exception in agentic completion hooks [call_id=%s model=%s]: %s",
|
||||
_call_id,
|
||||
model,
|
||||
str(e),
|
||||
)
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
|
||||
# Check if we need to convert response to fake stream
|
||||
# This happens when:
|
||||
|
|
@ -6648,111 +6590,23 @@ class BaseLLMHTTPHandler:
|
|||
|
||||
Returns the response from agentic loop, or None if no hook runs.
|
||||
"""
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
|
||||
tools: Final = optional_params.get("tools", [])
|
||||
depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
|
||||
|
||||
for callback in callbacks:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
continue
|
||||
if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
|
||||
continue
|
||||
|
||||
should_run: bool = False
|
||||
tool_calls: object = None
|
||||
try:
|
||||
(
|
||||
should_run,
|
||||
tool_calls,
|
||||
) = await callback.async_should_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=stream,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s",
|
||||
str(e),
|
||||
)
|
||||
continue
|
||||
|
||||
if not should_run:
|
||||
continue
|
||||
|
||||
# Safety guards must run OUTSIDE the callback try/except — they are
|
||||
# bounded-loop / cycle-break rails that must propagate to the caller.
|
||||
fingerprint = self._check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
agentic_response: Final = await bill_first_round_on_failure(
|
||||
run_chat_completion_agentic_hooks(
|
||||
response=response,
|
||||
model=model,
|
||||
)
|
||||
|
||||
try:
|
||||
kwargs_with_provider = kwargs.copy() if kwargs else {}
|
||||
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
|
||||
build_plan_overridden = (
|
||||
callback.__class__.async_build_chat_completion_agentic_loop_plan
|
||||
is not CustomLogger.async_build_chat_completion_agentic_loop_plan
|
||||
)
|
||||
if not build_plan_overridden:
|
||||
return await callback.async_run_chat_completion_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=kwargs_with_provider,
|
||||
)
|
||||
|
||||
plan = await callback.async_build_chat_completion_agentic_loop_plan(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=kwargs_with_provider,
|
||||
)
|
||||
|
||||
if plan.response_override is not None:
|
||||
return plan.response_override
|
||||
if plan.terminate:
|
||||
verbose_logger.debug(
|
||||
"Agentic chat loop terminated by callback=%s reason=%s",
|
||||
callback.__class__.__name__,
|
||||
plan.stop_reason,
|
||||
)
|
||||
return response
|
||||
if not plan.run_agentic_loop:
|
||||
continue
|
||||
|
||||
return await self._execute_chat_completion_agentic_plan(
|
||||
plan=plan,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs_with_provider,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
fingerprints=fingerprints,
|
||||
fingerprint=fingerprint,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e)
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
kwargs=kwargs or {},
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=stream,
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
)
|
||||
if agentic_response is not None:
|
||||
return agentic_response
|
||||
|
||||
# Check if we need to convert response to fake stream for chat completions
|
||||
# This happens when:
|
||||
|
|
|
|||
|
|
@ -395,7 +395,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
|
|||
return "databricks"
|
||||
|
||||
@classmethod
|
||||
def get_config(cls, *, model: str | None = None):
|
||||
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
|
||||
return super().get_config()
|
||||
|
||||
def get_json_schema_from_pydantic_object(
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ from litellm._logging import verbose_logger
|
|||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
|
||||
bill_first_round_on_failure,
|
||||
run_chat_completion_agentic_hooks,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
|
||||
|
|
@ -665,67 +669,23 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
|
||||
Returns the response from agentic loop, or None if no hook runs.
|
||||
"""
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
|
||||
# Avoid logging full callback objects to prevent leaking sensitive data
|
||||
verbose_logger.debug("LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks))
|
||||
tools: Final = optional_params.get("tools", [])
|
||||
# Avoid logging full tools payloads; they may contain sensitive parameters
|
||||
verbose_logger.debug(
|
||||
"LiteLLM.AgenticHooks: tools_count=%s",
|
||||
len(tools) if isinstance(tools, list) else 1 if tools else 0,
|
||||
if not isinstance(response, ModelResponse):
|
||||
return None
|
||||
return await bill_first_round_on_failure(
|
||||
run_chat_completion_agentic_hooks(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
kwargs=litellm_params or {},
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=litellm_params.get("custom_llm_provider", "openai"),
|
||||
stream=stream,
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
response=response,
|
||||
stream=stream,
|
||||
)
|
||||
# Get custom_llm_provider from litellm_params
|
||||
custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai")
|
||||
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if isinstance(callback, CustomLogger):
|
||||
# Check if the callback has the chat completion agentic loop methods
|
||||
if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
|
||||
continue
|
||||
|
||||
# First: Check if agentic loop should run (using chat completion method)
|
||||
(
|
||||
should_run,
|
||||
tool_calls,
|
||||
) = await callback.async_should_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
stream=stream,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=litellm_params,
|
||||
)
|
||||
|
||||
if should_run:
|
||||
# Second: Execute agentic loop
|
||||
kwargs_with_provider = litellm_params.copy() if litellm_params else {}
|
||||
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# For OpenAI Chat Completions, use the chat completion agentic loop method
|
||||
agentic_response: object = await callback.async_run_chat_completion_agentic_loop(
|
||||
tools=tool_calls,
|
||||
model=model,
|
||||
messages=messages,
|
||||
response=response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=logging_obj,
|
||||
stream=stream,
|
||||
kwargs=kwargs_with_provider,
|
||||
)
|
||||
# First hook that runs agentic loop wins
|
||||
return agentic_response
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def mock_streaming(
|
||||
self,
|
||||
|
|
@ -1076,8 +1036,28 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
hidden_params={"headers": headers},
|
||||
_response_headers=headers,
|
||||
)
|
||||
except openai.UnprocessableEntityError as e:
|
||||
if litellm.drop_params is True or drop_params is True:
|
||||
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
|
||||
else:
|
||||
raise e
|
||||
except Exception as e:
|
||||
exception_response: object = getattr(e, "response", None)
|
||||
status_code = getattr(e, "status_code", 500)
|
||||
exception_body = getattr(e, "body", None)
|
||||
error_headers = getattr(e, "headers", None)
|
||||
if error_headers is None and exception_response:
|
||||
error_headers = getattr(exception_response, "headers", None)
|
||||
message = getattr(e, "message", str(e))
|
||||
|
||||
# Call agentic completion hooks (e.g., for websearch_interception)
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=exception_body,
|
||||
)
|
||||
else:
|
||||
agentic_response = await self._call_agentic_completion_hooks_openai(
|
||||
response=final_response_obj,
|
||||
model=model,
|
||||
|
|
@ -1100,29 +1080,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
)
|
||||
|
||||
return final_response_obj
|
||||
except openai.UnprocessableEntityError as e:
|
||||
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
|
||||
if litellm.drop_params is True or drop_params is True:
|
||||
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
|
||||
else:
|
||||
raise e
|
||||
# e.message
|
||||
except Exception as e:
|
||||
exception_response: object = getattr(e, "response", None)
|
||||
status_code = getattr(e, "status_code", 500)
|
||||
exception_body = getattr(e, "body", None)
|
||||
error_headers = getattr(e, "headers", None)
|
||||
if error_headers is None and exception_response:
|
||||
error_headers = getattr(exception_response, "headers", None)
|
||||
message = getattr(e, "message", str(e))
|
||||
|
||||
raise OpenAIError(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
response=upstream_error_response(e),
|
||||
headers=error_headers,
|
||||
body=exception_body,
|
||||
)
|
||||
|
||||
def streaming(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -302,5 +302,14 @@
|
|||
"preserve_cache_control": true
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions"]
|
||||
},
|
||||
"wally": {
|
||||
"base_url": "https://inference.runanywhere.ai/v1",
|
||||
"api_key_env": "WALLY_API_KEY",
|
||||
"api_base_env": "WALLY_API_BASE",
|
||||
"provider_info": {
|
||||
"supports_function_calling": true,
|
||||
"supports_reasoning": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
173
litellm/main.py
173
litellm/main.py
|
|
@ -81,11 +81,13 @@ from litellm.litellm_core_utils.audio_utils.utils import (
|
|||
get_audio_file_for_health_check,
|
||||
)
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
|
||||
bill_first_round_on_failure,
|
||||
maybe_run_chat_completion_agentic_loop,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
|
||||
AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
|
|
@ -111,6 +113,7 @@ from litellm.litellm_core_utils.mock_functions import (
|
|||
mock_embedding,
|
||||
mock_image_generation,
|
||||
)
|
||||
from litellm.litellm_core_utils.per_call_pricing import per_call_model_info
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
|
|
@ -167,7 +170,6 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s
|
|||
from litellm.types.utils import (
|
||||
AgenticLoopParams,
|
||||
BaseUrlRoutingHint,
|
||||
CustomPricingLiteLLMParams,
|
||||
ModelResponseStream,
|
||||
ProviderRoutingHints,
|
||||
RawRequestTypedDict,
|
||||
|
|
@ -954,26 +956,31 @@ async def acompletion(
|
|||
# no equivalent: every provider already funnels through one shared
|
||||
# handler where the loop is dispatched.
|
||||
if isinstance(response, litellm.ModelResponse):
|
||||
looped: Final = await maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={
|
||||
k: v
|
||||
for k, v in completion_kwargs.items()
|
||||
if v is not None
|
||||
and k
|
||||
not in (
|
||||
"model",
|
||||
"messages",
|
||||
"stream",
|
||||
"acompletion",
|
||||
"deployment_id",
|
||||
)
|
||||
},
|
||||
kwargs=kwargs,
|
||||
looped: Final = await bill_first_round_on_failure(
|
||||
maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={
|
||||
k: v
|
||||
for k, v in completion_kwargs.items()
|
||||
if v is not None
|
||||
and k
|
||||
not in (
|
||||
"model",
|
||||
"messages",
|
||||
"stream",
|
||||
"acompletion",
|
||||
"deployment_id",
|
||||
)
|
||||
},
|
||||
kwargs=kwargs,
|
||||
logging_obj=kwargs.get("litellm_logging_obj"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=bool(stream),
|
||||
),
|
||||
logging_obj=kwargs.get("litellm_logging_obj"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response=response,
|
||||
stream=bool(stream),
|
||||
)
|
||||
if looped is not None:
|
||||
|
|
@ -1323,7 +1330,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
|
|||
)
|
||||
from litellm.responses.bridge_check import (
|
||||
bridge_reasoning_effort,
|
||||
keep_registered_mode,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
|
|
@ -1429,46 +1435,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return values
|
||||
|
||||
|
||||
def _build_custom_pricing_entry(
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None = None,
|
||||
) -> dict:
|
||||
"""Build a complete model cost entry from kwargs and model_info.
|
||||
|
||||
Collects all CustomPricingLiteLLMParams fields present in kwargs and
|
||||
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
so that register_model() receives the full pricing configuration.
|
||||
"""
|
||||
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
|
||||
|
||||
for field_name in CustomPricingLiteLLMParams.model_fields:
|
||||
value = kwargs.get(field_name)
|
||||
if value is not None:
|
||||
entry[field_name] = value
|
||||
|
||||
if model_info and isinstance(model_info, dict):
|
||||
for key in ("mode", "supports_prompt_caching", "max_tokens"):
|
||||
if key in model_info and model_info[key] is not None:
|
||||
entry.setdefault(key, model_info[key])
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def _get_router_deployment_id(kwargs: dict) -> str | None:
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
deployment_model_info = metadata.get("model_info") or {}
|
||||
if not isinstance(deployment_model_info, dict):
|
||||
continue
|
||||
deployment_id = deployment_model_info.get("id")
|
||||
if deployment_id is not None:
|
||||
return str(deployment_id)
|
||||
return None
|
||||
|
||||
|
||||
def _record_agentic_loop_params(
|
||||
logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool
|
||||
) -> None:
|
||||
|
|
@ -1479,53 +1445,6 @@ def _record_agentic_loop_params(
|
|||
)
|
||||
|
||||
|
||||
def _register_custom_pricing_for_request(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None,
|
||||
) -> None:
|
||||
"""Register per-request custom pricing in litellm.model_cost.
|
||||
|
||||
Router-originated requests (identified by the deployment id the router puts
|
||||
in metadata) get their full pricing registered under that unique id only;
|
||||
the shared ``{provider}/{model}`` key receives the entry with pricing fields
|
||||
stripped, mirroring Router._create_deployment. This keeps one deployment's
|
||||
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
|
||||
pricing used by sibling deployments of the same backend model. Direct SDK
|
||||
calls keep the legacy behavior of registering the shared key with pricing.
|
||||
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
|
||||
deployment registration, since a call routes by its own ``model_info``.
|
||||
"""
|
||||
entry: Final = _build_custom_pricing_entry(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
shared_key: Final = f"{custom_llm_provider}/{model}"
|
||||
deployment_id: Final = _get_router_deployment_id(kwargs)
|
||||
if deployment_id is None:
|
||||
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
|
||||
return
|
||||
shared_entry: Final = keep_registered_mode(
|
||||
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
|
||||
)
|
||||
litellm.register_model(
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: shared_entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
|
||||
return ctx.metadata
|
||||
|
||||
|
|
@ -5787,9 +5706,9 @@ def completion(
|
|||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
### CUSTOM PROMPT TEMPLATE ###
|
||||
initial_prompt_value: Final = kwargs.get("initial_prompt_value", None)
|
||||
roles: Final = kwargs.get("roles", None)
|
||||
|
|
@ -5987,19 +5906,7 @@ def completion(
|
|||
"aws_region_name", None
|
||||
) # support region-based pricing for bedrock
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
|
||||
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
|
||||
custom_prompt_dict = {}
|
||||
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
|
||||
|
|
@ -6269,6 +6176,7 @@ def completion(
|
|||
*ANTHROPIC_WIF_KWARGS_KEYS,
|
||||
*OPENAI_WIF_KWARGS_KEYS,
|
||||
*OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
*AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
|
||||
"github_copilot_auth_type",
|
||||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
|
|
@ -7006,12 +6914,6 @@ def embedding(
|
|||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
|
||||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
"dimensions",
|
||||
|
|
@ -7058,21 +6960,8 @@ def embedding(
|
|||
**non_default_params,
|
||||
)
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=kwargs.get("model_info"),
|
||||
)
|
||||
|
||||
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
|
||||
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
|
||||
logging.update_environment_variables(
|
||||
model=model,
|
||||
user=user,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5250,6 +5250,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"wally": {
|
||||
"display_name": "Wally (`wally`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/wally",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"empiriolabs": {
|
||||
"display_name": "EmpirioLabs (`empiriolabs`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",
|
||||
|
|
|
|||
|
|
@ -4796,11 +4796,14 @@ class UnregisteredJWTClientBehavior(str, enum.Enum):
|
|||
- auto_register: Automatically create a new virtual key and mapping on first
|
||||
encounter. The new key has no budget/model restrictions; admins can tighten
|
||||
it later via /jwt_client/update.
|
||||
- link_only: Map an eligible existing user key or return HTTP 403, without
|
||||
creating a key. Proxy admins authenticate without a mapped key.
|
||||
"""
|
||||
|
||||
FALLBACK_TEAM_MAPPING = "fallback_team_mapping"
|
||||
REJECT = "reject"
|
||||
AUTO_REGISTER = "auto_register"
|
||||
LINK_ONLY = "link_only"
|
||||
|
||||
|
||||
class JWTIssuerConfig(LiteLLMBaseModel):
|
||||
|
|
@ -5038,7 +5041,12 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
|
|||
"What to do when virtual_key_claim_field is set but the JWT claim value "
|
||||
"has no registered mapping. 'fallback_team_mapping' (default): fall through "
|
||||
"to team-based JWT auth. 'reject': return HTTP 403. "
|
||||
"'auto_register': auto-create a virtual key and mapping on first encounter."
|
||||
"'auto_register': auto-create a virtual key and mapping on first encounter. "
|
||||
"'link_only': map an eligible existing user key or return HTTP 403, without creating a key. "
|
||||
"link_only ignores auto_register_map_existing_key; user provisioning still follows user_id_upsert. "
|
||||
"Existing-key selection ignores allowed_routes; JWTs inherit the selected key's permissions, "
|
||||
"including management routes, with request-time key restrictions still enforced. "
|
||||
"Unmapped proxy admins authenticate directly in auto_register and link_only modes."
|
||||
),
|
||||
)
|
||||
auto_register_map_existing_key: bool = Field(
|
||||
|
|
@ -5048,8 +5056,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
|
|||
"field is the user_id_jwt_field or user_email_jwt_field, the JWT claim is mapped to a virtual key the "
|
||||
"JWT-resolved user already owns instead of minting a new one. If the user owns several, the most recently created key in the "
|
||||
"JWT-resolved team (or with no team when the JWT resolves none) is chosen among keys that never "
|
||||
"expire, are not blocked, are not Admin UI session keys, were not minted by auto_register, and "
|
||||
"have no allowed_routes or include llm_api_routes. Otherwise a new key is minted as usual."
|
||||
"expire, are not blocked, are not Admin UI session keys, and were not minted by auto_register. "
|
||||
"allowed_routes does not affect key selection. JWTs inherit the selected key's permissions, "
|
||||
"including management routes; request-time key restrictions still apply. "
|
||||
"Otherwise a new key is minted as usual."
|
||||
),
|
||||
)
|
||||
routing_overrides: list[JWTRoutingOverride] | None = Field(
|
||||
|
|
|
|||
|
|
@ -2,31 +2,22 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from typing import Final, Protocol
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
from litellm.proxy.auth.authorization import ReadScope, read_scope_bind_params
|
||||
from litellm.proxy.utils import query_raw_within_statement_timeout
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
UNKNOWN_CALL_TYPE: Final = "Unknown"
|
||||
INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value)
|
||||
|
||||
|
||||
class _SupportsQueryRaw(Protocol):
|
||||
"""The single database operation the cache-activity queries issue."""
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> Sequence[object]: ...
|
||||
|
||||
|
||||
class _SupportsRawQueryDb(Protocol):
|
||||
"""A prisma client handle, narrowed to the raw-query surface used here."""
|
||||
|
||||
@property
|
||||
def db(self) -> _SupportsQueryRaw: ...
|
||||
|
||||
|
||||
class CacheActivityGroup(LiteLLMBaseModel):
|
||||
call_type: str
|
||||
api_requests: int
|
||||
|
|
@ -174,7 +165,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals:
|
|||
|
||||
|
||||
async def get_cache_activity(
|
||||
prisma_client: _SupportsRawQueryDb,
|
||||
prisma_client: "PrismaClient",
|
||||
read_scope: ReadScope,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
|
|
@ -185,14 +176,46 @@ async def get_cache_activity(
|
|||
models_json: Final = json.dumps(list(models))
|
||||
scope_params: Final = read_scope_bind_params(read_scope)
|
||||
group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather(
|
||||
prisma_client.db.query_raw(
|
||||
GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params
|
||||
query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
GROUPS_SQL,
|
||||
start_date,
|
||||
end_date,
|
||||
key_aliases_json,
|
||||
models_json,
|
||||
INFO_ROUTES_JSON,
|
||||
*scope_params,
|
||||
what="Cache activity",
|
||||
),
|
||||
prisma_client.db.query_raw(
|
||||
ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params
|
||||
query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
ERROR_BREAKDOWN_SQL,
|
||||
start_date,
|
||||
end_date,
|
||||
key_aliases_json,
|
||||
models_json,
|
||||
INFO_ROUTES_JSON,
|
||||
*scope_params,
|
||||
what="Cache activity",
|
||||
),
|
||||
query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
KEY_ALIAS_OPTIONS_SQL,
|
||||
start_date,
|
||||
end_date,
|
||||
INFO_ROUTES_JSON,
|
||||
*scope_params,
|
||||
what="Cache activity",
|
||||
),
|
||||
query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
MODEL_OPTIONS_SQL,
|
||||
start_date,
|
||||
end_date,
|
||||
INFO_ROUTES_JSON,
|
||||
*scope_params,
|
||||
what="Cache activity",
|
||||
),
|
||||
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params),
|
||||
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params),
|
||||
)
|
||||
groups: Final = _groups_adapter.validate_python(group_rows or [])
|
||||
return CacheActivityResponse(
|
||||
|
|
|
|||
30
litellm/proxy/anthropic_endpoints/answered_status.py
Normal file
30
litellm/proxy/anthropic_endpoints/answered_status.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
from typing import Final, TypeVar
|
||||
|
||||
from litellm.exceptions import InternalServerError, MidStreamFallbackError
|
||||
|
||||
_Raised = TypeVar("_Raised", bound=Exception)
|
||||
|
||||
|
||||
def _provider_status_code(exc: Exception) -> int | None:
|
||||
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
|
||||
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
|
||||
unwrapped = unwrapped.original_exception
|
||||
match unwrapped:
|
||||
case InternalServerError(provider_status_code=int() as provider_status):
|
||||
return provider_status
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised:
|
||||
answered: Final = type(exc).__new__(type(exc), *exc.args)
|
||||
answered.__dict__.update({**exc.__dict__, attribute: value})
|
||||
answered.__cause__ = exc.__cause__
|
||||
answered.__context__ = exc.__context__
|
||||
answered.__suppress_context__ = exc.__suppress_context__
|
||||
return answered.with_traceback(exc.__traceback__)
|
||||
|
||||
|
||||
def with_answered_status(exc: Exception) -> Exception:
|
||||
provider_status_code: Final = _provider_status_code(exc)
|
||||
return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code)
|
||||
|
|
@ -16,12 +16,14 @@ from litellm.anthropic_interface.exceptions import (
|
|||
AnthropicExceptionMapping,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
)
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
|
|
@ -48,6 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
)
|
||||
from litellm.proxy.common_utils.token_count_log import rejected_token_count
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
forwarded_anthropic_oauth_token,
|
||||
should_auto_drop_params_for_agentic_cli,
|
||||
|
|
@ -225,15 +228,18 @@ async def anthropic_response(
|
|||
request_id=request.headers.get("x-request-id"),
|
||||
)
|
||||
return JSONResponse(status_code=e.status_code, content=body)
|
||||
except Exception as e:
|
||||
except Exception as raised:
|
||||
answered: Final = with_answered_status(raised)
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=answered,
|
||||
request_data=base_llm_response_processor.data,
|
||||
)
|
||||
log_llm_api_exception(e, base_llm_response_processor.litellm_call_id)
|
||||
log_llm_api_exception(answered, base_llm_response_processor.litellm_call_id)
|
||||
|
||||
if isinstance(e, ProxyException):
|
||||
if isinstance(answered, ProxyException):
|
||||
return _anthropic_error_json_response(
|
||||
with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request
|
||||
with_litellm_call_id(answered, base_llm_response_processor.litellm_call_id), request
|
||||
)
|
||||
|
||||
# Extract model_id from request metadata (same as success path)
|
||||
|
|
@ -247,23 +253,25 @@ async def anthropic_response(
|
|||
call_id=base_llm_response_processor.litellm_call_id,
|
||||
model_id=model_id,
|
||||
version=version,
|
||||
response_cost=0,
|
||||
response_cost=(
|
||||
failure_row_response_cost(base_llm_response_processor.data) if is_agentic_followup_failure(e) else 0
|
||||
),
|
||||
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
|
||||
request_data=data,
|
||||
timeout=getattr(e, "timeout", None),
|
||||
timeout=getattr(answered, "timeout", None),
|
||||
litellm_logging_obj=None,
|
||||
)
|
||||
|
||||
if isinstance(e, HTTPException):
|
||||
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
|
||||
if isinstance(answered, HTTPException):
|
||||
return _anthropic_error_json_response(proxy_exception_from_http_exception(answered, headers), request)
|
||||
|
||||
error_msg: Final = f"{e}"
|
||||
error_msg: Final = f"{answered}"
|
||||
return _anthropic_error_json_response(
|
||||
ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
code=error_status_code(e, 500),
|
||||
message=getattr(answered, "message", error_msg),
|
||||
type=openai_error_type(answered, error_status_code(answered, 500)),
|
||||
param=openai_error_param(answered),
|
||||
code=error_status_code(answered, 500),
|
||||
headers=headers,
|
||||
),
|
||||
request,
|
||||
|
|
@ -383,8 +391,9 @@ async def count_tokens(
|
|||
except ProxyException as e:
|
||||
return _anthropic_error_json_response(e, request)
|
||||
except Exception as e:
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
|
||||
answered: Final = with_answered_status(e)
|
||||
log_llm_api_exception(answered, litellm_call_id)
|
||||
return _anthropic_error_json_response(proxy_exception_from_route_error(answered), request)
|
||||
else:
|
||||
return {
|
||||
"input_tokens": token_response.total_tokens,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import re
|
|||
import secrets
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NamedTuple, Protocol, Union, cast
|
||||
|
||||
|
|
@ -1576,10 +1577,14 @@ _JWT_AUTH_DISABLED_HINT = (
|
|||
)
|
||||
|
||||
|
||||
class _JWTKeyMappingFailure(Enum):
|
||||
NO_ELIGIBLE_KEY = "JWT Key Mapping: No eligible existing virtual key for this identity. Access denied."
|
||||
|
||||
|
||||
class _PendingAutoRegister(NamedTuple):
|
||||
"""
|
||||
Signal returned by ``_resolve_jwt_to_virtual_key`` when the JWT's claim is
|
||||
unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER.
|
||||
unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER or LINK_ONLY.
|
||||
|
||||
The caller MUST run standard ``JWTAuthManager.auth_builder`` to apply RBAC,
|
||||
scope mappings, ``custom_validate``, and ``user_allowed_email_domain``
|
||||
|
|
@ -1593,23 +1598,22 @@ class _PendingAutoRegister(NamedTuple):
|
|||
claim_value: str
|
||||
cache_key: str
|
||||
jwt_issuer: str | None = None
|
||||
behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER
|
||||
|
||||
|
||||
def _claim_identifies_user(jwt_handler: JWTHandler, claim_field: str, jwt_issuer: str | None) -> bool:
|
||||
if not jwt_handler.litellm_jwtauth.auto_register_map_existing_key:
|
||||
return False
|
||||
if jwt_handler.litellm_jwtauth.is_user_identity_claim(claim_field, jwt_issuer):
|
||||
return True
|
||||
verbose_proxy_logger.warning(
|
||||
"JWT Key Mapping (auto_register_map_existing_key): claim '%s' is not the user_id or user_email JWT field "
|
||||
"and may be shared by several users, so a new key is minted instead of reusing one the user owns.",
|
||||
"JWT Key Mapping: claim '%s' is not the user_id or user_email JWT field "
|
||||
"and may be shared by several users, so existing user keys cannot be linked automatically.",
|
||||
claim_field,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def _reusable_key_hash_for_user(prisma_client: PrismaClient, user_id: str, team_id: str | None) -> str | None:
|
||||
key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_llm_api_key(user_id, team_id)
|
||||
key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_key(user_id, team_id)
|
||||
return None if key is None else key.token
|
||||
|
||||
|
||||
|
|
@ -1628,11 +1632,13 @@ async def _auto_register_jwt_mapping(
|
|||
org_id: str | None = None,
|
||||
end_user_id: str | None = None,
|
||||
agent_id: str | None = None,
|
||||
) -> UserAPIKeyAuth | None:
|
||||
behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER,
|
||||
) -> UserAPIKeyAuth | _JWTKeyMappingFailure | None:
|
||||
"""
|
||||
Auto-register: create a new virtual key + mapping for an unrecognised JWT
|
||||
claim value, or point the mapping at a key the resolved user already owns
|
||||
when ``auto_register_map_existing_key`` is set. ``team_id`` and ``user_id``
|
||||
when ``auto_register_map_existing_key`` is set. LINK_ONLY always searches
|
||||
for an existing key and rejects when none is eligible. ``team_id`` and ``user_id``
|
||||
must come from a successful ``JWTAuthManager.auth_builder`` run — they
|
||||
encode the JWT identity AFTER
|
||||
RBAC/scope/custom_validate/email-domain policy has been enforced. The key
|
||||
|
|
@ -1650,11 +1656,21 @@ async def _auto_register_jwt_mapping(
|
|||
generate_key_helper_fn,
|
||||
)
|
||||
|
||||
link_only: Final = behavior == UnregisteredJWTClientBehavior.LINK_ONLY
|
||||
reuse_existing: Final = link_only or jwt_handler.litellm_jwtauth.auto_register_map_existing_key
|
||||
existing_token_hash: Final = (
|
||||
await _reusable_key_hash_for_user(prisma_client, user_id, team_id)
|
||||
if user_id is not None and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer)
|
||||
await _reusable_key_hash_for_user(
|
||||
prisma_client,
|
||||
user_id,
|
||||
team_id,
|
||||
)
|
||||
if reuse_existing
|
||||
and user_id is not None
|
||||
and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer)
|
||||
else None
|
||||
)
|
||||
if link_only and existing_token_hash is None:
|
||||
return _JWTKeyMappingFailure.NO_ELIGIBLE_KEY
|
||||
minted: Final = existing_token_hash is None
|
||||
if existing_token_hash is not None:
|
||||
token_hash = existing_token_hash
|
||||
|
|
@ -1691,8 +1707,8 @@ async def _auto_register_jwt_mapping(
|
|||
"jwt_claim_name": virtual_key_claim_field,
|
||||
"jwt_claim_value": claim_value,
|
||||
"token": token_hash,
|
||||
"created_by": "auto_register",
|
||||
"updated_by": "auto_register",
|
||||
"created_by": behavior.value,
|
||||
"updated_by": behavior.value,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -1704,8 +1720,8 @@ async def _auto_register_jwt_mapping(
|
|||
# the DB and the cleartext is in memory on this request. Delete it
|
||||
# so orphans don't accumulate under sustained concurrency.
|
||||
verbose_proxy_logger.debug(
|
||||
"JWT Key Mapping (auto_register): unique conflict on create — "
|
||||
"deleting orphaned virtual key and fetching winner's mapping for %s='%s'.",
|
||||
"JWT Key Mapping (%s): unique conflict on create, fetching winner's mapping for %s='%s'.",
|
||||
behavior.value,
|
||||
virtual_key_claim_field,
|
||||
claim_value,
|
||||
)
|
||||
|
|
@ -1735,7 +1751,7 @@ async def _auto_register_jwt_mapping(
|
|||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=(
|
||||
"JWT Key Mapping: AUTO_REGISTER race resolution failed — "
|
||||
f"JWT Key Mapping: {behavior.value.upper()} race resolution failed — "
|
||||
"winner's mapping was concurrently removed. Retry the request."
|
||||
),
|
||||
)
|
||||
|
|
@ -1749,7 +1765,8 @@ async def _auto_register_jwt_mapping(
|
|||
)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
"JWT Key Mapping (auto_register): %s virtual key for %s='%s'.",
|
||||
"JWT Key Mapping (%s): %s virtual key for %s='%s'.",
|
||||
behavior.value,
|
||||
"created new" if minted else "mapped existing",
|
||||
virtual_key_claim_field,
|
||||
claim_value,
|
||||
|
|
@ -1821,7 +1838,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
- ``UserAPIKeyAuth``: a resolved virtual key (cache hit or DB hit). The
|
||||
caller may use this directly; JWT policy has been enforced previously
|
||||
(at key-creation time or, for cached results, before caching).
|
||||
- ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER.
|
||||
- ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER or LINK_ONLY.
|
||||
The caller MUST run ``JWTAuthManager.auth_builder`` to enforce JWT
|
||||
policy (RBAC, scope, custom_validate, email-domain), then invoke
|
||||
``_auto_register_jwt_mapping`` with the validated team_id/user_id.
|
||||
|
|
@ -1829,7 +1846,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
The caller falls through to standard team-based JWT auth (which itself
|
||||
enforces full JWT policy via auth_builder).
|
||||
- Raises HTTPException: REJECT policy hit, missing claim under
|
||||
REJECT/AUTO_REGISTER, or other policy violations.
|
||||
REJECT/AUTO_REGISTER/LINK_ONLY, or other policy violations.
|
||||
"""
|
||||
raw_issuer: Final = jwt_claims.get(JWTHandler.LITELLM_JWT_ISSUER_CLAIM)
|
||||
normalized_issuer: Final = raw_issuer if isinstance(raw_issuer, str) else None
|
||||
|
|
@ -1856,6 +1873,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
if behavior in (
|
||||
UnregisteredJWTClientBehavior.REJECT,
|
||||
UnregisteredJWTClientBehavior.AUTO_REGISTER,
|
||||
UnregisteredJWTClientBehavior.LINK_ONLY,
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
|
|
@ -1887,7 +1905,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
status_code=403,
|
||||
detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.",
|
||||
)
|
||||
if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER:
|
||||
if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY):
|
||||
# Stale sentinel written under a prior fallback_team_mapping config —
|
||||
# evict it and defer auto-register to after auth_builder runs. Raise
|
||||
# the same 500 as the fresh-path AUTO_REGISTER branch when there is
|
||||
|
|
@ -1897,7 +1915,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=(
|
||||
"JWT Key Mapping: AUTO_REGISTER requires a database connection. "
|
||||
f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. "
|
||||
"Configure a database or change unregistered_jwt_client_behavior."
|
||||
),
|
||||
)
|
||||
|
|
@ -1907,6 +1925,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
claim_value=str(claim_value),
|
||||
cache_key=cache_key,
|
||||
jwt_issuer=normalized_issuer,
|
||||
behavior=behavior,
|
||||
)
|
||||
return None
|
||||
elif cached_mapping is not None:
|
||||
|
|
@ -1967,12 +1986,12 @@ async def _resolve_jwt_to_virtual_key(
|
|||
detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.",
|
||||
)
|
||||
|
||||
if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER:
|
||||
if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY):
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=(
|
||||
"JWT Key Mapping: AUTO_REGISTER requires a database connection. "
|
||||
f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. "
|
||||
"Configure a database or change unregistered_jwt_client_behavior."
|
||||
),
|
||||
)
|
||||
|
|
@ -1985,6 +2004,7 @@ async def _resolve_jwt_to_virtual_key(
|
|||
claim_value=str(claim_value),
|
||||
cache_key=cache_key,
|
||||
jwt_issuer=normalized_issuer,
|
||||
behavior=behavior,
|
||||
)
|
||||
|
||||
# FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the
|
||||
|
|
@ -2477,10 +2497,13 @@ async def user_api_key_auth_builder(
|
|||
org_id=org_id,
|
||||
end_user_id=end_user_id,
|
||||
agent_id=agent_id,
|
||||
behavior=pending_auto_register.behavior,
|
||||
)
|
||||
if pending_auto_register is not None and prisma_client is not None
|
||||
else None
|
||||
)
|
||||
if isinstance(auto_registered, _JWTKeyMappingFailure):
|
||||
raise HTTPException(status_code=403, detail=auto_registered.value)
|
||||
if auto_registered is not None:
|
||||
auto_registered.jwt_claims = jwt_claims
|
||||
auto_registered.user_email = user_email
|
||||
|
|
@ -2496,7 +2519,13 @@ async def user_api_key_auth_builder(
|
|||
|
||||
falls_through_to_key_checks: Final = (
|
||||
auto_registered is not None
|
||||
and jwt_handler.litellm_jwtauth.auto_register_map_existing_key
|
||||
and (
|
||||
jwt_handler.litellm_jwtauth.auto_register_map_existing_key
|
||||
or (
|
||||
pending_auto_register is not None
|
||||
and pending_auto_register.behavior == UnregisteredJWTClientBehavior.LINK_ONLY
|
||||
)
|
||||
)
|
||||
and master_key is not None
|
||||
)
|
||||
if not falls_through_to_key_checks:
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ from litellm.constants import (
|
|||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
bug_report_notice,
|
||||
|
|
@ -132,6 +133,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
|
|||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import (
|
||||
endpoint_unsupported_by_model,
|
||||
|
|
@ -272,6 +274,7 @@ if TYPE_CHECKING:
|
|||
ProxyConfig = _ProxyConfig
|
||||
else:
|
||||
ProxyConfig = Any
|
||||
from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status
|
||||
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
|
||||
AnthropicStreamModelRestamper,
|
||||
)
|
||||
|
|
@ -1272,6 +1275,21 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
|
|||
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
|
||||
|
||||
|
||||
def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
_, error_obj = sse_error_payload(exc)
|
||||
return _sse_error_frames(error_obj)
|
||||
|
||||
|
||||
def _anthropic_late_failure_frames(exc: Exception) -> tuple[str, ...]:
|
||||
answered: Final = proxy_exception_from_route_error(with_answered_status(exc))
|
||||
return (
|
||||
anthropic_error_sse_frame(
|
||||
status_code=error_status_code(answered, status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
raw_message=answered.message,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
"""`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384)."""
|
||||
return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS})
|
||||
|
|
@ -1520,6 +1538,7 @@ async def open_sse_before_first_byte(
|
|||
ping_interval_seconds: float | str | None,
|
||||
media_type: str = "text/event-stream",
|
||||
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
|
||||
late_failure_frames: Callable[[Exception], tuple[str, ...]] = _openai_late_failure_frames,
|
||||
) -> _LateResponseT | StreamingResponse:
|
||||
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
|
||||
|
||||
|
|
@ -1565,8 +1584,7 @@ async def open_sse_before_first_byte(
|
|||
# would never fire and the failure would go unaudited. The hook
|
||||
# also gets to sanitize what reaches the client, by returning or
|
||||
# raising a replacement, so its answer decides the frame.
|
||||
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
|
||||
for frame in _sse_error_frames(error_obj):
|
||||
for frame in late_failure_frames(await _sanitized_late_failure(exc, on_late_failure)):
|
||||
yield frame.encode()
|
||||
return
|
||||
async for chunk in _relay_late_response(produced):
|
||||
|
|
@ -2982,13 +3000,14 @@ class ProxyBaseLLMRequestProcessing:
|
|||
idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets
|
||||
``litellm_settings.sse_keepalive_ping_interval_seconds``.
|
||||
"""
|
||||
answers_as_anthropic: Final = route_type == "anthropic_messages"
|
||||
|
||||
async def _audit_late_failure(exc: Exception) -> HTTPException | None:
|
||||
# Once a keepalive is on the wire this can no longer raise, so the
|
||||
# caller's `except` never runs its own post_call_failure_hook.
|
||||
return await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exc,
|
||||
original_exception=with_answered_status(exc) if answers_as_anthropic else exc,
|
||||
request_data=self.data,
|
||||
dispatched_call_type=self.route_type,
|
||||
)
|
||||
|
|
@ -3017,6 +3036,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
),
|
||||
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
|
||||
on_late_failure=_audit_late_failure,
|
||||
late_failure_frames=_anthropic_late_failure_frames if answers_as_anthropic else _openai_late_failure_frames,
|
||||
)
|
||||
|
||||
async def _process_llm_request(
|
||||
|
|
@ -4384,6 +4404,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
|
||||
log_llm_api_exception(e, self.litellm_call_id)
|
||||
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
|
||||
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
|
||||
# Allow callbacks to transform the error response
|
||||
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -4420,7 +4441,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
model_id=model_id,
|
||||
version=version,
|
||||
response_cost=(
|
||||
_litellm_logging_obj.finalized_cost.amount
|
||||
failure_row_response_cost(self.data)
|
||||
if failed_in_agentic_follow_up
|
||||
else _litellm_logging_obj.finalized_cost.amount
|
||||
if isinstance(_litellm_logging_obj, LiteLLMLoggingObj)
|
||||
and _litellm_logging_obj.finalized_cost is not None
|
||||
else 0
|
||||
|
|
@ -4641,6 +4664,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request: Request | None = None,
|
||||
flush_tail: Callable[[], bytes] | None = None,
|
||||
seal_open_frame: Callable[[bytes], str] | None = None,
|
||||
answer_status: Callable[[Exception], Exception] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Shared streaming data generator: runs proxy iterator hook, per-chunk hook,
|
||||
|
|
@ -4656,6 +4680,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
relays raw upstream bytes, so an upstream that hangs up mid-frame leaves the
|
||||
client inside an open frame, where an error frame would be swallowed or
|
||||
misparsed instead of raised.
|
||||
|
||||
``answer_status`` gives a failure the status the route answers it with
|
||||
before the failure hook records it.
|
||||
"""
|
||||
verbose_proxy_logger.debug("inside generator")
|
||||
# Resolve per-stream (not per-chunk) whether the heavy per-chunk path
|
||||
|
|
@ -4752,6 +4779,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e
|
||||
)
|
||||
answered: Final = e if answer_status is None else answer_status(e)
|
||||
already_reported: Final = (
|
||||
isinstance(stream_logging_obj, LiteLLMLoggingObj)
|
||||
and not stream_logging_obj.claim_stream_failure_report()
|
||||
|
|
@ -4761,24 +4789,25 @@ class ProxyBaseLLMRequestProcessing:
|
|||
if already_reported
|
||||
else await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
original_exception=answered,
|
||||
request_data=request_data,
|
||||
)
|
||||
)
|
||||
if transformed_exception is not None:
|
||||
e = transformed_exception
|
||||
failure: Final = answered if transformed_exception is None else transformed_exception
|
||||
verbose_proxy_logger.debug(
|
||||
"An error occurred: %s",
|
||||
e,
|
||||
failure,
|
||||
)
|
||||
|
||||
if isinstance(e, HTTPException):
|
||||
raise e
|
||||
stream_error_status: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||
if isinstance(failure, HTTPException):
|
||||
if failure is e:
|
||||
raise
|
||||
raise failure from e
|
||||
stream_error_status: Final = error_status_code(failure, status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||
proxy_exception: Final = ProxyException(
|
||||
message=redact_internal_details_from_client_message(getattr(e, "message", str(e))),
|
||||
type=openai_error_type(e, stream_error_status),
|
||||
param=openai_error_param(e),
|
||||
message=redact_internal_details_from_client_message(getattr(failure, "message", str(failure))),
|
||||
type=openai_error_type(failure, stream_error_status),
|
||||
param=openai_error_param(failure),
|
||||
code=stream_error_status,
|
||||
)
|
||||
stream_completed = True
|
||||
|
|
@ -4834,6 +4863,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
request=request,
|
||||
flush_tail=None if restamper is None else restamper.flush,
|
||||
seal_open_frame=seal_open_sse_frame,
|
||||
answer_status=with_answered_status,
|
||||
)
|
||||
|
||||
@overload
|
||||
|
|
|
|||
90
litellm/proxy/common_utils/fips.py
Normal file
90
litellm/proxy/common_utils/fips.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
FIPS_MODE_SETTING: Final = "fips_mode"
|
||||
_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
|
||||
|
||||
|
||||
class FipsModeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None:
|
||||
"""general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including
|
||||
a key with no value."""
|
||||
if FIPS_MODE_SETTING not in general_settings:
|
||||
return False
|
||||
value: Final = general_settings[FIPS_MODE_SETTING]
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str_to_bool(value) if isinstance(value, str) else None
|
||||
|
||||
|
||||
def openssl_enforces_fips() -> bool:
|
||||
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5."""
|
||||
try:
|
||||
import _hashlib
|
||||
|
||||
if _hashlib.get_fips_mode() != 1:
|
||||
return False
|
||||
except (ImportError, AttributeError, ValueError):
|
||||
return False
|
||||
try:
|
||||
_hashlib.new("md5", b"", usedforsecurity=True)
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def cryptography_enforces_fips() -> bool:
|
||||
"""The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256."""
|
||||
try:
|
||||
from cryptography.exceptions import InternalError, UnsupportedAlgorithm
|
||||
from cryptography.hazmat.bindings._rust import openssl as rust_openssl
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
except ImportError:
|
||||
return False
|
||||
try:
|
||||
if not rust_openssl.is_fips_enabled():
|
||||
return False
|
||||
hashes.Hash(hashes.SHA256()).finalize()
|
||||
except (AttributeError, InternalError, UnsupportedAlgorithm):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None:
|
||||
"""Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the
|
||||
cryptography library's OpenSSL does not enforce FIPS. None when it may start."""
|
||||
setting: Final = _fips_mode_setting(general_settings)
|
||||
if setting is None:
|
||||
value: Final = general_settings.get(FIPS_MODE_SETTING)
|
||||
if value is None:
|
||||
return (
|
||||
f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. "
|
||||
"Set it to true or false, or remove it."
|
||||
)
|
||||
return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it."
|
||||
if not setting:
|
||||
return None
|
||||
if not openssl_enforces_fips():
|
||||
return (
|
||||
f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS "
|
||||
f"provider active and default_properties = fips=yes, or turn {_SETTING} off."
|
||||
)
|
||||
if not cryptography_enforces_fips():
|
||||
return (
|
||||
f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it "
|
||||
"against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn "
|
||||
f"{_SETTING} off."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
|
||||
"""Raise FipsModeError with the fips_mode_refusal reason, if there is one."""
|
||||
refusal: Final = fips_mode_refusal(general_settings)
|
||||
if refusal is not None:
|
||||
raise FipsModeError(refusal)
|
||||
|
|
@ -32,6 +32,7 @@ class _ResolvedDeprecation:
|
|||
litellm_model: str | None
|
||||
litellm_provider: str | None
|
||||
copied_from_another_row: bool = False
|
||||
successor_model: str | None = None
|
||||
|
||||
|
||||
def _parse_deprecation_date(raw_value: object) -> date | None:
|
||||
|
|
@ -58,10 +59,11 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
|
|||
entry: Final = litellm.model_cost.get(model_key)
|
||||
if not isinstance(entry, Mapping):
|
||||
return None
|
||||
parsed: Final = _parse_deprecation_date(entry.get("deprecation_date"))
|
||||
fields: Final[Mapping[str, object]] = entry
|
||||
parsed: Final = _parse_deprecation_date(fields.get("deprecation_date"))
|
||||
if parsed is None:
|
||||
return None
|
||||
provider: Final = entry.get("litellm_provider")
|
||||
provider: Final = fields.get("litellm_provider")
|
||||
return _ResolvedDeprecation(
|
||||
deprecation_date=parsed,
|
||||
litellm_model=model_key,
|
||||
|
|
@ -69,6 +71,7 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
|
|||
copied_from_another_row=isinstance(entry, AliasedCostEntry)
|
||||
and "deprecation_date" not in entry.registered_fields
|
||||
and model_key not in catalog,
|
||||
successor_model=_successor_model(fields.get("successor_model")),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -127,15 +130,6 @@ def _resolve_deployment_deprecation(
|
|||
litellm_params: Final = _mapping_field(deployment, "litellm_params")
|
||||
model_key: Final = _as_model_key(litellm_params.get("model"))
|
||||
|
||||
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
|
||||
if override is not None:
|
||||
override_provider: Final = model_info.get("litellm_provider")
|
||||
return _ResolvedDeprecation(
|
||||
deprecation_date=override,
|
||||
litellm_model=model_key,
|
||||
litellm_provider=override_provider if isinstance(override_provider, str) else None,
|
||||
)
|
||||
|
||||
provider: Final = _deployment_provider(litellm_params)
|
||||
base_model: Final = _as_model_key(model_info.get("base_model"))
|
||||
model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key
|
||||
|
|
@ -150,7 +144,18 @@ def _resolve_deployment_deprecation(
|
|||
if pick is not None
|
||||
)
|
||||
own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row)
|
||||
return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
|
||||
from_catalog: Final = min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
|
||||
|
||||
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
|
||||
if override is None:
|
||||
return from_catalog
|
||||
override_provider: Final = model_info.get("litellm_provider")
|
||||
return _ResolvedDeprecation(
|
||||
deprecation_date=override,
|
||||
litellm_model=model_key,
|
||||
litellm_provider=override_provider if isinstance(override_provider, str) else None,
|
||||
successor_model=from_catalog.successor_model if from_catalog is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus:
|
||||
|
|
@ -182,7 +187,7 @@ def _build_info(
|
|||
today: date,
|
||||
warn_within_days: int,
|
||||
catalog: Mapping[str, object],
|
||||
) -> ModelDeprecationInfo | None:
|
||||
) -> _DeploymentInfo | None:
|
||||
model_name: Final = deployment.get("model_name")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
return None
|
||||
|
|
@ -192,21 +197,40 @@ def _build_info(
|
|||
return None
|
||||
|
||||
days_until: Final = (resolved.deprecation_date - today).days
|
||||
return ModelDeprecationInfo(
|
||||
model_name=model_name,
|
||||
litellm_model=resolved.litellm_model,
|
||||
deprecation_date=resolved.deprecation_date,
|
||||
days_until_deprecation=days_until,
|
||||
status=_classify(days_until, warn_within_days),
|
||||
litellm_provider=resolved.litellm_provider,
|
||||
successor_model=_checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router),
|
||||
configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
|
||||
return _DeploymentInfo(
|
||||
info=ModelDeprecationInfo(
|
||||
model_name=model_name,
|
||||
litellm_model=resolved.litellm_model,
|
||||
deprecation_date=resolved.deprecation_date,
|
||||
days_until_deprecation=days_until,
|
||||
status=_classify(days_until, warn_within_days),
|
||||
litellm_provider=resolved.litellm_provider,
|
||||
successor_model=configured or resolved.successor_model,
|
||||
),
|
||||
configured_successor=configured is not None,
|
||||
)
|
||||
|
||||
|
||||
def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
|
||||
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names"""
|
||||
first: Final = group[0]
|
||||
successors: Final = tuple(model.successor_model for model in group if model.successor_model is not None)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DeploymentInfo:
|
||||
info: ModelDeprecationInfo
|
||||
configured_successor: bool
|
||||
|
||||
|
||||
def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
|
||||
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names,
|
||||
a successor configured in model_info ahead of one the cost map supplies"""
|
||||
first: Final = group[0].info
|
||||
tiers: Final = tuple(
|
||||
tuple(
|
||||
deployment.info.successor_model
|
||||
for deployment in group
|
||||
if deployment.configured_successor is configured and deployment.info.successor_model is not None
|
||||
)
|
||||
for configured in (True, False)
|
||||
)
|
||||
successors: Final = next((tier for tier in tiers if tier), ())
|
||||
if not successors:
|
||||
return first
|
||||
if len(frozenset(successors)) > 1:
|
||||
|
|
@ -220,13 +244,13 @@ def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
|
|||
|
||||
|
||||
def _dedupe(
|
||||
models: Sequence[ModelDeprecationInfo],
|
||||
models: Sequence[_DeploymentInfo],
|
||||
) -> tuple[ModelDeprecationInfo, ...]:
|
||||
"""Report a model group carrying the same date on several deployments once"""
|
||||
ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date))
|
||||
ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date))
|
||||
return tuple(
|
||||
_merge_group(tuple(group))
|
||||
for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date))
|
||||
for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date))
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -920,8 +920,6 @@ def update_db_credential(
|
|||
# update model info
|
||||
if encrypted_credential.credential_info:
|
||||
"""Update credential info"""
|
||||
if "credential_info" not in merged_credential.credential_info:
|
||||
merged_credential.credential_info = {}
|
||||
merged_credential.credential_info.update(encrypted_credential.credential_info)
|
||||
|
||||
return merged_credential
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
|
|||
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
|
||||
_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
|
||||
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
|
||||
_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014"
|
||||
_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout"
|
||||
|
||||
|
||||
def exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
|
|
@ -302,6 +304,17 @@ class PrismaDBExceptionHandler:
|
|||
return True
|
||||
return _LOCK_TIMEOUT_PHRASE in str(e).lower()
|
||||
|
||||
@staticmethod
|
||||
def is_statement_timeout_error(e: Exception) -> bool:
|
||||
"""True iff Postgres cancelled a statement for exceeding ``statement_timeout`` (SQLSTATE 57014)."""
|
||||
import prisma
|
||||
|
||||
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
if PrismaDBExceptionHandler.postgres_sqlstate(e) == _STATEMENT_TIMEOUT_SQLSTATE:
|
||||
return True
|
||||
return _STATEMENT_TIMEOUT_PHRASE in str(e).lower()
|
||||
|
||||
@staticmethod
|
||||
def postgres_sqlstate(e: Exception) -> str | None:
|
||||
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import json
|
|||
import time
|
||||
from collections import Counter, OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeGuard
|
||||
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
|
@ -507,6 +507,8 @@ class _CompressionResult:
|
|||
|
||||
|
||||
class CompresrGuardrail(CustomGuardrail):
|
||||
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({COMPRESR_RETRIEVE_TOOL_NAME})
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_base: str | None = None,
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
ensure_response_additional_headers,
|
||||
response_has_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
|
|
@ -64,7 +63,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
|
||||
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
estimate_media_block_tokens,
|
||||
offload_token_count,
|
||||
split_embedding_input_tokens,
|
||||
)
|
||||
from litellm.llms.azure.ptu_capacity import normalized_tokens
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
|
|
@ -156,6 +159,12 @@ def _resolve_media_models(router: "Router | None", model: str, team_id: str | No
|
|||
return tuple(dict.fromkeys(router.registry().names(model, kind="litellm_base_models", team_id=team_id))) or (model,)
|
||||
|
||||
|
||||
def _media_models(
|
||||
resolver: Callable[[str, str | None], tuple[str, ...]] | None, model: str, team_id: str | None
|
||||
) -> tuple[str, ...]:
|
||||
return resolver(model, team_id) if resolver is not None else (model,)
|
||||
|
||||
|
||||
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -1378,20 +1387,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
media_source: Final = messages if messages is not None else original_input
|
||||
is_embedding_call: Final = call_type in EMBEDDING_API_CALL_TYPES
|
||||
projected, media = (
|
||||
split_media_for_tpm(media_source)
|
||||
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
|
||||
else (original_input, ())
|
||||
)
|
||||
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
|
||||
fallback_model: Final = request_model if isinstance(request_model, str) else None
|
||||
resolved_models: Final = (
|
||||
media_model_resolver(fallback_model or "", team_id)
|
||||
if media and media_model_resolver is not None
|
||||
else (fallback_model or "",)
|
||||
_media_models(media_model_resolver, fallback_model or "", team_id) if media else (fallback_model or "",)
|
||||
)
|
||||
media_tokens: Final = sum(
|
||||
self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
|
||||
embedding_text, embedding_media_tokens = (
|
||||
split_embedding_input_tokens(
|
||||
original_input,
|
||||
lambda: _media_models(media_model_resolver, fallback_model or "", team_id),
|
||||
fallback_model,
|
||||
)
|
||||
if include_media and is_embedding_call
|
||||
else (original_input, 0)
|
||||
)
|
||||
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else embedding_text
|
||||
media_tokens: Final = embedding_media_tokens + sum(
|
||||
estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
|
||||
)
|
||||
|
||||
total_chars: Final = (
|
||||
|
|
@ -1436,24 +1453,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
|
||||
|
||||
@staticmethod
|
||||
def _estimate_media_block_tokens(
|
||||
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
|
||||
) -> int:
|
||||
if block.get("type") == "input_audio":
|
||||
audio: Final = _callback_kwargs_mapping(block.get("input_audio"))
|
||||
data: Final = audio.get("data")
|
||||
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
|
||||
image: Final = _callback_kwargs_mapping(block.get("image_url"))
|
||||
url: Final = image.get("url")
|
||||
detail: Final = image.get("detail")
|
||||
return estimate_image_tokens(
|
||||
models,
|
||||
url if isinstance(url, str) else "",
|
||||
detail if isinstance(detail, str) else "auto",
|
||||
fallback_model=fallback_model,
|
||||
)
|
||||
|
||||
def _is_redis_cluster(self) -> bool:
|
||||
"""
|
||||
Check if the dual cache is using Redis cluster.
|
||||
|
|
@ -4098,12 +4097,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
|
||||
)
|
||||
is_embedding_request: Final = self._is_embedding_request(data, call_type)
|
||||
embedding_text: Final = data.get("input") if is_embedding_request else None
|
||||
request: Final = as_str_mapping(data)
|
||||
raw_embedding_input: Final = request.get("input") if is_embedding_request and request is not None else None
|
||||
pretokenized_input_tokens: Final = (
|
||||
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
|
||||
self._count_pretokenized_embedding_input(raw_embedding_input) if is_embedding_request else None
|
||||
)
|
||||
if pretokenized_input_tokens is not None:
|
||||
return pretokenized_input_tokens
|
||||
embedding_text, embedding_media_tokens = (
|
||||
split_embedding_input_tokens(
|
||||
raw_embedding_input, lambda: self._media_model_resolver(model or "", team_id), model
|
||||
)
|
||||
if is_embedding_request
|
||||
else (None, 0)
|
||||
)
|
||||
|
||||
prompt: Final = data.get("prompt")
|
||||
fallback_text: Final = prompt if prompt is not None else data.get("input")
|
||||
|
|
@ -4153,12 +4160,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",))
|
||||
),
|
||||
)
|
||||
return estimate + audio_token_estimate
|
||||
return estimate + audio_token_estimate + embedding_media_tokens
|
||||
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
|
||||
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
|
||||
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
|
||||
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
|
||||
data=data, call_type=call_type, include_media=False
|
||||
data=data,
|
||||
call_type=call_type,
|
||||
include_media=is_embedding_request,
|
||||
team_id=team_id,
|
||||
media_model_resolver=self._media_model_resolver,
|
||||
)
|
||||
return estimated_input_tokens + audio_token_estimate
|
||||
|
||||
|
|
|
|||
|
|
@ -225,6 +225,32 @@ def _guardrail_info_lifted_from_litellm_metadata(
|
|||
return {"standard_logging_guardrail_information": guardrail_info}
|
||||
|
||||
|
||||
def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object:
|
||||
metadata: Final = _str_keyed_fields(request_data.get("metadata"))
|
||||
lifted: Final = _guardrail_info_lifted_from_litellm_metadata(
|
||||
_str_keyed_fields(request_data.get("litellm_metadata")),
|
||||
guardrail_info_already_set="standard_logging_guardrail_information" in metadata,
|
||||
)
|
||||
return {**metadata, **lifted}.get("standard_logging_guardrail_information")
|
||||
|
||||
|
||||
def failure_row_response_cost(request_data: Mapping[str, object]) -> float:
|
||||
stashed_cost: Final = request_data.get("response_cost")
|
||||
recovered_stream_cost: Final = (
|
||||
max(float(stashed_cost), 0.0)
|
||||
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
|
||||
and isinstance(stashed_cost, (int, float))
|
||||
else 0.0
|
||||
)
|
||||
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
|
||||
finalized_amount: Final = finalized_failure.get("response_cost")
|
||||
return (
|
||||
float(finalized_amount)
|
||||
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
|
||||
else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data))
|
||||
)
|
||||
|
||||
|
||||
def _redaction_opt_in_lifted_from_litellm_metadata(
|
||||
litellm_metadata: ObjectMapping, *, caller_set_headers: bool
|
||||
) -> ObjectMapping:
|
||||
|
|
@ -577,19 +603,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
# recovered cost onto request_data (the usage rides along in
|
||||
# ``combined_usage_object`` for the token columns), so attribute the
|
||||
# real partial spend to this failure row instead of zero.
|
||||
recovered_stream_cost: Final = (
|
||||
max(float(request_data.get("response_cost") or 0.0), 0.0)
|
||||
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
|
||||
else 0.0
|
||||
)
|
||||
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
|
||||
finalized_amount: Final = finalized_failure.get("response_cost")
|
||||
recovered_response_cost: Final = (
|
||||
float(finalized_amount)
|
||||
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
|
||||
else recovered_stream_cost
|
||||
+ guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information"))
|
||||
)
|
||||
recovered_response_cost: Final = failure_row_response_cost(request_data)
|
||||
|
||||
failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it(
|
||||
request_data.get("standard_logging_object"),
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ from litellm.proxy.auth.password_policy import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.openai_error_payload import openai_shaped_proxy_exception, proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
AUTH_OBJECTS_TARGET,
|
||||
object_permission_cache_key,
|
||||
|
|
@ -1048,6 +1048,7 @@ async def _check_user_info_v2_access(
|
|||
1. Proxy admins / proxy admin viewers can access any user
|
||||
2. User can access their own info
|
||||
3. Team admins can access info of users in their teams
|
||||
4. Org admins can access info of users in their organizations
|
||||
|
||||
Raises on unexpected DB errors so they surface as 500s, not silent 404s.
|
||||
"""
|
||||
|
|
@ -1090,9 +1091,33 @@ async def _check_user_info_v2_access(
|
|||
if team.team_id in (target_user.teams or []):
|
||||
return target_user
|
||||
|
||||
if user_api_key_dict.user_id is not None and await _target_in_caller_admin_org(
|
||||
prisma_client=prisma_client,
|
||||
caller_user_id=user_api_key_dict.user_id,
|
||||
target_user_id=target_user_id,
|
||||
):
|
||||
return await _fetch_target_user()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def _target_in_caller_admin_org(
|
||||
prisma_client: "PrismaClient",
|
||||
caller_user_id: str,
|
||||
target_user_id: str,
|
||||
) -> bool:
|
||||
admin_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
|
||||
where={"user_id": caller_user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value}
|
||||
)
|
||||
if not admin_memberships:
|
||||
return False
|
||||
admin_org_ids: Final = {membership.organization_id for membership in admin_memberships}
|
||||
target_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
|
||||
where={"user_id": target_user_id}
|
||||
)
|
||||
return any(membership.organization_id in admin_org_ids for membership in target_memberships)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/v2/user/info",
|
||||
tags=["Internal User management"],
|
||||
|
|
@ -1197,6 +1222,8 @@ async def user_info_v2(
|
|||
),
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, HTTPException) and e.status_code < 500:
|
||||
raise openai_shaped_proxy_exception(e, str(e.detail), e.status_code)
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e)
|
||||
raise handle_exception_on_proxy(e)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,7 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
marks_gateway_injection,
|
||||
prompt_caching_savings_for_request,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import (
|
||||
_query_raw_rows, # pyright: ignore[reportPrivateUsage] # existing typed spend-query adapter; rows validated below
|
||||
)
|
||||
from litellm.proxy.utils import query_raw_within_statement_timeout
|
||||
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.management_endpoints.prompt_caching_requests import (
|
||||
|
|
@ -158,7 +156,7 @@ async def get_prompt_caching_requests(
|
|||
else cursor_start_time
|
||||
)
|
||||
rows: Final = _REQUEST_ROWS.validate_python(
|
||||
await _query_raw_rows(
|
||||
await query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
prompt_caching_requests_sql(filter),
|
||||
start.isoformat(),
|
||||
|
|
@ -166,8 +164,8 @@ async def get_prompt_caching_requests(
|
|||
page_size + 1,
|
||||
cursor_time.isoformat() if cursor_time is not None else None,
|
||||
cursor_request_id,
|
||||
what="Prompt caching requests",
|
||||
)
|
||||
or ()
|
||||
)
|
||||
|
||||
def current_router() -> "Router | None":
|
||||
|
|
|
|||
|
|
@ -224,7 +224,12 @@ from litellm.proxy.management_helpers.utils import (
|
|||
add_new_member,
|
||||
management_endpoint_wrapper,
|
||||
)
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
|
||||
from litellm.proxy.utils import (
|
||||
PrismaClient,
|
||||
ProxyLogging,
|
||||
handle_exception_on_proxy,
|
||||
query_raw_within_statement_timeout,
|
||||
)
|
||||
from litellm.repositories.budget_repository import BudgetRepository
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in
|
||||
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
|
||||
|
|
@ -7473,6 +7478,9 @@ class _TeamUserSpendDbRow(TypedDict):
|
|||
failed_requests: ReadOnly[int]
|
||||
|
||||
|
||||
_TEAM_USER_SPEND_ROWS: Final = TypeAdapter(tuple[_TeamUserSpendDbRow, ...])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/team/spend/by_user",
|
||||
response_model=TeamUserSpendResponse,
|
||||
|
|
@ -7523,12 +7531,16 @@ async def get_team_spend_by_user(
|
|||
own_user_only: Final = scope.api_key_filter is not None
|
||||
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
|
||||
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
|
||||
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
|
||||
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
|
||||
start_date,
|
||||
end_date,
|
||||
*scoped_team_ids,
|
||||
*user_param,
|
||||
rows: Final = _TEAM_USER_SPEND_ROWS.validate_python(
|
||||
await query_raw_within_statement_timeout(
|
||||
prisma_client,
|
||||
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
|
||||
start_date,
|
||||
end_date,
|
||||
*scoped_team_ids,
|
||||
*user_param,
|
||||
what="Spend per user",
|
||||
)
|
||||
)
|
||||
results: Final = tuple(
|
||||
TeamUserSpendRow(
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text
|
||||
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
|
||||
from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, DEFAULT_NUM_WORKERS_LITELLM_PROXY
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
|
|
@ -409,6 +409,7 @@ class ProxyInitializationHelpers:
|
|||
log_config: str | None = None,
|
||||
keepalive_timeout: int | None = None,
|
||||
timeout_worker_healthcheck: int | None = None,
|
||||
max_request_header_size: int | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Get the arguments for `uvicorn` worker
|
||||
|
|
@ -427,6 +428,9 @@ class ProxyInitializationHelpers:
|
|||
"port": port,
|
||||
"server_header": False,
|
||||
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
|
||||
"h11_max_incomplete_event_size": (
|
||||
max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE
|
||||
),
|
||||
}
|
||||
if log_config is not None:
|
||||
print_startup_message(f"Using log_config: {log_config}")
|
||||
|
|
@ -680,6 +684,7 @@ class ProxyInitializationHelpers:
|
|||
ssl_keyfile_path: str,
|
||||
max_requests_before_restart: int | None = None,
|
||||
max_requests_before_restart_jitter: int | None = None,
|
||||
max_request_header_size: int | None = None,
|
||||
):
|
||||
"""
|
||||
Run litellm with `gunicorn`
|
||||
|
|
@ -754,7 +759,7 @@ class ProxyInitializationHelpers:
|
|||
gunicorn_options: Final = {
|
||||
"bind": f"{host}:{port}",
|
||||
"workers": num_workers, # default is 1
|
||||
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
|
||||
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout, max_request_header_size),
|
||||
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
|
||||
"preload": True, # Add the preload flag,
|
||||
"accesslog": "-", # Log to stdout
|
||||
|
|
@ -1108,6 +1113,18 @@ class ProxyInitializationHelpers:
|
|||
help="Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)",
|
||||
envvar="KEEPALIVE_TIMEOUT",
|
||||
)
|
||||
@click.option(
|
||||
"--max_request_header_size",
|
||||
default=None,
|
||||
show_default=f"{DEFAULT_MAX_REQUEST_HEADER_SIZE} bytes with h11",
|
||||
type=click.IntRange(min=1),
|
||||
help=(
|
||||
"Maximum bytes buffered for an incomplete HTTP request header block. "
|
||||
"Applies when uvicorn or --run_gunicorn uses h11. Parser selection is unchanged; httptools ignores this setting. "
|
||||
"Ignored under --run_hypercorn / --run_granian."
|
||||
),
|
||||
envvar="MAX_REQUEST_HEADER_SIZE",
|
||||
)
|
||||
@click.option(
|
||||
"--timeout_worker_healthcheck",
|
||||
default=None,
|
||||
|
|
@ -1253,6 +1270,7 @@ def run_server(
|
|||
check_migrations_against_existing_data: bool,
|
||||
keepalive_timeout,
|
||||
timeout_worker_healthcheck,
|
||||
max_request_header_size: int | None,
|
||||
max_requests_before_restart: int | None,
|
||||
max_requests_before_restart_jitter: int | None,
|
||||
limit_concurrency: int | None,
|
||||
|
|
@ -1747,6 +1765,7 @@ def run_server(
|
|||
log_config=log_config,
|
||||
keepalive_timeout=keepalive_timeout,
|
||||
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
|
||||
max_request_header_size=max_request_header_size,
|
||||
)
|
||||
# Optional: recycle uvicorn workers after N requests
|
||||
if max_requests_before_restart is not None:
|
||||
|
|
@ -1821,6 +1840,7 @@ def run_server(
|
|||
ssl_keyfile_path=ssl_keyfile_path,
|
||||
max_requests_before_restart=max_requests_before_restart,
|
||||
max_requests_before_restart_jitter=max_requests_before_restart_jitter,
|
||||
max_request_header_size=max_request_header_size,
|
||||
)
|
||||
elif run_hypercorn is True:
|
||||
ProxyInitializationHelpers.init_hypercorn_server(
|
||||
|
|
|
|||
|
|
@ -530,6 +530,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
|
||||
from litellm.proxy.common_utils.fips import enforce_fips_mode
|
||||
from litellm.proxy.common_utils.healthy_model_filter import (
|
||||
get_hidden_unhealthy_model_names,
|
||||
is_healthy_only_listing_default,
|
||||
|
|
@ -1673,6 +1674,8 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener
|
|||
if decoded_worker_config is not None:
|
||||
await initialize_from_worker_config(decoded_worker_config)
|
||||
|
||||
enforce_fips_mode(general_settings_view())
|
||||
|
||||
async with AsyncExitStack() as boot_database:
|
||||
enforce_master_key_boot_verdict(
|
||||
await with_stored_secrets_counted(
|
||||
|
|
@ -18611,8 +18614,9 @@ async def model_deprecations(
|
|||
|
||||
Reads `deprecation_date` metadata from `model_prices_and_context_window.json`
|
||||
(and any per-deployment `model_info.deprecation_date` overrides) for the
|
||||
models configured on this proxy. An entry also names its `successor_model`
|
||||
when the deployment sets `model_info.successor_model`.
|
||||
models configured on this proxy. An entry also names its `successor_model`:
|
||||
`model_info.successor_model` when the deployment sets one, else the
|
||||
replacement the cost map records from the provider's deprecation notice.
|
||||
|
||||
Parameters:
|
||||
warn_within_days: Window (in days) used to bucket "imminent" models,
|
||||
|
|
|
|||
|
|
@ -3567,6 +3567,34 @@
|
|||
],
|
||||
"default_model_placeholder": "reka/reka-flash"
|
||||
},
|
||||
{
|
||||
"provider": "WALLY",
|
||||
"provider_display_name": "Wally",
|
||||
"litellm_provider": "wally",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://inference.runanywhere.ai/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "wally/glm-5.3-flash"
|
||||
},
|
||||
{
|
||||
"provider": "Sail",
|
||||
"provider_display_name": "Sail",
|
||||
|
|
|
|||
|
|
@ -1,10 +1,21 @@
|
|||
from uvicorn.workers import UvicornWorker
|
||||
|
||||
from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE
|
||||
|
||||
def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]:
|
||||
|
||||
def build_graceful_shutdown_uvicorn_worker(
|
||||
timeout_graceful_shutdown: float,
|
||||
max_request_header_size: int | None = None,
|
||||
) -> type[UvicornWorker]:
|
||||
class GracefulShutdownUvicornWorker(UvicornWorker):
|
||||
def __init__(self, *args: object) -> None:
|
||||
self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown}
|
||||
self.CONFIG_KWARGS = {
|
||||
**UvicornWorker.CONFIG_KWARGS,
|
||||
"timeout_graceful_shutdown": timeout_graceful_shutdown,
|
||||
"h11_max_incomplete_event_size": (
|
||||
max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE
|
||||
),
|
||||
}
|
||||
super().__init__(*args)
|
||||
|
||||
return GracefulShutdownUvicornWorker
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import collections
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
from itertools import groupby
|
||||
|
|
@ -72,7 +73,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
|
|||
get_spend_by_team,
|
||||
get_spend_by_team_and_customer,
|
||||
)
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
from litellm.proxy.utils import handle_exception_on_proxy, query_raw_within_statement_timeout
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import SpendLogsRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
|
|
@ -267,6 +268,30 @@ async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args:
|
|||
return await _query_raw(prisma_client, sql_query, *args)
|
||||
|
||||
|
||||
async def _query_spend_logs(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]:
|
||||
"""``_query_raw`` for raw ``LiteLLM_SpendLogs`` reads, which Postgres cancels at the statement timeout."""
|
||||
return cast( # cast-ok: callers declare the row shape their SELECT list produces, as with _query_raw
|
||||
"Sequence[_RowT]", await query_raw_within_statement_timeout(prisma_client, sql_query, *args)
|
||||
)
|
||||
|
||||
|
||||
async def _query_spend_logs_or_none(
|
||||
prisma_client: PrismaClient, sql_query: str, *args: object
|
||||
) -> Sequence[_RowT] | None:
|
||||
"""``_query_spend_logs`` for the call sites that guard the result against ``None``."""
|
||||
return await _query_spend_logs(prisma_client, sql_query, *args)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SpendLogsReader:
|
||||
"""``read_session_page``'s raw-query handle, each read cancelled by Postgres at the statement timeout."""
|
||||
|
||||
prisma_client: PrismaClient
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
|
||||
return await query_raw_within_statement_timeout(self.prisma_client, query, *args)
|
||||
|
||||
|
||||
class _TeamTable(Protocol):
|
||||
"""The subset of the Prisma team table API this module uses."""
|
||||
|
||||
|
|
@ -1566,7 +1591,9 @@ async def get_global_spend_report(
|
|||
ORDER BY
|
||||
total_cost DESC;
|
||||
"""
|
||||
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, api_key)
|
||||
db_response = await _query_spend_logs_or_none(
|
||||
prisma_client, sql_query, start_date_obj, end_date_obj, api_key
|
||||
)
|
||||
if db_response is None:
|
||||
return []
|
||||
|
||||
|
|
@ -1609,7 +1636,7 @@ async def get_global_spend_report(
|
|||
ORDER BY
|
||||
total_cost DESC;
|
||||
"""
|
||||
db_response = await _query_raw_or_none(
|
||||
db_response = await _query_spend_logs_or_none(
|
||||
prisma_client, sql_query, start_date_obj, end_date_obj, internal_user_id
|
||||
)
|
||||
if db_response is None:
|
||||
|
|
@ -1676,7 +1703,7 @@ async def get_global_spend_report(
|
|||
group_by_day;
|
||||
"""
|
||||
|
||||
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
|
||||
db_response = await _query_spend_logs_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
|
||||
if db_response is None:
|
||||
return []
|
||||
|
||||
|
|
@ -1717,12 +1744,14 @@ async def get_global_spend_report(
|
|||
ORDER BY
|
||||
total_cost DESC;
|
||||
"""
|
||||
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
|
||||
db_response = await _query_spend_logs_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
|
||||
if db_response is None:
|
||||
return []
|
||||
|
||||
return db_response
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
|
|
@ -1966,7 +1995,7 @@ async def get_key_spend_report(
|
|||
caller_value=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
|
||||
scope_name="api_key",
|
||||
)
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
|
||||
prisma_client,
|
||||
_scoped_spend_report_sql(scope_column="api_key"),
|
||||
start_date_obj,
|
||||
|
|
@ -2008,7 +2037,7 @@ async def get_user_spend_report(
|
|||
caller_value=user_api_key_dict.user_id,
|
||||
scope_name="internal_user_id",
|
||||
)
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
|
||||
prisma_client,
|
||||
_scoped_spend_report_sql(scope_column="user"),
|
||||
start_date_obj,
|
||||
|
|
@ -2050,7 +2079,7 @@ async def get_team_spend_report(
|
|||
caller_value=user_api_key_dict.team_id,
|
||||
scope_name="team_id",
|
||||
)
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
|
||||
prisma_client,
|
||||
_scoped_spend_report_sql(scope_column="team_id"),
|
||||
start_date_obj,
|
||||
|
|
@ -2092,7 +2121,7 @@ async def get_organization_spend_report(
|
|||
organization_id=organization_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
|
||||
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
|
||||
prisma_client,
|
||||
_ORG_SPEND_REPORT_SQL,
|
||||
start_date_obj,
|
||||
|
|
@ -3017,7 +3046,7 @@ async def ui_view_spend_logs(
|
|||
LIMIT ${p}
|
||||
) AS bounded_matches
|
||||
"""
|
||||
count_rows: Final[Sequence[_SpendLogsCountRow] | None] = await _query_raw_or_none(
|
||||
count_rows: Final[Sequence[_SpendLogsCountRow] | None] = await _query_spend_logs_or_none(
|
||||
prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1
|
||||
)
|
||||
raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0
|
||||
|
|
@ -3048,7 +3077,7 @@ async def ui_view_spend_logs(
|
|||
)
|
||||
sql_params.extend([page_size, skip])
|
||||
|
||||
data: Final[Sequence[Mapping[str, object]]] = await _query_raw(prisma_client, sql_query, *sql_params)
|
||||
data: Final[Sequence[Mapping[str, object]]] = await _query_spend_logs(prisma_client, sql_query, *sql_params)
|
||||
|
||||
if request_id is not None and not is_v2 and not is_admin_view:
|
||||
await _assert_user_owns_fetched_spend_rows(
|
||||
|
|
@ -3102,7 +3131,7 @@ async def _fetch_session_representatives(
|
|||
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
|
||||
) AS session_representatives
|
||||
"""
|
||||
rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place
|
||||
rep_rows: Final[Sequence[dict[str, object]]] = await _query_spend_logs( # mutable-ok: rows are enriched in place
|
||||
prisma_client,
|
||||
rep_query,
|
||||
*sql_params,
|
||||
|
|
@ -3130,7 +3159,7 @@ async def _ui_session_grouped_spend_logs(
|
|||
and each is shown by its final agent outcome or newest non-MCP row."""
|
||||
where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE"
|
||||
session_page: Final = await read_session_page(
|
||||
prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__
|
||||
_SpendLogsReader(prisma_client), # pyright: ignore[reportArgumentType] # rows are the TypedDict the session SQL selects
|
||||
where_clause=where_clause,
|
||||
params=sql_params,
|
||||
status=status_filter,
|
||||
|
|
@ -4605,7 +4634,7 @@ async def _build_ui_spend_logs_response(
|
|||
if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) is not None
|
||||
}
|
||||
)
|
||||
rows: Final[Sequence[_SessionSpendRow]] = await _query_raw(
|
||||
rows: Final[Sequence[_SessionSpendRow]] = await _query_spend_logs(
|
||||
prisma_client,
|
||||
f"""
|
||||
SELECT s.*, COALESCE(m.session_models, ARRAY[]::text[]) AS session_models
|
||||
|
|
@ -4676,7 +4705,7 @@ async def _build_ui_spend_logs_response(
|
|||
if row.get("session_id") and row.get("api_key") is not None
|
||||
for models in (TypeAdapter(list[str]).validate_python(row.get("session_models") or ()),)
|
||||
}
|
||||
except PrismaError:
|
||||
except (PrismaError, HTTPException):
|
||||
verbose_proxy_logger.debug(
|
||||
"Failed to enrich session spend aggregates for spend logs UI",
|
||||
exc_info=True,
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.proxy.utils import PrismaClient, hash_token
|
||||
from litellm.proxy.utils import PrismaClient, hash_token, query_raw_within_statement_timeout
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
compaction_summary,
|
||||
summary_only_compaction_content,
|
||||
|
|
@ -649,10 +649,12 @@ def get_logging_payload(
|
|||
usage = _usage
|
||||
|
||||
# A request that failed mid-stream has no usable response_obj usage, and a stream
|
||||
# continued on a fallback only has the fallback's. combined_usage_object holds the
|
||||
# usage of every call the client's answer took, so it wins in both cases.
|
||||
# continued on a fallback or a success that bills a failed follow-up only has its own.
|
||||
# combined_usage_object holds the usage of every call the row bills, so it wins in each case.
|
||||
_combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage):
|
||||
if (
|
||||
not usage or "continued_partial_usage" in kwargs or "failed_attempts_billed_on_success" in kwargs
|
||||
) and isinstance(_combined_usage, litellm.Usage):
|
||||
usage = _combined_usage.model_dump()
|
||||
|
||||
id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata)
|
||||
|
|
@ -1092,7 +1094,8 @@ async def _query_raw_rows(
|
|||
sql_query: str,
|
||||
*args: object,
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
return await prisma_client.db.query_raw(sql_query, *args)
|
||||
"""A raw ``LiteLLM_SpendLogs`` read that Postgres cancels at the statement timeout."""
|
||||
return await query_raw_within_statement_timeout(prisma_client, sql_query, *args)
|
||||
|
||||
|
||||
async def get_spend_by_team(
|
||||
|
|
|
|||
|
|
@ -1360,7 +1360,7 @@ async def update_sso_settings(
|
|||
if isinstance(environment_variables_param, Mapping)
|
||||
else MappingProxyType({})
|
||||
)
|
||||
stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None}
|
||||
stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None and v != ""}
|
||||
fallback_values: Final = {**legacy_env_values, **stored_values}
|
||||
submitted: Final = sso_config.model_dump(include=sso_config.model_fields_set)
|
||||
sso_data: Final = {
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT,
|
||||
SPEND_LOG_QUEUE_MAX_BYTES,
|
||||
SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
|
||||
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
|
||||
VLLM_CHOICE_FIELDS,
|
||||
|
|
@ -5309,6 +5310,39 @@ _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(ma
|
|||
_DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60
|
||||
_PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5)
|
||||
_PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2)
|
||||
_SPEND_LOG_READ_STATEMENT_TIMEOUT: Final = timedelta(seconds=SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS)
|
||||
_STATEMENT_TIMEOUT_TRANSACTION_GRACE: Final = timedelta(seconds=5)
|
||||
|
||||
|
||||
class _ReadTransactionClient(Protocol):
|
||||
@property
|
||||
def read_db(self) -> PrismaWrapper: ...
|
||||
|
||||
|
||||
async def query_raw_within_statement_timeout(
|
||||
prisma_client: _ReadTransactionClient,
|
||||
sql: str,
|
||||
*params: object,
|
||||
what: str = "Spend logs",
|
||||
timeout: timedelta = _SPEND_LOG_READ_STATEMENT_TIMEOUT,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""One raw read on the read target that Postgres cancels at ``timeout``.
|
||||
|
||||
Prisma's HTTP client gives up on a statement after 30s anyway, so without
|
||||
this the Admin UI's error is the same but Postgres keeps running the scan.
|
||||
A cancelled read surfaces as a 503 asking for a narrower date range.
|
||||
"""
|
||||
try:
|
||||
async with prisma_client.read_db.tx(timeout=timeout + _STATEMENT_TIMEOUT_TRANSACTION_GRACE) as transaction:
|
||||
await transaction.execute_raw(f"SET LOCAL statement_timeout = {int(timeout / timedelta(milliseconds=1))}")
|
||||
return await transaction.query_raw(sql, *params)
|
||||
except Exception as e:
|
||||
if not PrismaDBExceptionHandler.is_statement_timeout_error(e):
|
||||
raise
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail={"error": f"{what} took longer than {int(timeout.total_seconds())}s. Pick a shorter date range"},
|
||||
) from e
|
||||
|
||||
|
||||
async def _lookup_deprecated_key(
|
||||
|
|
|
|||
|
|
@ -124,9 +124,7 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
|
|||
records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"user_id": user_id})
|
||||
return self._to_model_list(records)
|
||||
|
||||
async def find_newest_reusable_llm_api_key(
|
||||
self, user_id: str, team_id: str | None
|
||||
) -> LiteLLM_VerificationToken | None:
|
||||
async def find_newest_reusable_key(self, user_id: str, team_id: str | None) -> LiteLLM_VerificationToken | None:
|
||||
records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(
|
||||
where={
|
||||
"user_id": user_id,
|
||||
|
|
@ -140,12 +138,6 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
|
|||
{"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}},
|
||||
]
|
||||
},
|
||||
{
|
||||
"OR": [
|
||||
{"allowed_routes": {"is_empty": True}},
|
||||
{"allowed_routes": {"has": "llm_api_routes"}},
|
||||
]
|
||||
},
|
||||
],
|
||||
},
|
||||
order={"created_at": "desc"},
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
replace_complexity_router_headers,
|
||||
response_total_token_count,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
|
|
@ -135,7 +136,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models, deployment_pricing_models
|
||||
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
|
|
@ -483,11 +484,13 @@ from litellm.types.services import ServiceTypes
|
|||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
COST_MAP_LOOKUP_KEY,
|
||||
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS,
|
||||
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
|
||||
CustomPricingLiteLLMParams,
|
||||
GenericBudgetConfigType,
|
||||
LiteLLMBatch,
|
||||
LlmProviders,
|
||||
LlmProvidersSet,
|
||||
ModelInfo,
|
||||
ModelResponseStream,
|
||||
StandardLoggingPayload,
|
||||
|
|
@ -510,9 +513,11 @@ from litellm.utils import (
|
|||
TextCompletionResponse,
|
||||
_restore_correlation_context_if_supported, # pyright: ignore[reportPrivateUsage] # same task-local cleanup as the SDK wrapper
|
||||
aresolve_api_key_secret_reference,
|
||||
catalog_with_runtime_registrations,
|
||||
function_setup,
|
||||
get_llm_provider,
|
||||
get_non_default_completion_params,
|
||||
get_potential_model_names,
|
||||
get_secret,
|
||||
get_utc_datetime,
|
||||
is_generalized_model_info,
|
||||
|
|
@ -4044,6 +4049,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if continuation_output is not None
|
||||
else None
|
||||
)
|
||||
if cut_partial is not None:
|
||||
model_response.logging_obj.mark_stash_billed_by_continuation()
|
||||
if continue_after_content:
|
||||
emitted_tokens: Final = int(
|
||||
getattr(complete_response_object_usage, "completion_tokens", 0) or 0
|
||||
|
|
@ -7373,6 +7380,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
|
||||
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
|
||||
trigger = _mid_stream_retry_trigger(last_error)
|
||||
if is_agentic_followup_failure(trigger):
|
||||
return _StreamRetriesExhausted(last_error)
|
||||
if not policy_applies and not self._stream_should_retry(
|
||||
trigger, healthy_deployments, all_deployments, initial_kwargs
|
||||
):
|
||||
|
|
@ -9620,7 +9629,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
|
||||
return response
|
||||
except Exception as e:
|
||||
if is_guardrail_intervention(e):
|
||||
if is_guardrail_intervention(e) or is_agentic_followup_failure(e):
|
||||
raise
|
||||
if is_create_that_may_already_exist(_OPAQUE_VALUE_ADAPTER.validate_python(original_function), e):
|
||||
log_create_not_resent(model_group)
|
||||
|
|
@ -9718,6 +9727,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Update retry tracking metadata before each retry attempt
|
||||
_metadata["attempted_retries"] = current_attempt + 1
|
||||
_metadata["max_retries"] = num_retries
|
||||
carry_superseded_attempt_usage(kwargs)
|
||||
# if the function call is successful, no exception will be raised and we'll break out of the loop
|
||||
response = await self.make_call(original_function, *args, **kwargs)
|
||||
if coroutine_checker.is_async_callable(response): # async errors are often returned as coroutines
|
||||
|
|
@ -9731,6 +9741,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
if is_agentic_followup_failure(e):
|
||||
raise
|
||||
if is_create_that_may_already_exist(_OPAQUE_VALUE_ADAPTER.validate_python(original_function), e):
|
||||
log_create_not_resent(model_group)
|
||||
raise
|
||||
|
|
@ -12186,6 +12198,25 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
return (backend_key, backend_key.replace("responses/", ""))
|
||||
return (backend_key,)
|
||||
|
||||
@staticmethod
|
||||
def _catalog_entry_has_token_limit(catalog_entry: Mapping[str, object], field: str) -> bool:
|
||||
return catalog_entry.get(field) is not None or (
|
||||
field == "max_tokens" and catalog_entry.get("max_output_tokens") is not None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _shared_backend_model_info_for_keys(
|
||||
catalog: Mapping[str, Mapping[str, object]],
|
||||
backend_keys: tuple[str, ...],
|
||||
shared_model_info: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
catalog_token_limit_fields: Final = frozenset(
|
||||
field
|
||||
for field in DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS
|
||||
if any(PythonCore._catalog_entry_has_token_limit(catalog.get(key) or {}, field) for key in backend_keys)
|
||||
)
|
||||
return {field: value for field, value in shared_model_info.items() if field not in catalog_token_limit_fields}
|
||||
|
||||
@staticmethod
|
||||
def _deployment_model_cost_payload(deployment: Deployment) -> dict[str, object]: # mutable-ok: cost-map entry
|
||||
"""The ``model_info`` a deployment contributes to ``litellm.model_cost``.
|
||||
|
|
@ -12298,6 +12329,31 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
|
||||
backend_keys: Final = PythonCore._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider)
|
||||
model_provider_prefix: Final = model.partition("/")[0]
|
||||
catalog_provider: Final = (
|
||||
custom_llm_provider
|
||||
if custom_llm_provider is not None
|
||||
else model_provider_prefix
|
||||
if model_provider_prefix in LlmProvidersSet
|
||||
else None
|
||||
)
|
||||
potential_names: Final = (
|
||||
get_potential_model_names(model, catalog_provider) if catalog_provider is not None else None
|
||||
)
|
||||
catalog_backend_keys: Final = (
|
||||
(
|
||||
*backend_keys,
|
||||
model,
|
||||
potential_names["split_model"],
|
||||
potential_names["stripped_model_name"],
|
||||
potential_names["combined_model_name"],
|
||||
potential_names["combined_stripped_model_name"],
|
||||
potential_names["provider_prefixed_model_name"],
|
||||
potential_names["region_free_combined_model_name"],
|
||||
)
|
||||
if potential_names is not None
|
||||
else (*backend_keys, model)
|
||||
)
|
||||
|
||||
# For the shared backend key, keep only cost-map schema fields
|
||||
# (minus custom pricing) so that one deployment's pricing overrides
|
||||
|
|
@ -12305,7 +12361,12 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# don't pollute another deployment sharing the same backend model
|
||||
# name. Each deployment's full model_info is already stored under
|
||||
# its unique model_id above.
|
||||
shared_model_info: Final = shared_backend_model_info(model_info)
|
||||
catalog: Final = catalog_with_runtime_registrations()
|
||||
shared_model_info: Final = PythonCore._shared_backend_model_info_for_keys(
|
||||
catalog=catalog,
|
||||
backend_keys=catalog_backend_keys,
|
||||
shared_model_info=shared_backend_model_info(model_info),
|
||||
)
|
||||
python_host.invoke("register_model")(
|
||||
model_cost={_key: keep_registered_mode(_key, shared_model_info) for _key in backend_keys},
|
||||
persist_across_reloads=False,
|
||||
|
|
@ -14894,6 +14955,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
input: str | list[object] | None,
|
||||
request_kwargs: Mapping[str, object] | None = None,
|
||||
tokenizer_model: str = "",
|
||||
deployments: Sequence[Mapping[str, object]] = (),
|
||||
) -> int:
|
||||
"""
|
||||
Count input tokens for context-window pre-call checks.
|
||||
|
|
@ -14928,11 +14990,15 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
if input is not None:
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
|
||||
from litellm.litellm_core_utils.token_counter import split_embedding_input_tokens
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
)
|
||||
|
||||
typed_input: Final = cast(str | ResponseInputParam, input) # cast-ok: str | list matches transform input
|
||||
text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments))
|
||||
typed_input: Final = cast( # cast-ok: str | list matches transform input
|
||||
str | ResponseInputParam, text_input
|
||||
)
|
||||
input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=typed_input,
|
||||
responses_api_request={"instructions": instructions} if instructions is not None else {},
|
||||
|
|
@ -15000,6 +15066,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self.served_openai_models(model) or served_openai_models(healthy_deployments)
|
||||
)
|
||||
or "",
|
||||
deployments=healthy_deployments,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
|
||||
verbose_router_logger.error(
|
||||
|
|
@ -15030,6 +15097,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self.served_openai_models(model) or served_openai_models(deployments)
|
||||
)
|
||||
or "",
|
||||
deployments=deployments,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_router_logger.error(
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ def _detect_loop(history: list[str], new_calls: Sequence[Mapping[str, object]])
|
|||
return False
|
||||
|
||||
|
||||
_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504}
|
||||
_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504, 529}
|
||||
|
||||
_EXHAUSTION_KEYWORDS: Final = (
|
||||
"context length",
|
||||
|
|
|
|||
|
|
@ -2,16 +2,22 @@
|
|||
# picks based on response time (for streaming, this is time to first token)
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse, token_counter, verbose_logger
|
||||
from litellm import ModelResponse, verbose_logger
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
from litellm.litellm_core_utils.deployment_pricing import (
|
||||
deployment_backend_models,
|
||||
deployment_pricing_models,
|
||||
str_keyed,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
|
||||
|
||||
|
||||
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
|
||||
|
|
@ -235,7 +241,9 @@ class LowestCostLoggingHandler(CustomLogger):
|
|||
}
|
||||
|
||||
try:
|
||||
input_tokens = token_counter(messages=messages, text=input)
|
||||
input_tokens = embedding_aware_token_count(
|
||||
messages, input, partial(deployment_backend_models, healthy_deployments)
|
||||
)
|
||||
except Exception:
|
||||
input_tokens = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -3,18 +3,21 @@
|
|||
import random
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime, timedelta
|
||||
from functools import partial
|
||||
from math import ceil, isfinite
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse, token_counter, verbose_logger
|
||||
from litellm import ModelResponse, verbose_logger
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
|
||||
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
|
|
@ -439,7 +442,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
}
|
||||
|
||||
try:
|
||||
input_tokens = token_counter(messages=messages, text=input)
|
||||
input_tokens = embedding_aware_token_count(
|
||||
messages, input, partial(deployment_backend_models, healthy_deployments)
|
||||
)
|
||||
except Exception:
|
||||
input_tokens = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,16 @@
|
|||
# identifies lowest tpm deployment
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from typing import Final
|
||||
|
||||
from litellm import token_counter
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
|
||||
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
|
|
@ -174,7 +176,9 @@ class LowestTPMLoggingHandler(CustomLogger):
|
|||
|
||||
verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict)
|
||||
try:
|
||||
input_tokens = token_counter(messages=messages, text=input)
|
||||
input_tokens = embedding_aware_token_count(
|
||||
messages, input, partial(deployment_backend_models, healthy_deployments)
|
||||
)
|
||||
except Exception:
|
||||
input_tokens = 0
|
||||
verbose_router_logger.debug("input_tokens=%s", input_tokens)
|
||||
|
|
|
|||
|
|
@ -5,18 +5,20 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm import token_counter
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
|
||||
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
|
||||
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
|
||||
from litellm.utils import get_utc_datetime, print_verbose
|
||||
|
|
@ -422,7 +424,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
|
|||
rpm_dict[rpm_keys[idx].split(":")[0]] = rpm_values[idx]
|
||||
|
||||
try:
|
||||
input_tokens = token_counter(messages=messages, text=input)
|
||||
input_tokens = embedding_aware_token_count(
|
||||
messages, input, partial(deployment_backend_models, healthy_deployments)
|
||||
)
|
||||
except Exception:
|
||||
input_tokens = 0
|
||||
verbose_router_logger.debug("input_tokens=%s", input_tokens)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import token_counter
|
||||
|
|
@ -28,6 +29,7 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError
|
||||
from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION
|
||||
from litellm.litellm_core_utils.core_helpers import coerce_token_limit
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
IO_TOKEN_RESERVATION_KEYS,
|
||||
ITPM_CACHE_KEY,
|
||||
|
|
@ -41,6 +43,8 @@ from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
|
|||
from litellm.types.router import RouterCacheEnum, RouterErrors
|
||||
from litellm.utils import get_utc_datetime
|
||||
|
||||
_MODEL_INFO_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
|
|
@ -227,8 +231,20 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme
|
|||
)
|
||||
|
||||
|
||||
def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int:
|
||||
model_max: Final = _model_max_output_tokens(model_name) if model_name else None
|
||||
def _default_otpm_reservation(deployment: Mapping[str, object], model_name: str, otpm_limit: int) -> int:
|
||||
model_info: Final[object] = deployment.get("model_info")
|
||||
configured_model_max: Final = (
|
||||
coerce_token_limit(_MODEL_INFO_ADAPTER.validate_python(model_info).get("max_output_tokens"))
|
||||
if isinstance(model_info, Mapping)
|
||||
else None
|
||||
)
|
||||
model_max: Final = (
|
||||
max(0, configured_model_max)
|
||||
if configured_model_max is not None
|
||||
else _model_max_output_tokens(model_name)
|
||||
if model_name
|
||||
else None
|
||||
)
|
||||
share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))
|
||||
return min(4096 if model_max is None else model_max, share_of_limit)
|
||||
|
||||
|
|
@ -242,7 +258,7 @@ def _otpm_reservation(
|
|||
from_request: Final = _explicit_max_tokens(request_kwargs)
|
||||
if from_request is not None:
|
||||
return from_request
|
||||
default: Final = _default_otpm_reservation(model_name, otpm_limit)
|
||||
default: Final = _default_otpm_reservation(deployment, model_name, otpm_limit)
|
||||
from_deployment: Final = _deployment_output_cap(request_kwargs, deployment)
|
||||
return default if from_deployment is None else min(from_deployment, default)
|
||||
|
||||
|
|
|
|||
|
|
@ -74,14 +74,13 @@ class AgenticLoopSafetyError(ValueError):
|
|||
Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the
|
||||
repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so
|
||||
callers that already catch the broader type keep working.
|
||||
|
||||
Only the anthropic messages loop raises this today. The chat completions
|
||||
loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises
|
||||
a plain ``ValueError`` from its own copy of the same rails, so catching
|
||||
this type alone will not cover that surface until it is moved over.
|
||||
"""
|
||||
|
||||
|
||||
class AgenticLoopLimitError(AgenticLoopSafetyError):
|
||||
pass
|
||||
|
||||
|
||||
class StandardCustomLoggerInitParams(LiteLLMBaseModel):
|
||||
"""
|
||||
Params for initializing a CustomLogger.
|
||||
|
|
|
|||
|
|
@ -34,7 +34,10 @@ class ModelDeprecationInfo(LiteLLMBaseModel):
|
|||
litellm_provider: str | None = Field(default=None, description="The provider this model belongs to.")
|
||||
successor_model: str | None = Field(
|
||||
default=None,
|
||||
description=("The model to migrate to, as set in model_info.successor_model on the deployment."),
|
||||
description=(
|
||||
"The model to migrate to: model_info.successor_model on the deployment when set, "
|
||||
"else the successor_model the cost map names for the deprecated model."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -383,6 +383,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
input_cost_per_character: float | None # only for vertex ai models
|
||||
input_cost_per_audio_token: float | None
|
||||
input_cost_per_audio_token_flex: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_above_200k_tokens: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_above_200k_tokens_flex: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_above_200k_tokens_priority: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_priority: ReadOnly[float | None]
|
||||
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
|
||||
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
|
||||
|
|
@ -408,10 +412,12 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
input_cost_per_token_batches: float | None
|
||||
input_cost_per_video_token_batches: ReadOnly[float | None]
|
||||
input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
input_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None]
|
||||
input_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token_batches: float | None
|
||||
output_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None]
|
||||
output_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token: Required[float | None]
|
||||
|
|
@ -444,6 +450,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
output_cost_per_pixel: ReadOnly[float | None]
|
||||
output_cost_per_image_token: float | None
|
||||
output_cost_per_image_token_batches: ReadOnly[float | None]
|
||||
output_cost_per_image_token_priority: ReadOnly[float | None]
|
||||
output_cost_per_video_token: float | None # for gemini omni models with video output
|
||||
output_vector_size: int | None
|
||||
output_cost_per_reasoning_token: float | None
|
||||
|
|
@ -4173,6 +4180,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
input_cost_per_character_above_128k_tokens: float | None = None
|
||||
input_cost_per_audio_token: float | None = None
|
||||
input_cost_per_audio_token_flex: float | None = None
|
||||
input_cost_per_audio_token_above_200k_tokens: float | None = None
|
||||
input_cost_per_audio_token_above_200k_tokens_flex: float | None = None
|
||||
input_cost_per_audio_token_above_200k_tokens_priority: float | None = None
|
||||
input_cost_per_audio_token_above_200k_tokens_batches: float | None = None
|
||||
input_cost_per_audio_token_priority: float | None = None
|
||||
input_cost_per_token_cache_hit: float | None = None
|
||||
input_cost_per_token_above_128k_tokens: float | None = None
|
||||
|
|
@ -4183,6 +4194,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
input_cost_per_token_above_272k_tokens_flex: float | None = None
|
||||
input_cost_per_token_above_272k_tokens_ultrafast: float | None = None
|
||||
input_cost_per_token_above_200k_tokens_batches: float | None = None
|
||||
input_cost_per_token_above_200k_tokens_flex: float | None = None
|
||||
input_cost_per_token_above_100k_tokens_batches: float | None = None
|
||||
input_cost_per_token_above_272k_tokens_batches: float | None = None
|
||||
input_cost_per_query: float | None = None
|
||||
|
|
@ -4212,12 +4224,14 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
output_cost_per_token_above_272k_tokens_flex: float | None = None
|
||||
output_cost_per_token_above_272k_tokens_ultrafast: float | None = None
|
||||
output_cost_per_token_above_200k_tokens_batches: float | None = None
|
||||
output_cost_per_token_above_200k_tokens_flex: float | None = None
|
||||
output_cost_per_token_above_100k_tokens_batches: float | None = None
|
||||
output_cost_per_token_above_272k_tokens_batches: float | None = None
|
||||
output_cost_per_character_above_128k_tokens: float | None = None
|
||||
output_cost_per_image: float | None = None
|
||||
output_cost_per_image_token: float | None = None
|
||||
output_cost_per_image_token_batches: float | None = None
|
||||
output_cost_per_image_token_priority: float | None = None
|
||||
output_cost_per_video_token: float | None = None
|
||||
output_cost_per_reasoning_token: float | None = None
|
||||
output_cost_per_reasoning_token_flex: float | None = None
|
||||
|
|
@ -4284,6 +4298,10 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{"max_input_tokens", "max_output_tokens", "max_tokens"}
|
||||
)
|
||||
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"token_count_framing",
|
||||
|
|
@ -4756,6 +4774,7 @@ class LlmProviders(str, Enum):
|
|||
SAIL = "sail"
|
||||
REKA = "reka"
|
||||
STREAMLAKE = "streamlake"
|
||||
WALLY = "wally"
|
||||
LITELLM_AGENT = "litellm_agent"
|
||||
CURSOR = "cursor"
|
||||
BEDROCK_MANTLE = "bedrock_mantle"
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ from litellm.constants import (
|
|||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
|
||||
CHAT_ALLOWED_TOOLS_LAYOUT,
|
||||
parse_chat_allowed_tools_tool_choice,
|
||||
|
|
@ -2312,7 +2313,7 @@ def client(original_function):
|
|||
|
||||
is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
|
||||
if (
|
||||
num_retries and not is_completion_litellm_router_call
|
||||
num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
if (
|
||||
isinstance(e, openai.APIError)
|
||||
|
|
@ -2346,7 +2347,7 @@ def client(original_function):
|
|||
|
||||
is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
|
||||
if (
|
||||
num_retries and not is_responses_litellm_router_call
|
||||
num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
if (
|
||||
isinstance(e, openai.APIError)
|
||||
|
|
@ -2645,7 +2646,10 @@ def client(original_function):
|
|||
is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
|
||||
|
||||
if (
|
||||
num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError)
|
||||
num_retries
|
||||
and not is_acompletion_litellm_router_call
|
||||
and not isinstance(e, ImportError)
|
||||
and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
try:
|
||||
litellm.num_retries = None # set retries to None to prevent infinite loops
|
||||
|
|
@ -2676,7 +2680,10 @@ def client(original_function):
|
|||
is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
|
||||
|
||||
if (
|
||||
num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError)
|
||||
num_retries
|
||||
and not is_aresponses_litellm_router_call
|
||||
and not isinstance(e, ImportError)
|
||||
and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
try:
|
||||
litellm.num_retries = None # set retries to None to prevent infinite loops
|
||||
|
|
@ -7445,6 +7452,18 @@ def get_model_info_helper(
|
|||
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
|
||||
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
|
||||
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
|
||||
input_cost_per_audio_token_above_200k_tokens=_model_info.get(
|
||||
"input_cost_per_audio_token_above_200k_tokens", None
|
||||
),
|
||||
input_cost_per_audio_token_above_200k_tokens_flex=_model_info.get(
|
||||
"input_cost_per_audio_token_above_200k_tokens_flex", None
|
||||
),
|
||||
input_cost_per_audio_token_above_200k_tokens_priority=_model_info.get(
|
||||
"input_cost_per_audio_token_above_200k_tokens_priority", None
|
||||
),
|
||||
input_cost_per_audio_token_above_200k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_audio_token_above_200k_tokens_batches", None
|
||||
),
|
||||
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
|
||||
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
|
||||
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
|
||||
|
|
@ -7458,6 +7477,9 @@ def get_model_info_helper(
|
|||
input_cost_per_token_above_200k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_token_above_200k_tokens_batches"
|
||||
),
|
||||
input_cost_per_token_above_200k_tokens_flex=_model_info.get(
|
||||
"input_cost_per_token_above_200k_tokens_flex", None
|
||||
),
|
||||
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
|
|
@ -7465,6 +7487,9 @@ def get_model_info_helper(
|
|||
output_cost_per_token_above_200k_tokens_batches=_model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token_above_200k_tokens_flex=_model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens_flex", None
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
|
|
@ -7526,6 +7551,7 @@ def get_model_info_helper(
|
|||
output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
|
||||
output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
|
||||
output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
|
||||
output_cost_per_image_token_priority=_model_info.get("output_cost_per_image_token_priority", None),
|
||||
output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None),
|
||||
output_vector_size=_model_info.get("output_vector_size", None),
|
||||
citation_cost_per_token=_model_info.get("citation_cost_per_token", None),
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5250,6 +5250,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"wally": {
|
||||
"display_name": "Wally (`wally`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/wally",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"empiriolabs": {
|
||||
"display_name": "EmpirioLabs (`empiriolabs`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",
|
||||
|
|
|
|||
|
|
@ -266,6 +266,7 @@ proxy-dev = [
|
|||
"azure-identity==1.25.2",
|
||||
"a2a-sdk==1.1.0",
|
||||
"backports-zstd==1.8.0; python_version < '3.14'",
|
||||
"httptools>=0.7.1,<0.8",
|
||||
]
|
||||
ci = [
|
||||
"psutil==7.2.2",
|
||||
|
|
|
|||
54
tests/e2e/ui/tests/navigation/commandPalette.spec.ts
Normal file
54
tests/e2e/ui/tests/navigation/commandPalette.spec.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { dismissFeedbackPopup, navigateToPage } from "../../helpers/navigation";
|
||||
import { createVirtualKey, deleteVirtualKey, uniqueSuffix } from "../../helpers/traffic";
|
||||
|
||||
const paletteSearch = (page: PlaywrightPage) => page.getByRole("combobox", { name: "Search" });
|
||||
const headerTrigger = (page: PlaywrightPage) => page.getByRole("button", { name: "Search keys…", exact: true });
|
||||
|
||||
test.describe("Command palette", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await navigateToPage(page, Page.ApiKeys);
|
||||
await dismissFeedbackPopup(page);
|
||||
await expect(headerTrigger(page)).toBeVisible();
|
||||
});
|
||||
|
||||
test("opens a key found by its alias", async ({ page }) => {
|
||||
const alias = `e2e-palette-key-${uniqueSuffix()}`;
|
||||
const created = await createVirtualKey(page.request, { key_alias: alias });
|
||||
try {
|
||||
await page.keyboard.press("ControlOrMeta+K");
|
||||
await paletteSearch(page).fill(alias);
|
||||
await expect(page.getByRole("option", { name: new RegExp(`^${alias}`) })).toBeVisible();
|
||||
await page.keyboard.press("Enter");
|
||||
|
||||
await expect(page).toHaveURL(new RegExp(`[?&]key=${created.token}`));
|
||||
await expect(paletteSearch(page)).toBeHidden();
|
||||
} finally {
|
||||
await deleteVirtualKey(page.request, created.token);
|
||||
}
|
||||
});
|
||||
|
||||
test("does not open over the Create New Key dialog", async ({ page }) => {
|
||||
await page.getByRole("button", { name: /Create New Key/i }).click();
|
||||
const keyName = page.getByLabel(/Key Name/);
|
||||
await keyName.click();
|
||||
|
||||
await page.keyboard.press("ControlOrMeta+K");
|
||||
await page.keyboard.type("still-in-the-dialog");
|
||||
|
||||
await expect(keyName).toHaveValue("still-in-the-dialog");
|
||||
await expect(paletteSearch(page)).toBeHidden();
|
||||
});
|
||||
|
||||
test("opens from the header trigger and closes with Escape", async ({ page }) => {
|
||||
await headerTrigger(page).click();
|
||||
await expect(paletteSearch(page)).toBeFocused();
|
||||
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(paletteSearch(page)).toBeHidden();
|
||||
});
|
||||
});
|
||||
|
|
@ -7,13 +7,15 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
from contextlib import contextmanager
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
import pytest
|
||||
import yaml
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from pydantic import JsonValue
|
||||
from redis import Redis
|
||||
|
||||
from tests.integration._support.client import Gateway, eventually, string_value
|
||||
from tests.integration._support.database import read_rows
|
||||
|
|
@ -31,7 +33,15 @@ def _hash(key: str) -> str:
|
|||
return sha256(key.encode()).hexdigest()
|
||||
|
||||
|
||||
def _config(directory: Path, claim_field: str) -> Path:
|
||||
def _config(
|
||||
directory: Path,
|
||||
claim_field: str,
|
||||
*,
|
||||
behavior: Literal["auto_register", "link_only", "reject"] = "auto_register",
|
||||
map_existing_key: bool = True,
|
||||
user_id_upsert: bool = False,
|
||||
issuers: tuple[dict[str, JsonValue], ...] = (),
|
||||
) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["general_settings"] = {
|
||||
**config["general_settings"],
|
||||
|
|
@ -40,8 +50,10 @@ def _config(directory: Path, claim_field: str) -> Path:
|
|||
"user_id_jwt_field": "sub",
|
||||
"user_email_jwt_field": "email",
|
||||
"virtual_key_claim_field": claim_field,
|
||||
"unregistered_jwt_client_behavior": "auto_register",
|
||||
"auto_register_map_existing_key": True,
|
||||
"unregistered_jwt_client_behavior": behavior,
|
||||
"auto_register_map_existing_key": map_existing_key,
|
||||
"user_id_upsert": user_id_upsert,
|
||||
"issuers": list(issuers),
|
||||
},
|
||||
}
|
||||
path: Final = directory / f"jwt_map_existing_key_{claim_field}.yaml"
|
||||
|
|
@ -107,7 +119,7 @@ def _billed_key(response: httpx.Response) -> str:
|
|||
return string_value(rows[0]["api_key"])
|
||||
|
||||
|
||||
def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_ineligible_newer_key(
|
||||
def test_first_jwt_call_reuses_the_newest_durable_key_and_skips_every_ineligible_newer_key(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
|
|
@ -118,7 +130,6 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig
|
|||
skipped: Final = {
|
||||
"older_durable": older_durable,
|
||||
"expiring": scenario.key(user_id=user, duration="1h"),
|
||||
"management_only": scenario.key(user_id=user, allowed_routes=["management_routes"]),
|
||||
"auto_registered_look_alike": scenario.key(user_id=user, metadata={"auto_registered": True}),
|
||||
"other_team": scenario.key(user_id=user, team_id=scenario.team()),
|
||||
"blocked": scenario.key(user_id=user),
|
||||
|
|
@ -140,6 +151,84 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig
|
|||
assert _billed_key(response) == _hash(durable)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("behavior", "map_existing_key"), [("auto_register", True), ("link_only", False), ("link_only", True)]
|
||||
)
|
||||
def test_route_restricted_keys_are_linked_without_bypassing_other_key_checks(
|
||||
gateway: Gateway,
|
||||
tmp_path: Path,
|
||||
behavior: Literal["auto_register", "link_only"],
|
||||
map_existing_key: bool,
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
scenario.key(user_id=user)
|
||||
restricted: Final = scenario.key(user_id=user, allowed_routes=["/v1/chat/completions"])
|
||||
scenario.key(user_id=user, duration="1h")
|
||||
scenario.key(user_id=user, metadata={"auto_registered": True})
|
||||
scenario.key(user_id=user, team_id=scenario.team())
|
||||
blocked: Final = scenario.key(user_id=user)
|
||||
gateway.post("/key/block", {"key": blocked})
|
||||
keys_before: Final = _user_key_hashes(user)
|
||||
token: Final = _token(private_key, user)
|
||||
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(
|
||||
tmp_path,
|
||||
"sub",
|
||||
behavior=behavior,
|
||||
map_existing_key=map_existing_key,
|
||||
),
|
||||
) as candidate:
|
||||
response: Final = _chat(candidate, model, token)
|
||||
repeated: Final = _chat(candidate, model, token)
|
||||
restricted_route: Final = candidate.request("GET", "/v1/models", key=token)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert repeated.status_code == 200, repeated.text
|
||||
assert restricted_route.status_code == 403, restricted_route.text
|
||||
assert _mapped_token("sub", user) == _hash(restricted)
|
||||
assert _user_key_hashes(user) == keys_before, "a new key was minted instead of linking an existing key"
|
||||
assert _billed_key(response) == _hash(restricted)
|
||||
assert _billed_key(repeated) == _hash(restricted)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("behavior", ["auto_register", "link_only"])
|
||||
def test_management_route_keys_are_linked_by_default_and_keep_their_route_permissions(
|
||||
gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"]
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
target: Final = f"integration-{uuid.uuid4().hex}"
|
||||
scenario.key(user_id=user)
|
||||
management_key: Final = scenario.key(user_id=user, allowed_routes=["/user/new"])
|
||||
keys_before: Final = _user_key_hashes(user)
|
||||
token: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=_config(tmp_path, "sub", behavior=behavior)
|
||||
) as candidate:
|
||||
denied: Final = _chat(candidate, model, token)
|
||||
allowed: Final = candidate.request(
|
||||
"POST",
|
||||
"/user/new",
|
||||
{"user_id": target, "auto_create_key": False, "user_role": "internal_user"},
|
||||
key=token,
|
||||
)
|
||||
assert allowed.status_code == 200, allowed.text
|
||||
scenario.cleanups.callback(scenario.delete_user, target)
|
||||
repeated: Final = _chat(candidate, model, token)
|
||||
assert denied.status_code == 403, denied.text
|
||||
assert repeated.status_code == 403, repeated.text
|
||||
assert allowed.json()["user_id"] == target
|
||||
assert _mapped_token("sub", user) == _hash(management_key)
|
||||
assert _user_key_hashes(user) == keys_before
|
||||
|
||||
|
||||
def test_user_matched_by_email_instead_of_sub_still_reuses_their_existing_key(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
|
|
@ -183,8 +272,9 @@ def test_shared_client_claim_never_maps_a_second_user_onto_the_first_users_perso
|
|||
assert read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (_hash(personal),)) == []
|
||||
|
||||
|
||||
def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviving_mapped_key(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
@pytest.mark.parametrize("behavior", ["auto_register", "link_only"])
|
||||
def test_concurrent_first_jwt_calls_both_succeed_on_one_surviving_mapped_key(
|
||||
gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"]
|
||||
) -> None:
|
||||
writer_url: Final = os.environ.get("INTEGRATION_PROXY_DATABASE_URL") or os.environ["DATABASE_URL"]
|
||||
with (
|
||||
|
|
@ -194,6 +284,7 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
|
|||
):
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
existing: Final = scenario.key(user_id=user) if behavior == "link_only" else None
|
||||
token: Final = _token(private_key, user)
|
||||
overrides: Final = {
|
||||
"JWT_PUBLIC_KEY_URL": jwks_url,
|
||||
|
|
@ -202,7 +293,12 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
|
|||
}
|
||||
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, overrides, config=_config(tmp_path, "sub")) as candidate,
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
overrides,
|
||||
config=_config(tmp_path, "sub", behavior=behavior, map_existing_key=False),
|
||||
) as candidate,
|
||||
ThreadPoolExecutor(max_workers=1) as pool,
|
||||
):
|
||||
held_call: Final = pool.submit(_chat, candidate, model, token)
|
||||
|
|
@ -215,5 +311,240 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
|
|||
assert held.status_code == 200, held.text
|
||||
keys: Final = _user_key_hashes(user)
|
||||
assert len(keys) == 1, keys
|
||||
if existing is not None:
|
||||
assert keys == frozenset({_hash(existing)})
|
||||
assert _mapped_token("sub", user) in keys
|
||||
assert (_billed_key(held), _billed_key(racing)) == (_mapped_token("sub", user),) * 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ineligible_keys", [False, True])
|
||||
def test_link_only_rejects_without_writes_then_links_a_newly_available_key(
|
||||
gateway: Gateway, tmp_path: Path, ineligible_keys: bool
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
if ineligible_keys:
|
||||
scenario.key(user_id=user, duration="1h")
|
||||
scenario.key(user_id=user, metadata={"auto_registered": True})
|
||||
scenario.key(user_id=user, team_id=scenario.team())
|
||||
blocked: Final = scenario.key(user_id=user)
|
||||
gateway.post("/key/block", {"key": blocked})
|
||||
keys_before: Final = _user_key_hashes(user)
|
||||
token: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
|
||||
) as candidate:
|
||||
first: Final = _chat(candidate, model, token)
|
||||
repeated: Final = _chat(candidate, model, token)
|
||||
assert first.status_code == 403, first.text
|
||||
assert repeated.status_code == 403, repeated.text
|
||||
assert _user_key_hashes(user) == keys_before
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
|
||||
available: Final = scenario.key(user_id=user)
|
||||
accepted: Final = _chat(candidate, model, token)
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
assert _mapped_token("sub", user) == _hash(available)
|
||||
assert _user_key_hashes(user) == keys_before | {_hash(available)}
|
||||
assert _billed_key(accepted) == _hash(available)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("claim_field", ["client_id", "email"])
|
||||
def test_link_only_rejects_shared_or_missing_claims_without_linking_a_personal_key(
|
||||
gateway: Gateway, tmp_path: Path, claim_field: str
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
key: Final = scenario.key(user_id=user)
|
||||
shared_claim: Final = f"integration-client-{uuid.uuid4().hex}"
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, claim_field, behavior="link_only", map_existing_key=False),
|
||||
) as candidate:
|
||||
response: Final = _chat(candidate, model, _token(private_key, user, client_id=shared_claim))
|
||||
assert response.status_code == 403, response.text
|
||||
assert _user_key_hashes(user) == frozenset({_hash(key)})
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE token=%s', (_hash(key),)) == []
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (shared_claim,)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("restriction", ["models", "allowed_routes"])
|
||||
def test_link_only_enforces_key_permissions_on_the_first_and_cached_requests(
|
||||
gateway: Gateway, tmp_path: Path, restriction: str
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
allowed: Final = scenario.model()
|
||||
denied: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
key: Final = scenario.key(
|
||||
user_id=user,
|
||||
**({"models": [allowed]} if restriction == "models" else {"allowed_routes": ["/v1/models"]}),
|
||||
)
|
||||
token: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
|
||||
) as candidate:
|
||||
first: Final = _chat(candidate, denied, token)
|
||||
repeated: Final = _chat(candidate, denied, token)
|
||||
control: Final = (
|
||||
_chat(candidate, allowed, token)
|
||||
if restriction == "models"
|
||||
else candidate.request("GET", "/v1/models", key=token)
|
||||
)
|
||||
assert first.status_code == 403, first.text
|
||||
assert repeated.status_code == 403, repeated.text
|
||||
assert control.status_code == 200, control.text
|
||||
assert _mapped_token("sub", user) == _hash(key)
|
||||
assert _user_key_hashes(user) == frozenset({_hash(key)})
|
||||
|
||||
|
||||
def test_link_only_does_not_replace_an_existing_mapping_to_a_blocked_key(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
blocked: Final = scenario.key(user_id=user)
|
||||
available: Final = scenario.key(user_id=user)
|
||||
gateway.post("/jwt/key/mapping/new", {"jwt_claim_name": "sub", "jwt_claim_value": user, "key": blocked})
|
||||
gateway.post("/key/block", {"key": blocked})
|
||||
token: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
|
||||
) as candidate:
|
||||
first: Final = _chat(candidate, model, token)
|
||||
repeated: Final = _chat(candidate, model, token)
|
||||
assert first.status_code == 401, first.text
|
||||
assert repeated.status_code == 401, repeated.text
|
||||
assert _mapped_token("sub", user) == _hash(blocked)
|
||||
assert _user_key_hashes(user) == frozenset({_hash(blocked), _hash(available)})
|
||||
|
||||
|
||||
def test_link_only_exempts_proxy_admins_but_rechecks_the_role_after_a_cached_exemption(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
admin: Final = _token(private_key, user, scope="litellm_proxy_admin")
|
||||
ordinary: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
|
||||
) as candidate:
|
||||
first: Final = candidate.request("GET", "/v1/models", key=admin)
|
||||
repeated: Final = candidate.request("GET", "/v1/models", key=admin)
|
||||
denied: Final = candidate.request("GET", "/v1/models", key=ordinary)
|
||||
forbidden_route: Final = _chat(candidate, model, admin)
|
||||
assert first.status_code == 200, first.text
|
||||
assert repeated.status_code == 200, repeated.text
|
||||
assert denied.status_code == 403, denied.text
|
||||
assert forbidden_route.status_code == 401, forbidden_route.text
|
||||
assert _user_key_hashes(user) == frozenset()
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("user_id_upsert", [False, True])
|
||||
def test_link_only_keeps_user_provisioning_independent_of_key_creation(
|
||||
gateway: Gateway, tmp_path: Path, user_id_upsert: bool
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
subject: Final = f"integration-link-only-{uuid.uuid4().hex}"
|
||||
try:
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(
|
||||
tmp_path, "sub", behavior="link_only", map_existing_key=False, user_id_upsert=user_id_upsert
|
||||
),
|
||||
) as candidate:
|
||||
response: Final = _chat(candidate, model, _token(private_key, subject))
|
||||
assert response.status_code == (403 if user_id_upsert else 401), response.text
|
||||
users: Final = read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,))
|
||||
assert users == ([{"user_id": subject}] if user_id_upsert else [])
|
||||
assert _user_key_hashes(subject) == frozenset()
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (subject,)) == []
|
||||
finally:
|
||||
if read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,)):
|
||||
scenario.delete_user(subject)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("issuer_behavior", ["auto_register", "link_only"])
|
||||
def test_issuer_specific_registration_mode_overrides_the_global_mode(
|
||||
gateway: Gateway, tmp_path: Path, issuer_behavior: Literal["auto_register", "link_only"]
|
||||
) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
issuer: Final = f"https://integration.example.com/{uuid.uuid4().hex}"
|
||||
token: Final = _token(private_key, user, iss=issuer, aud="integration")
|
||||
path: Final = _config(
|
||||
tmp_path,
|
||||
"sub",
|
||||
behavior="auto_register" if issuer_behavior == "link_only" else "link_only",
|
||||
map_existing_key=False,
|
||||
issuers=(
|
||||
{
|
||||
"issuer": issuer,
|
||||
"jwks_url": jwks_url,
|
||||
"audience": "integration",
|
||||
"user_id_jwt_field": "sub",
|
||||
"virtual_key_claim_field": "sub",
|
||||
"unregistered_jwt_client_behavior": issuer_behavior,
|
||||
},
|
||||
),
|
||||
)
|
||||
with owned_proxy(gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=path) as candidate:
|
||||
response: Final = _chat(candidate, model, token)
|
||||
assert response.status_code == (403 if issuer_behavior == "link_only" else 200), response.text
|
||||
if issuer_behavior == "link_only":
|
||||
assert _user_key_hashes(user) == frozenset()
|
||||
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
|
||||
else:
|
||||
assert len(_user_key_hashes(user)) == 1
|
||||
assert _mapped_token("sub", user) in _user_key_hashes(user)
|
||||
assert _billed_key(response) == _mapped_token("sub", user)
|
||||
|
||||
|
||||
def test_link_only_links_after_a_cached_rejection_from_the_previous_mode(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
user: Final = scenario.user(user_role="internal_user")
|
||||
key: Final = scenario.key(user_id=user)
|
||||
token: Final = _token(private_key, user)
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="reject", map_existing_key=False),
|
||||
) as rejecting:
|
||||
denied: Final = _chat(rejecting, model, token)
|
||||
assert denied.status_code == 403, denied.text
|
||||
with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache:
|
||||
cached: Final = cache.get(f"jwt_key_mapping:sub:{user}")
|
||||
assert cached is not None and b"__NO_MAPPING__" in cached, cached
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{"JWT_PUBLIC_KEY_URL": jwks_url},
|
||||
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
|
||||
) as linking:
|
||||
accepted: Final = _chat(linking, model, token)
|
||||
assert accepted.status_code == 200, accepted.text
|
||||
assert _mapped_token("sub", user) == _hash(key)
|
||||
assert _user_key_hashes(user) == frozenset({_hash(key)})
|
||||
assert _billed_key(accepted) == _hash(key)
|
||||
|
|
|
|||
39
tests/integration/configuration/test_fips_mode_boot.py
Normal file
39
tests/integration/configuration/test_fips_mode_boot.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import _hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.client import Gateway
|
||||
from tests.integration._support.process import refused_boot_log
|
||||
|
||||
|
||||
def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path:
|
||||
config: Final = directory / "fips_boot.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "fips-model",
|
||||
"litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"},
|
||||
}
|
||||
],
|
||||
"general_settings": {"fips_mode": fips_mode},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None:
|
||||
if _hashlib.get_fips_mode() == 1:
|
||||
pytest.skip("this runner's OpenSSL is in FIPS mode")
|
||||
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True))
|
||||
assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log
|
||||
|
||||
|
||||
def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None:
|
||||
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced"))
|
||||
assert "general_settings.fips_mode='enforced' is not a boolean" in log, log
|
||||
|
|
@ -122,6 +122,9 @@ def test_patch_rename_is_rejected_and_model_keeps_working(gateway: Gateway) -> N
|
|||
"PATCH", f"/credentials/{name}", {"credential_name": name, "credential_info": {"description": "y"}}
|
||||
)
|
||||
assert same_name.status_code == 200, same_name.text
|
||||
assert read_rows(
|
||||
'SELECT credential_info FROM "LiteLLM_CredentialsTable" WHERE credential_name = %s', (name,)
|
||||
) == [{"credential_info": {"custom_llm_provider": "openai", "description": "y"}}]
|
||||
assert _served_credential(gateway, name)["credential_info"] == {
|
||||
"custom_llm_provider": "openai",
|
||||
"description": "y",
|
||||
|
|
|
|||
|
|
@ -669,7 +669,7 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
|
|||
import httpx2
|
||||
import yaml
|
||||
from integration._support.asgi import asgi_server
|
||||
from integration._support.mcp import _drain_sse_streams
|
||||
from integration._support.mcp import _acknowledge_notifications_after_dispatch, _drain_sse_streams
|
||||
from integration._support.process import owned_proxy
|
||||
from mcp import ClientSession, MCPError
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
|
|
@ -769,7 +769,9 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
|
|||
watchers = tuple(task for task in asyncio.all_tasks() if "_shutdown_watcher" in repr(task.get_coro()))
|
||||
await asyncio.wait_for(asyncio.gather(*watchers), 2)
|
||||
|
||||
hosted: Final = Starlette(routes=[Mount("/", app=app)], lifespan=lifespan)
|
||||
hosted: Final = Starlette(
|
||||
routes=[Mount("/", app=_acknowledge_notifications_after_dispatch(app))], lifespan=lifespan
|
||||
)
|
||||
with asgi_server(hosted) as upstream_url, httpx.Client() as client:
|
||||
config: Final = tmp_path / "proxy.yaml"
|
||||
config.write_text(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,724 @@
|
|||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import unquote
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
from integration._support.anthropic_sse import (
|
||||
LIFECYCLE,
|
||||
error_frame,
|
||||
event_type,
|
||||
event_types,
|
||||
message_id,
|
||||
message_json,
|
||||
message_stream,
|
||||
parse_sse,
|
||||
status_reply,
|
||||
stream_reply,
|
||||
user_prompt,
|
||||
)
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
JsonValue,
|
||||
Scenario,
|
||||
eventually,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.openai_wire import openai_error
|
||||
from integration._support.upstream import aws_event_stream_frame
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from redis import Redis
|
||||
|
||||
_MODEL: Final = "claude-sonnet-4-5-20250929"
|
||||
_API_KEY: Final = "synthetic-anthropic-key"
|
||||
_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
_BEDROCK_TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}'
|
||||
_CONVERSE_MODEL: Final = f"bedrock/converse/{_BEDROCK_MODEL}"
|
||||
_CONVERSE_STREAM_TARGET: Final = f"/model/{_BEDROCK_MODEL}/converse-stream"
|
||||
_OPENAI_BACKEND: Final = "gpt-4o-mini"
|
||||
_CHAT_ONLY_BACKEND: Final = "audit-chat-only"
|
||||
_CODEX_BACKEND: Final = "gpt-5.3-codex"
|
||||
_TEXT: Final = "Hello"
|
||||
_KEEPALIVE_SECONDS: Final = 0.2
|
||||
_LATE_ANSWER_SECONDS: Final = 1.0
|
||||
_OVERLOAD_CLASS: Final = "503_PROVIDER_OVERLOADED"
|
||||
|
||||
Shape = Literal["stream", "non-stream"]
|
||||
|
||||
|
||||
def _prompt() -> str:
|
||||
return "overloaded-" + uuid.uuid4().hex
|
||||
|
||||
|
||||
def _always(reply: Reply) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST", request
|
||||
return reply
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _late(reply: Reply, seconds: float) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST", request
|
||||
time.sleep(seconds)
|
||||
return reply
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _anthropic_sdk(gateway: Gateway) -> anthropic.Anthropic:
|
||||
return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
|
||||
|
||||
|
||||
def _async_anthropic_sdk(gateway: Gateway) -> anthropic.AsyncAnthropic:
|
||||
return anthropic.AsyncAnthropic(
|
||||
base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60
|
||||
)
|
||||
|
||||
|
||||
def _openai_sdk(gateway: Gateway) -> openai.OpenAI:
|
||||
return openai.OpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
|
||||
|
||||
|
||||
def _async_openai_sdk(gateway: Gateway) -> openai.AsyncOpenAI:
|
||||
return openai.AsyncOpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
|
||||
|
||||
|
||||
def _messages_body(model: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"stream": stream,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"num_retries": 0,
|
||||
}
|
||||
|
||||
|
||||
def _anthropic_error(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
assert response.headers["content-type"].startswith("application/json"), dict(response.headers)
|
||||
payload: Final = object_value(json.loads(response.content))
|
||||
assert payload["type"] == "error", response.text
|
||||
return object_value(payload["error"])
|
||||
|
||||
|
||||
def _openai_error(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
return object_value(object_value(json.loads(response.content))["error"])
|
||||
|
||||
|
||||
def _assert_overloaded(response: httpx.Response, *, message: str) -> None:
|
||||
assert response.status_code == 529, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == "overloaded_error", response.text
|
||||
assert message in string_value(error["message"]), response.text
|
||||
assert "content_block_delta" not in response.text, response.text
|
||||
|
||||
|
||||
def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None:
|
||||
assert response.status_code == status, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert message in string_value(error["message"]), response.text
|
||||
|
||||
|
||||
def _assert_internal_server_error(response: httpx.Response) -> None:
|
||||
assert response.status_code == 500, response.text
|
||||
error: Final = _openai_error(response)
|
||||
assert error["code"] == "500", response.text
|
||||
assert "InternalServerError" in string_value(error["message"]), response.text
|
||||
|
||||
|
||||
def _one_post(wire: Wire, target: str) -> None:
|
||||
assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)]
|
||||
|
||||
|
||||
def _spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows(
|
||||
"SELECT status, metadata->'error_information'->>'error_code' AS code, "
|
||||
"metadata->'error_information'->>'error_class' AS error_class, "
|
||||
"metadata->'error_information'->>'normalized_error' AS normalized_error "
|
||||
'FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
|
||||
(model,),
|
||||
),
|
||||
lambda rows: len(rows) >= expected,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
|
||||
def _assert_logged_failure(model: str, *, rows: int, error_class: str, code: str = "529") -> None:
|
||||
assert (
|
||||
_spend_rows(model, rows)
|
||||
== [{"status": "failure", "code": code, "error_class": error_class, "normalized_error": _OVERLOAD_CLASS}] * rows
|
||||
)
|
||||
|
||||
|
||||
def test_messages_529_answers_overloaded_error_to_the_sync_sdk_and_logs_each_call_once(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
sdk: Final = _anthropic_sdk(gateway)
|
||||
prompt: Final = _prompt()
|
||||
for _ in range(2):
|
||||
with pytest.raises(anthropic.APIStatusError) as caught:
|
||||
sdk.messages.create(
|
||||
model=model,
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
extra_body={"num_retries": 0},
|
||||
)
|
||||
assert caught.value.status_code == 529, caught.value.response.text
|
||||
_assert_overloaded(caught.value.response, message="scripted 529")
|
||||
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2
|
||||
_assert_logged_failure(model, rows=2, error_class="InternalServerError")
|
||||
|
||||
|
||||
async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(anthropic.APIStatusError) as caught:
|
||||
await _async_anthropic_sdk(gateway).messages.create(
|
||||
model=model,
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": _prompt()}],
|
||||
extra_body={"num_retries": 0},
|
||||
)
|
||||
assert caught.value.status_code == 529, caught.value.response.text
|
||||
_assert_overloaded(caught.value.response, message="scripted 529")
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_status", "error_class", "error_type"),
|
||||
((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")),
|
||||
)
|
||||
def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json(
|
||||
gateway: Gateway, upstream_status: int, error_class: str, error_type: str
|
||||
) -> None:
|
||||
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
assert response.status_code == upstream_status, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == error_type, response.text
|
||||
assert f"scripted {upstream_status}" in string_value(error["message"]), response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
_assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status))
|
||||
|
||||
|
||||
def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(anthropic.APIStatusError) as caught:
|
||||
with _anthropic_sdk(gateway).messages.stream(
|
||||
model=model,
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": _prompt()}],
|
||||
extra_body={"num_retries": 0},
|
||||
) as stream:
|
||||
stream.get_final_message()
|
||||
assert caught.value.status_code == 529, caught.value.response.text
|
||||
_assert_overloaded(caught.value.response, message="scripted 529")
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def _error_frames(text: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
object_value(json.loads(line.removeprefix("data:").strip()))
|
||||
for line in text.splitlines()
|
||||
if line.startswith("data:") and '"error"' in line
|
||||
)
|
||||
|
||||
|
||||
def test_messages_529_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_late(status_reply(529), _LATE_ANSWER_SECONDS)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, keepalive_seconds=_KEEPALIVE_SECONDS
|
||||
)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.headers["content-type"].startswith("text/event-stream"), dict(response.headers)
|
||||
assert response.text.startswith(":"), response.text
|
||||
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
|
||||
assert failure.event == "error", response.text
|
||||
assert object_value(failure.data["error"])["type"] == "overloaded_error", response.text
|
||||
assert "[DONE]" not in response.text, response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def _late_bedrock_deployment(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(
|
||||
model=f"bedrock/{_BEDROCK_MODEL}",
|
||||
api_key=_BEDROCK_TOKEN,
|
||||
aws_region_name="us-east-1",
|
||||
api_base=wire.url,
|
||||
keepalive_seconds=_KEEPALIVE_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _one_bedrock_post(wire: Wire) -> None:
|
||||
(request,) = wire.drain()
|
||||
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
|
||||
|
||||
|
||||
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = _late_bedrock_deployment(scenario, wire)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.text.startswith(":"), response.text
|
||||
assert "[DONE]" not in response.text, response.text
|
||||
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
|
||||
assert failure.event == "error", response.text
|
||||
error: Final = object_value(failure.data["error"])
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
|
||||
_one_bedrock_post(wire)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = _late_bedrock_deployment(scenario, wire)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.text.startswith(":"), response.text
|
||||
assert "event: error" not in response.text, response.text
|
||||
(frame,) = _error_frames(response.text)
|
||||
assert object_value(frame["error"])["code"] == "503", response.text
|
||||
assert response.text.rstrip().endswith("data: [DONE]"), response.text
|
||||
_one_bedrock_post(wire)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def test_chat_completions_overloaded_frame_after_content_ends_with_a_500_error_frame(gateway: Gateway) -> None:
|
||||
prompt: Final = _prompt()
|
||||
upstream: Final = (*message_stream(prompt, _MODEL, _TEXT)[:3], error_frame(529, "Overloaded"))
|
||||
with wire_server(_always(stream_reply(upstream))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _TEXT in response.text, response.text
|
||||
(frame,) = _error_frames(response.text)
|
||||
error: Final = object_value(frame["error"])
|
||||
assert error["code"] == "500", response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def _overloaded_before_content(prompt: str) -> Reply:
|
||||
return stream_reply((*message_stream(prompt, _MODEL, _TEXT)[:1], error_frame(529, "Overloaded")))
|
||||
|
||||
|
||||
def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_relayed_as_an_overloaded_event(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
prompt: Final = _prompt()
|
||||
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=True))
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = parse_sse(response.text)
|
||||
assert event_types(events) == ("message_start", "error"), response.text
|
||||
assert message_id(events) == prompt, response.text
|
||||
assert events[1].data == {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
|
||||
_one_post(wire, "/v1/messages")
|
||||
assert _spend_rows(model, 1) == [
|
||||
{"status": "success", "code": None, "error_class": None, "normalized_error": None}
|
||||
]
|
||||
|
||||
|
||||
def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None:
|
||||
prompt: Final = _prompt()
|
||||
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
|
||||
)
|
||||
_assert_internal_server_error(response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(openai.APIStatusError) as caught:
|
||||
_openai_sdk(gateway).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0}
|
||||
)
|
||||
assert caught.value.status_code == 500, caught.value.response.text
|
||||
_assert_internal_server_error(caught.value.response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500")
|
||||
|
||||
|
||||
async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
with pytest.raises(openai.APIStatusError) as caught:
|
||||
await _async_openai_sdk(gateway).chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": _prompt()}],
|
||||
stream=True,
|
||||
extra_body={"num_retries": 0},
|
||||
)
|
||||
assert caught.value.status_code == 500, caught.value.response.text
|
||||
_assert_internal_server_error(caught.value.response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0}
|
||||
)
|
||||
_assert_internal_server_error(response)
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_api_error(response, status=503, message="Bedrock is unable to process your request.")
|
||||
(request,) = wire.drain()
|
||||
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes:
|
||||
return aws_event_stream_frame(
|
||||
{":event-type": event, ":content-type": "application/json", ":message-type": "event"},
|
||||
json.dumps(payload).encode(),
|
||||
)
|
||||
|
||||
|
||||
def _converse_503_after_text() -> Reply:
|
||||
unavailable: Final = aws_event_stream_frame(
|
||||
{
|
||||
":exception-type": "serviceUnavailableException",
|
||||
":content-type": "application/json",
|
||||
":message-type": "exception",
|
||||
},
|
||||
_BEDROCK_503,
|
||||
)
|
||||
frames: Final = (
|
||||
_converse_event("messageStart", {"role": "assistant"}),
|
||||
_converse_event("contentBlockDelta", {"delta": {"text": _TEXT}, "contentBlockIndex": 0}),
|
||||
unavailable,
|
||||
)
|
||||
return Reply(body=b"".join(frames), content_type="application/vnd.amazon.eventstream")
|
||||
|
||||
|
||||
def _converse_deployment(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url)
|
||||
|
||||
|
||||
def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None:
|
||||
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = parse_sse(response.text)
|
||||
types: Final = event_types(events)
|
||||
assert "message_stop" not in types, response.text
|
||||
assert types.index("content_block_delta") < types.index("error"), response.text
|
||||
assert _TEXT in response.text, response.text
|
||||
(failure,) = (event for event in events if event_type(event) == "error")
|
||||
error: Final = object_value(failure.data["error"])
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
|
||||
_one_post(wire, _CONVERSE_STREAM_TARGET)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None:
|
||||
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _TEXT in response.text, response.text
|
||||
(frame,) = _error_frames(response.text)
|
||||
error: Final = object_value(frame["error"])
|
||||
assert error["code"] == "503", response.text
|
||||
_one_post(wire, _CONVERSE_STREAM_TARGET)
|
||||
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_api_error(response, status=503, message="scripted 503")
|
||||
_one_post(wire, "/v1/chat/completions")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
|
||||
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
_assert_api_error(response, status=503, message="scripted 503")
|
||||
_one_post(wire, "/v1/responses")
|
||||
|
||||
|
||||
_SETTLED_ANSWERS: Final = 4
|
||||
|
||||
|
||||
def _messages_to(gateway: Gateway, model: str) -> tuple[tuple[str, httpx.Response], ...]:
|
||||
prompts: Final = tuple(_prompt() for _ in range(_SETTLED_ANSWERS))
|
||||
return tuple(
|
||||
(prompt, gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False)))
|
||||
for prompt in prompts
|
||||
)
|
||||
|
||||
|
||||
def _failure_rows_of(models: tuple[str, str], expected: int) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows(
|
||||
"SELECT metadata->'error_information'->>'error_code' AS code, "
|
||||
"metadata->'error_information'->>'error_class' AS error_class, "
|
||||
"metadata->'error_information'->>'normalized_error' AS normalized_error "
|
||||
'FROM "LiteLLM_SpendLogs" WHERE status = %s AND model_group IN (%s, %s)',
|
||||
("failure", *models),
|
||||
),
|
||||
lambda rows: len(rows) >= expected,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _paused_deployment(gateway: Gateway) -> Generator[tuple[Wire, str, str]]:
|
||||
unpaused: Final = Reply(body=message_json("msg_unpaused", _MODEL, _TEXT))
|
||||
with wire_server(_always(unpaused)) as wire, gateway.scenario() as scenario:
|
||||
identity: Final = f"audit-paused-{uuid.uuid4().hex}"
|
||||
name: Final = scenario.model(
|
||||
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, model_info={"id": identity}
|
||||
)
|
||||
paused: Final = gateway.request("PATCH", f"/model/{identity}/update", {"blocked": True})
|
||||
assert paused.status_code == 200, paused.text
|
||||
yield wire, identity, name
|
||||
|
||||
|
||||
def _assert_refused_without_an_upstream_call(
|
||||
gateway: Gateway, wire: Wire, target: str, *, status: int, error_type: str, message: str
|
||||
) -> None:
|
||||
answers: Final = eventually(
|
||||
lambda: _messages_to(gateway, target),
|
||||
lambda settled: all(response.status_code != 200 for _, response in settled),
|
||||
seconds=90,
|
||||
)
|
||||
texts: Final = [response.text for _, response in answers]
|
||||
assert [response.status_code for _, response in answers] == [status] * _SETTLED_ANSWERS, texts
|
||||
errors: Final = [_anthropic_error(response) for _, response in answers]
|
||||
assert [error["type"] for error in errors] == [error_type] * _SETTLED_ANSWERS, texts
|
||||
assert all(message in string_value(error["message"]) for error in errors), texts
|
||||
reached: Final = {user_prompt(JSON_OBJECT.validate_json(request.body)) for request in wire.drain()}
|
||||
assert reached.isdisjoint(prompt for prompt, _ in answers), reached
|
||||
|
||||
|
||||
def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None:
|
||||
with _paused_deployment(gateway) as (wire, identity, name):
|
||||
_assert_refused_without_an_upstream_call(
|
||||
gateway, wire, identity, status=503, error_type="api_error", message="is currently paused"
|
||||
)
|
||||
rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS)
|
||||
assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == {
|
||||
("503", "ServiceUnavailableError", _OVERLOAD_CLASS)
|
||||
}, rows
|
||||
|
||||
|
||||
def test_messages_to_a_paused_model_group_named_by_its_name_answer_403_permission_error(gateway: Gateway) -> None:
|
||||
with _paused_deployment(gateway) as (wire, _, name):
|
||||
_assert_refused_without_an_upstream_call(
|
||||
gateway, wire, name, status=403, error_type="permission_error", message="Model is blocked"
|
||||
)
|
||||
|
||||
|
||||
_MALFORMED_529_BODIES: Final = {
|
||||
"empty": b"",
|
||||
"text-5k": b"x" * 5000,
|
||||
"html": b"<html><body><h1>529</h1></body></html>",
|
||||
"api_error-type": b'{"type": "error", "error": {"type": "api_error", "message": "capacity"}}',
|
||||
}
|
||||
|
||||
|
||||
def _overloaded_for_sad_prompts(body: bytes) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
prompt: Final = user_prompt(object_value(json.loads(request.body)))
|
||||
if prompt.startswith("sad:"):
|
||||
return Reply(status=529, body=body, content_type="text/plain" if body[:1] != b"{" else "application/json")
|
||||
return Reply(body=message_json(f"msg_{prompt}", _MODEL, _TEXT))
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", _MALFORMED_529_BODIES.keys())
|
||||
def test_messages_529_with_a_malformed_body_still_answers_overloaded_error_and_unrelated_calls_keep_working(
|
||||
gateway: Gateway, body: str
|
||||
) -> None:
|
||||
with wire_server(_overloaded_for_sad_prompts(_MALFORMED_529_BODIES[body])) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
sad: Final = gateway.request("POST", "/v1/messages", _messages_body(model, "sad:" + _prompt(), stream=False))
|
||||
assert sad.status_code == 529, sad.text
|
||||
assert _anthropic_error(sad)["type"] == "overloaded_error", sad.text
|
||||
prompt: Final = _prompt()
|
||||
healthy: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False))
|
||||
assert healthy.status_code == 200, healthy.text
|
||||
assert object_value(json.loads(healthy.content))["id"] == f"msg_{prompt}", healthy.text
|
||||
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2
|
||||
assert sorted(_spend_rows(model, 2), key=lambda row: str(row["status"])) == [
|
||||
{
|
||||
"status": "failure",
|
||||
"code": "529",
|
||||
"error_class": "InternalServerError",
|
||||
"normalized_error": _OVERLOAD_CLASS,
|
||||
},
|
||||
{"status": "success", "code": None, "error_class": None, "normalized_error": None},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("upstream_status", "error_type"), ((500, "api_error"), (429, "rate_limit_error")))
|
||||
def test_messages_other_upstream_statuses_keep_their_status_and_type(
|
||||
gateway: Gateway, upstream_status: int, error_type: str
|
||||
) -> None:
|
||||
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
assert response.status_code == upstream_status, response.text
|
||||
assert _anthropic_error(response)["type"] == error_type, response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
|
||||
|
||||
def _anthropic_rejection(status: int, error_type: str, message: str) -> Reply:
|
||||
return Reply(
|
||||
status=status, body=json.dumps({"type": "error", "error": {"type": error_type, "message": message}}).encode()
|
||||
)
|
||||
|
||||
|
||||
_REJECTIONS_SAYING_OVERLOADED: Final = {
|
||||
"400-tool-name": _anthropic_rejection(
|
||||
400, "invalid_request_error", "tools.0.name: 'Overloaded' is not a valid tool name"
|
||||
),
|
||||
"404-model": _anthropic_rejection(404, "not_found_error", "model: claude-Overloaded"),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
|
||||
@pytest.mark.parametrize("rejection", _REJECTIONS_SAYING_OVERLOADED.keys())
|
||||
def test_messages_4xx_whose_text_says_overloaded_is_not_answered_as_an_overload(
|
||||
gateway: Gateway, rejection: str, shape: Shape
|
||||
) -> None:
|
||||
with wire_server(_always(_REJECTIONS_SAYING_OVERLOADED[rejection])) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
|
||||
)
|
||||
assert response.status_code == 500, response.text
|
||||
error: Final = _anthropic_error(response)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Overloaded" in string_value(error["message"]), response.text
|
||||
_one_post(wire, "/v1/messages")
|
||||
assert _spend_rows(model, 1) == [
|
||||
{
|
||||
"status": "failure",
|
||||
"code": "500",
|
||||
"error_class": "InternalServerError",
|
||||
"normalized_error": _OVERLOAD_CLASS,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _string_values(cache: Redis) -> tuple[bytes, ...]:
|
||||
keys: Final = tuple(key for key in cache.scan_iter(count=1000) if cache.type(key) == b"string")
|
||||
return tuple(value for value in cache.mget(keys) if value is not None) if keys else ()
|
||||
|
||||
|
||||
def _cached_somewhere(served_id: str) -> bool:
|
||||
with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache:
|
||||
return any(served_id.encode() in value for value in _string_values(cache))
|
||||
|
||||
|
||||
def _healthy_until(overloaded: threading.Event) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
if overloaded.is_set():
|
||||
return status_reply(529)
|
||||
prompt: Final = user_prompt(object_value(json.loads(request.body)))
|
||||
return stream_reply(message_stream(f"msg_{prompt}", _MODEL, _TEXT))
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def test_messages_cache_hit_is_served_while_the_upstream_is_overloaded(gateway: Gateway) -> None:
|
||||
overloaded: Final = threading.Event()
|
||||
with wire_server(_healthy_until(overloaded)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
|
||||
prompt: Final = _prompt()
|
||||
body: Final = _messages_body(model, prompt, stream=True)
|
||||
first: Final = gateway.request("POST", "/v1/messages", body)
|
||||
assert first.status_code == 200, first.text
|
||||
assert event_types(parse_sse(first.text)) == LIFECYCLE, first.text
|
||||
assert message_id(parse_sse(first.text)) == f"msg_{prompt}", first.text
|
||||
eventually(lambda: _cached_somewhere(f"msg_{prompt}"), bool, seconds=30)
|
||||
overloaded.set()
|
||||
replay: Final = gateway.request("POST", "/v1/messages", body)
|
||||
assert replay.status_code == 200, replay.text
|
||||
assert message_id(parse_sse(replay.text)) == f"msg_{prompt}", replay.text
|
||||
fresh: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
|
||||
_assert_overloaded(fresh, message="scripted 529")
|
||||
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2
|
||||
|
|
@ -135,6 +135,13 @@ def _drop_or_500(request: Request, marker: str) -> Reply:
|
|||
return openai_error(500)
|
||||
|
||||
|
||||
def _overloaded_while(outage: threading.Event) -> Callable[[Request, str], Reply]:
|
||||
def first_attempt(request: Request, marker: str) -> Reply:
|
||||
return status_reply(529) if outage.is_set() else _served(request, marker, 1)
|
||||
|
||||
return first_attempt
|
||||
|
||||
|
||||
def _outage(statuses: Mapping[str, int]) -> Callable[[Request, str], Reply]:
|
||||
def first_attempt(request: Request, marker: str) -> Reply:
|
||||
if request.target == "/v1/messages":
|
||||
|
|
@ -165,7 +172,7 @@ class _Upstream:
|
|||
return self.first_attempt(request, marker)
|
||||
|
||||
|
||||
def _config(wire: Wire, directory: Path, models: _Models) -> Path:
|
||||
def _config(wire: Wire, directory: Path, models: _Models, *, retries: int = 1) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["model_list"] = [
|
||||
{
|
||||
|
|
@ -174,7 +181,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path:
|
|||
"model": f"anthropic/{_ANTHROPIC_BACKEND}",
|
||||
"api_base": wire.url,
|
||||
"api_key": _ANTHROPIC_KEY,
|
||||
"num_retries": 1,
|
||||
"num_retries": retries,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -183,7 +190,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path:
|
|||
"model": f"openai/{_OPENAI_BACKEND}",
|
||||
"api_base": wire.url + "/v1",
|
||||
"api_key": _OPENAI_KEY,
|
||||
"num_retries": 1,
|
||||
"num_retries": retries,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -408,3 +415,68 @@ async def test_outage_on_every_first_attempt_is_absorbed_by_the_deployment_budge
|
|||
_assert_completed_on_the_second_attempt(item)
|
||||
_assert_each_served_id_landed_exactly_once(models, served)
|
||||
assert all(upstream.attempts.count(call.marker) == 2 for call in calls)
|
||||
|
||||
|
||||
def _messages_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows(
|
||||
"SELECT request_id, status, metadata->'error_information'->>'error_code' AS code "
|
||||
'FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
|
||||
(model,),
|
||||
),
|
||||
lambda found: len(found) >= expected,
|
||||
seconds=90,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
async def test_overloaded_outage_during_a_burst_with_a_worker_sigkill_answers_529_and_lands_every_call_once(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
models: Final = _models()
|
||||
calls: Final = _calls((("messages", True, 12), ("messages", False, 6)))
|
||||
release: Final = threading.Event()
|
||||
held: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
outage: Final = threading.Event()
|
||||
outage.set()
|
||||
upstream: Final = _Upstream(Attempts(), _overloaded_while(outage), held, release)
|
||||
with wire_server(upstream) as wire:
|
||||
config: Final = _config(wire, tmp_path, models, retries=0)
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned:
|
||||
candidate: Final = owned.gateway
|
||||
base_url: Final = str(candidate.client.base_url)
|
||||
workers: Final = eventually(
|
||||
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
|
||||
lambda pids: len(pids) == 2,
|
||||
seconds=30,
|
||||
)
|
||||
burst: Final = asyncio.create_task(
|
||||
_burst(base_url, candidate.key, models, calls, tolerate_transport_errors=True)
|
||||
)
|
||||
await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(calls), 90)
|
||||
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
|
||||
assert sum(held_by.values()) == len(calls), held_by
|
||||
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
|
||||
with owned.replacing_worker(victim_pid) as kill:
|
||||
kill.freeze()
|
||||
await asyncio.to_thread(kill)
|
||||
release.set()
|
||||
served: Final = await burst
|
||||
assert held_by[survivor_pid] >= 5, held_by
|
||||
assert len(served) == held_by[survivor_pid], (held_by, len(served))
|
||||
for item in served:
|
||||
assert item.status == 529, (item.call, item.text)
|
||||
assert "overloaded_error" in item.text, (item.call, item.text)
|
||||
assert "content_block_delta" not in item.text, (item.call, item.text)
|
||||
outage.clear()
|
||||
follow_up: Final = _Call(endpoint="messages", stream=True, marker=uuid.uuid4().hex)
|
||||
(answered,) = await _burst(base_url, candidate.key, models, (follow_up,))
|
||||
assert answered.status == 200, answered.text
|
||||
assert message_id(parse_sse(answered.text)) == f"msg_{follow_up.marker}_a1", answered.text
|
||||
rows: Final = _messages_rows(models.messages, len(served) + 1)
|
||||
assert sorted(string_value(row["status"]) for row in rows) == ["failure"] * len(served) + ["success"], rows
|
||||
assert {row["code"] for row in rows if row["status"] == "failure"} == {"529"}, rows
|
||||
assert [row["request_id"] for row in rows if row["status"] == "success"] == [
|
||||
f"msg_{follow_up.marker}_a1"
|
||||
], rows
|
||||
assert all(upstream.attempts.count(item.call.marker) == 1 for item in (*served, answered))
|
||||
|
|
|
|||
|
|
@ -397,14 +397,14 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway:
|
|||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1)
|
||||
response: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "deployment"))
|
||||
_assert_rejected_before_content(response, wire, status=500, attempts=2, error="error")
|
||||
_assert_rejected_before_content(response, wire, status=529, attempts=2, error="overloaded_error")
|
||||
rows: Final = eventually(
|
||||
lambda: _request_rows(gateway, response.headers["x-litellm-call-id"]),
|
||||
lambda values: len(values) >= 1,
|
||||
seconds=70,
|
||||
)
|
||||
rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0))
|
||||
_assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error")
|
||||
_assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error")
|
||||
rejection_rows: Final = eventually(
|
||||
lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]),
|
||||
lambda values: len(values) >= 1,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,404 @@
|
|||
import gzip
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.anthropic_sse import error_body
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
JsonValue,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.otlp_sink import Span, SpanSinks, recorded_spans, spans_for_trace
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.prometheus_series import Sample, scrape
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
_MODEL: Final = "claude-sonnet-4-5-20250929"
|
||||
_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}'
|
||||
_MESSAGES_ANTHROPIC: Final = "audit-signals-messages-anthropic"
|
||||
_MESSAGES_BEDROCK: Final = "audit-signals-messages-bedrock"
|
||||
_CHAT_ANTHROPIC: Final = "audit-signals-chat-anthropic"
|
||||
_TRACED_MESSAGES: Final = "audit-signals-traced-messages"
|
||||
_TRACED_CHAT: Final = "audit-signals-traced-chat"
|
||||
_DATADOG_MESSAGES_ANTHROPIC: Final = "audit-signals-datadog-messages-anthropic"
|
||||
_DATADOG_MESSAGES_BEDROCK: Final = "audit-signals-datadog-messages-bedrock"
|
||||
_DATADOG_CHAT_ANTHROPIC: Final = "audit-signals-datadog-chat-anthropic"
|
||||
_GUARDED_MESSAGES: Final = "audit-signals-guarded-messages"
|
||||
_GUARDED_CHAT: Final = "audit-signals-guarded-chat"
|
||||
_AKTO_DOWN: Final = "audit-signals-akto-down"
|
||||
_AKTO_UNREACHABLE: Final = "Akto guardrail service unreachable"
|
||||
_LATE_PASS_THROUGH_BACKEND: Final = "claude-late-pass-through"
|
||||
_KEEPALIVE_SECONDS: Final = 0.2
|
||||
_LATE_ANSWER_SECONDS: Final = 1.0
|
||||
_OVERLOADED: Final = Reply(status=529, body=error_body(529, "Overloaded"))
|
||||
_SERVER: Final = 2
|
||||
|
||||
AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
proxy: Gateway
|
||||
wire: Wire
|
||||
spans: str
|
||||
datadog: Wire
|
||||
|
||||
|
||||
def _respond(request: Request) -> Reply:
|
||||
assert request.method == "POST", request
|
||||
if unquote(request.target).startswith("/model/"):
|
||||
return Reply(status=503, body=_BEDROCK_503)
|
||||
if json.loads(request.body).get("model") == _LATE_PASS_THROUGH_BACKEND:
|
||||
time.sleep(_LATE_ANSWER_SECONDS)
|
||||
return _OVERLOADED
|
||||
|
||||
|
||||
def _datadog_intake(request: Request) -> Reply:
|
||||
assert request.target == "/api/v2/logs", request.target
|
||||
return Reply(status=202, body=b"{}")
|
||||
|
||||
|
||||
def _config(directory: Path, tracing: Path, wire: Wire) -> Path:
|
||||
configuration: Final = JSON_OBJECT.validate_python(yaml.safe_load(tracing.read_text()))
|
||||
anthropic: Final = {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": "synthetic-anthropic-key"}
|
||||
bedrock: Final = {
|
||||
"model": f"bedrock/{_BEDROCK_MODEL}",
|
||||
"api_base": wire.url,
|
||||
"api_key": "synthetic-bedrock-bearer",
|
||||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
deployments: Final = (
|
||||
(_MESSAGES_ANTHROPIC, anthropic),
|
||||
(_MESSAGES_BEDROCK, bedrock),
|
||||
(_CHAT_ANTHROPIC, anthropic),
|
||||
(_TRACED_MESSAGES, anthropic),
|
||||
(_TRACED_CHAT, anthropic),
|
||||
(_DATADOG_MESSAGES_ANTHROPIC, anthropic),
|
||||
(_DATADOG_MESSAGES_BEDROCK, bedrock),
|
||||
(_DATADOG_CHAT_ANTHROPIC, anthropic),
|
||||
(_GUARDED_MESSAGES, anthropic),
|
||||
(_GUARDED_CHAT, anthropic),
|
||||
)
|
||||
akto_down: Final = {
|
||||
"guardrail_name": _AKTO_DOWN,
|
||||
"litellm_params": {
|
||||
"guardrail": "akto",
|
||||
"mode": "pre_call",
|
||||
"akto_base_url": "http://127.0.0.1:9",
|
||||
"akto_api_key": "synthetic-akto-key",
|
||||
"unreachable_fallback": "fail_closed",
|
||||
},
|
||||
}
|
||||
path: Final = directory / "anthropic-overloaded-failure-signals.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**configuration,
|
||||
"model_list": [{"model_name": name, "litellm_params": params} for name, params in deployments],
|
||||
"router_settings": {**object_value(configuration.get("router_settings", {})), "num_retries": 0},
|
||||
"litellm_settings": {
|
||||
**object_value(configuration["litellm_settings"]),
|
||||
"sse_keepalive_ping_interval_seconds": _KEEPALIVE_SECONDS,
|
||||
},
|
||||
"guardrails": [akto_down],
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(
|
||||
tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter
|
||||
) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals")
|
||||
multiprocess: Final = directory / "prometheus"
|
||||
multiprocess.mkdir()
|
||||
tracing: Final = otel_audit_config(directory, {"callbacks": ["otel", "prometheus", "datadog"]})
|
||||
with (
|
||||
wire_server(_respond) as wire,
|
||||
wire_server(_datadog_intake) as datadog,
|
||||
gateway_from_environment() as shared,
|
||||
owned_proxy(
|
||||
shared,
|
||||
directory,
|
||||
{
|
||||
"OTEL_EXPORTER": "http/json",
|
||||
"OTEL_ENDPOINT": audit_sinks.operator,
|
||||
"PROMETHEUS_MULTIPROC_DIR": str(multiprocess),
|
||||
"DD_API_KEY": "synthetic-datadog-key",
|
||||
"DD_SITE": "datadog.invalid",
|
||||
"DD_BASE_URL": datadog.url,
|
||||
"DD_BATCH_SIZE": "1",
|
||||
"ANTHROPIC_API_BASE": wire.url,
|
||||
"ANTHROPIC_API_KEY": "synthetic-anthropic-key",
|
||||
},
|
||||
config=_config(directory, tracing, wire),
|
||||
workers=2,
|
||||
) as owned,
|
||||
):
|
||||
yield _Rig(owned, wire, audit_sinks.operator, datadog)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def v2_rig(
|
||||
tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter
|
||||
) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals-v2")
|
||||
tracing: Final = otel_audit_config(directory, {"callbacks": ["otel"]})
|
||||
with (
|
||||
wire_server(_respond) as wire,
|
||||
wire_server(_datadog_intake) as datadog,
|
||||
gateway_from_environment() as shared,
|
||||
owned_proxy(
|
||||
shared,
|
||||
directory,
|
||||
{"OTEL_EXPORTER": "http/json", "OTEL_ENDPOINT": audit_sinks.operator, "LITELLM_OTEL_V2": "1"},
|
||||
config=_config(directory, tracing, wire),
|
||||
workers=2,
|
||||
) as owned,
|
||||
):
|
||||
yield _Rig(owned, wire, audit_sinks.operator, datadog)
|
||||
|
||||
|
||||
def _send(
|
||||
rig: _Rig,
|
||||
route: str,
|
||||
model: str,
|
||||
trace_id: str | None = None,
|
||||
*,
|
||||
key: str | None = None,
|
||||
extra: Mapping[str, JsonValue] | None = None,
|
||||
) -> httpx.Response:
|
||||
traceparent: Final = {} if trace_id is None else {"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}
|
||||
return rig.proxy.request(
|
||||
"POST",
|
||||
route,
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}],
|
||||
"num_retries": 0,
|
||||
**(extra or {}),
|
||||
},
|
||||
key=key,
|
||||
headers={"anthropic-version": "2023-06-01", **traceparent},
|
||||
)
|
||||
|
||||
|
||||
def _series(samples: Sequence[Sample], name: str, model: str) -> tuple[Sample, ...]:
|
||||
return tuple(sample for sample in samples if sample.name == name and sample.labels.get("requested_model") == model)
|
||||
|
||||
|
||||
def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
|
||||
return eventually(lambda: _series(scrape(rig.proxy), name, model), bool, seconds=30)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered", "deployment_status", "deployment_class"),
|
||||
(
|
||||
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status(
|
||||
rig: _Rig, route: str, model: str, answered: int, deployment_status: str, deployment_class: str
|
||||
) -> None:
|
||||
rig.wire.drain()
|
||||
response: Final = _send(rig, route, model)
|
||||
assert response.status_code == answered, response.text
|
||||
assert len(rig.wire.drain()) == 1
|
||||
failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model)
|
||||
assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [
|
||||
(str(answered), route, 1.0)
|
||||
], failed
|
||||
deployment: Final = _until(rig, "litellm_deployment_failure_responses_total", model)
|
||||
assert [(sample.labels["exception_status"], sample.value) for sample in deployment] == [(deployment_status, 1.0)], (
|
||||
deployment
|
||||
)
|
||||
assert all(deployment_class in sample.labels["exception_class"] for sample in deployment), deployment
|
||||
|
||||
|
||||
def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]:
|
||||
traced: Final = spans_for_trace(recorded_spans(rig.spans, since)[1], trace_id)
|
||||
return tuple(span for span in traced if span["kind"] == _SERVER)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"),
|
||||
),
|
||||
)
|
||||
def test_the_server_span_carries_the_status_the_route_answered(
|
||||
rig: _Rig, route: str, model: str, answered: int
|
||||
) -> None:
|
||||
rig.wire.drain()
|
||||
since, _ = recorded_spans(rig.spans)
|
||||
trace_id: Final = uuid.uuid4().hex
|
||||
response: Final = _send(rig, route, model, trace_id)
|
||||
assert response.status_code == answered, response.text
|
||||
assert len(rig.wire.drain()) == 1
|
||||
server: Final = eventually(
|
||||
lambda: _server_spans(rig, since, trace_id),
|
||||
lambda spans: any("error.code" in span["attributes"] for span in spans),
|
||||
seconds=40,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert [
|
||||
(str(span["attributes"].get("http.response.status_code")), str(span["attributes"].get("error.code")))
|
||||
for span in server
|
||||
] == [(str(answered), str(answered))], server
|
||||
|
||||
|
||||
def _inflated(request: Request) -> bytes:
|
||||
return gzip.decompress(request.body) if request.headers.get("content-encoding") == "gzip" else request.body
|
||||
|
||||
|
||||
def _datadog_events(batches: Sequence[Request]) -> Iterator[JsonValue]:
|
||||
for batch in batches:
|
||||
yield from json.loads(_inflated(batch))
|
||||
|
||||
|
||||
def _datadog_messages(batches: Sequence[Request]) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
JSON_OBJECT.validate_json(string_value(object_value(event)["message"])) for event in _datadog_events(batches)
|
||||
)
|
||||
|
||||
|
||||
def _keyed_by(message: Mapping[str, JsonValue], alias: str) -> bool:
|
||||
proxy_side: Final = message.get("user_api_key_dict")
|
||||
llm_side: Final = message.get("metadata")
|
||||
return any(isinstance(side, dict) and side.get("user_api_key_alias") == alias for side in (proxy_side, llm_side))
|
||||
|
||||
|
||||
def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[str, str], ...]:
|
||||
batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches
|
||||
|
||||
def delivered() -> tuple[tuple[str, str], ...]:
|
||||
batches.extend(rig.datadog.drain())
|
||||
return tuple(
|
||||
sorted(
|
||||
("proxy", str(message["status_code"]))
|
||||
if "user_api_key_dict" in message
|
||||
else ("llm", str(object_value(message["error_information"])["error_code"]))
|
||||
for message in _datadog_messages(batches)
|
||||
if _keyed_by(message, alias)
|
||||
)
|
||||
)
|
||||
|
||||
return eventually(delivered, lambda codes: len(codes) >= expected, seconds=40, return_last_on_timeout=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered", "deployment_status"),
|
||||
(
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status(
|
||||
rig: _Rig, route: str, model: str, answered: int, deployment_status: str
|
||||
) -> None:
|
||||
rig.wire.drain()
|
||||
rig.datadog.drain()
|
||||
alias: Final = f"audit-signals-datadog-{uuid.uuid4().hex}"
|
||||
with rig.proxy.scenario() as scenario:
|
||||
response: Final = _send(rig, route, model, key=scenario.key(key_alias=alias, models=[model]))
|
||||
assert response.status_code == answered, response.text
|
||||
assert len(rig.wire.drain()) == 1
|
||||
assert _datadog_status_codes(rig, alias, 2) == (("llm", deployment_status), ("proxy", str(answered)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"),
|
||||
pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"),
|
||||
),
|
||||
)
|
||||
def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes_status(
|
||||
rig: _Rig, route: str, model: str, answered: int
|
||||
) -> None:
|
||||
rig.wire.drain()
|
||||
rig.datadog.drain()
|
||||
alias: Final = f"audit-signals-guarded-{uuid.uuid4().hex}"
|
||||
with rig.proxy.scenario() as scenario:
|
||||
response: Final = _send(
|
||||
rig, route, model, key=scenario.key(key_alias=alias, models=[model]), extra={"guardrails": [_AKTO_DOWN]}
|
||||
)
|
||||
assert response.status_code == answered, response.text
|
||||
assert _AKTO_UNREACHABLE in response.text, response.text
|
||||
assert rig.wire.drain() == ()
|
||||
failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model)
|
||||
assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [
|
||||
(str(answered), route, 1.0)
|
||||
], failed
|
||||
assert _datadog_status_codes(rig, alias, 2) == (("llm", str(answered)), ("proxy", str(answered)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "model", "answered"),
|
||||
(
|
||||
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
|
||||
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
|
||||
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
|
||||
),
|
||||
)
|
||||
def test_the_otel_v2_server_span_carries_the_status_the_route_answered(
|
||||
v2_rig: _Rig, route: str, model: str, answered: int
|
||||
) -> None:
|
||||
v2_rig.wire.drain()
|
||||
since, _ = recorded_spans(v2_rig.spans)
|
||||
trace_id: Final = uuid.uuid4().hex
|
||||
response: Final = _send(v2_rig, route, model, trace_id)
|
||||
assert response.status_code == answered, response.text
|
||||
assert len(v2_rig.wire.drain()) == 1
|
||||
server: Final = eventually(
|
||||
lambda: _server_spans(v2_rig, since, trace_id),
|
||||
lambda spans: any("litellm.provider.error.code" in span["attributes"] for span in spans),
|
||||
seconds=40,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert [str(span["attributes"].get("litellm.provider.error.code")) for span in server] == [str(answered)], server
|
||||
|
||||
|
||||
def test_an_anthropic_pass_through_stream_that_fails_after_the_keepalive_opened_relays_the_upstream_body(
|
||||
rig: _Rig,
|
||||
) -> None:
|
||||
rig.wire.drain()
|
||||
response: Final = rig.proxy.request(
|
||||
"POST",
|
||||
"/anthropic/v1/messages",
|
||||
{
|
||||
"model": _LATE_PASS_THROUGH_BACKEND,
|
||||
"max_tokens": 16,
|
||||
"stream": True,
|
||||
"messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}],
|
||||
},
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
pings, _, relayed = response.text.rpartition("\n\n")
|
||||
assert pings and set(pings.split("\n")) <= {"", ": ping"}, response.text
|
||||
assert relayed == _OVERLOADED.body.decode(), response.text
|
||||
assert [unquote(request.target) for request in rig.wire.drain()] == ["/v1/messages"]
|
||||
|
|
@ -5,6 +5,7 @@ from pathlib import Path
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from litellm import get_model_info
|
||||
from integration._support.client import Gateway, object_value
|
||||
from integration._support.process import owned_proxy
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -104,6 +105,21 @@ def test_v1_models_carries_cost_map_context_window_for_a_known_model(gateway: Ga
|
|||
assert listed["max_output_tokens"] == 16384, listed
|
||||
|
||||
|
||||
def test_v1_models_keeps_a_sibling_override_off_a_plain_deployment_of_the_same_model(gateway: Gateway) -> None:
|
||||
catalog_info: Final = get_model_info("openai/gpt-4o-mini")
|
||||
with gateway.scenario() as scenario:
|
||||
overridden: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini",
|
||||
model_info={"max_input_tokens": 4321, "max_output_tokens": 987},
|
||||
)
|
||||
plain: Final = scenario.model(model="openai/gpt-4o-mini")
|
||||
listed_override: Final = _listed_model(gateway, overridden)
|
||||
listed_plain: Final = _listed_model(gateway, plain)
|
||||
|
||||
assert _limits(listed_plain) == (catalog_info["max_input_tokens"], catalog_info["max_output_tokens"])
|
||||
assert _limits(listed_override) == (4321, 987)
|
||||
|
||||
|
||||
def test_v1_models_carries_deployment_model_info_limits_for_an_unknown_model(gateway: Gateway) -> None:
|
||||
unknown: Final = f"openai/custom-{uuid.uuid4().hex}"
|
||||
with gateway.scenario() as scenario:
|
||||
|
|
|
|||
|
|
@ -419,7 +419,7 @@ class _LoginDeployment:
|
|||
_LOGIN_DEPLOYMENTS: Final = (
|
||||
_LoginDeployment("github_copilot", f"github_copilot/{_CODEX}", None),
|
||||
_LoginDeployment("chatgpt", f"chatgpt/{_CODEX}", None),
|
||||
_LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3", "synthetic-azure-key"),
|
||||
_LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3.2", "synthetic-azure-key"),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.tls import server_context, write_self_signed_cert
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
_LOCATION: Final = "us-central1"
|
||||
_HOST: Final = f"{_LOCATION}-aiplatform.googleapis.com"
|
||||
_PROJECT: Final = "audit-vertex-pass-through"
|
||||
_STREAMING_MODEL: Final = "gemini-3.8-flash"
|
||||
_UNAVAILABLE_MODEL: Final = "gemini-3.8-flash-unavailable"
|
||||
_VENDOR_TOKEN: Final = "Bearer synthetic-vertex-token"
|
||||
_CHUNKS: Final = (
|
||||
b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "Hel"}]}}]}\r\n\r\n',
|
||||
b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "lo"}]}, "finishReason": "STOP"}],'
|
||||
b' "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}}\r\n\r\n',
|
||||
)
|
||||
_UNAVAILABLE_MESSAGE: Final = "The service is currently unavailable."
|
||||
_UNAVAILABLE_BODY: Final = json.dumps(
|
||||
{"error": {"code": 503, "message": _UNAVAILABLE_MESSAGE, "status": "UNAVAILABLE"}}
|
||||
).encode()
|
||||
_SCRUBBED: Final = frozenset(
|
||||
{
|
||||
"HTTPS_PROXY",
|
||||
"HTTP_PROXY",
|
||||
"ALL_PROXY",
|
||||
"NO_PROXY",
|
||||
"SSL_CERT_FILE",
|
||||
"DEFAULT_VERTEXAI_PROJECT",
|
||||
"DEFAULT_VERTEXAI_LOCATION",
|
||||
"DEFAULT_GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Vertex:
|
||||
wire: Wire
|
||||
tunnel: ForwardProxy
|
||||
proxy: Gateway
|
||||
|
||||
def stream(self, model: str, marker: str) -> httpx.Response:
|
||||
return self.proxy.request(
|
||||
"POST",
|
||||
f"/vertex_ai/v1/{_model_path(model)}:streamGenerateContent",
|
||||
{"contents": [{"role": "user", "parts": [{"text": marker}]}]},
|
||||
headers={"Authorization": _VENDOR_TOKEN, "x-litellm-api-key": f"Bearer {self.proxy.key}"},
|
||||
)
|
||||
|
||||
|
||||
def _model_path(model: str) -> str:
|
||||
return f"projects/{_PROJECT}/locations/{_LOCATION}/publishers/google/models/{model}"
|
||||
|
||||
|
||||
def _respond(request: Request) -> Reply:
|
||||
if request.target.startswith(f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:"):
|
||||
return Reply(status=503, body=_UNAVAILABLE_BODY)
|
||||
if request.target.startswith(f"/v1/{_model_path(_STREAMING_MODEL)}:"):
|
||||
return Reply(content_type="text/event-stream", chunks=_CHUNKS)
|
||||
return Reply(status=404, body=b'{"error": "no scripted Vertex model"}')
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vertex(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Vertex]:
|
||||
directory: Final = tmp_path_factory.mktemp("vertex-pass-through-stream")
|
||||
cert, key = write_self_signed_cert(directory, (_HOST,))
|
||||
with (
|
||||
wire_server(_respond, tls=server_context(cert, key)) as wire,
|
||||
tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel,
|
||||
gateway_from_environment() as rig,
|
||||
owned_proxy(
|
||||
rig,
|
||||
directory,
|
||||
{"HTTPS_PROXY": tunnel.url, "NO_PROXY": "127.0.0.1,localhost", "SSL_CERT_FILE": str(cert)},
|
||||
remove_environment=tuple(name for name in os.environ if name.upper() in _SCRUBBED),
|
||||
workers=2,
|
||||
) as proxy,
|
||||
):
|
||||
yield _Vertex(wire, tunnel, proxy)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh_edges(vertex: _Vertex) -> None:
|
||||
vertex.wire.drain()
|
||||
vertex.tunnel.drain()
|
||||
|
||||
|
||||
def test_vertex_pass_through_stream_relays_every_sse_chunk_byte_for_byte(vertex: _Vertex) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = vertex.stream(_STREAMING_MODEL, marker)
|
||||
sent: Final = vertex.wire.drain()
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.headers["content-type"].startswith("text/event-stream"), response.headers
|
||||
assert response.content == b"".join(_CHUNKS), response.text
|
||||
assert [request.target for request in sent] == [
|
||||
f"/v1/{_model_path(_STREAMING_MODEL)}:streamGenerateContent?alt=sse"
|
||||
], sent
|
||||
assert sent[0].headers["authorization"] == _VENDOR_TOKEN
|
||||
assert json.loads(sent[0].body) == {"contents": [{"role": "user", "parts": [{"text": marker}]}]}
|
||||
assert set(vertex.tunnel.targets()) == {f"{_HOST}:443"}
|
||||
|
||||
|
||||
def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Vertex) -> None:
|
||||
response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex)
|
||||
|
||||
assert response.status_code == 503, response.text
|
||||
assert response.content == _UNAVAILABLE_BODY, response.text
|
||||
assert [request.target for request in vertex.wire.drain()] == [
|
||||
f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse"
|
||||
]
|
||||
|
|
@ -0,0 +1,247 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.anthropic_sse import (
|
||||
LIFECYCLE,
|
||||
error_body,
|
||||
event_types,
|
||||
message_id,
|
||||
message_stream,
|
||||
parse_sse,
|
||||
status_reply,
|
||||
stream_reply,
|
||||
user_prompt,
|
||||
)
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
JsonValue,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
_OVERLOADED: Final = "claude-overloaded"
|
||||
_HEALTHY: Final = "claude-healthy"
|
||||
_COUNTED: Final = "claude-counted"
|
||||
_COUNTED_OVERLOADED: Final = "claude-counted-overloaded"
|
||||
_COUNTED_RATE_LIMITED: Final = "claude-counted-rate-limited"
|
||||
_CHAT_ONLY_BACKEND: Final = "audit-chat-only"
|
||||
_API_KEY: Final = "synthetic-anthropic-key"
|
||||
_TEXT: Final = "Hello"
|
||||
_COOLDOWN_GROUP: Final = "audit-overloaded-cooldown"
|
||||
_COUNT_DISABLED_GROUP: Final = "audit-count-tokens-disabled"
|
||||
_COUNT_UPSTREAM_GROUP: Final = "audit-count-tokens-upstream-503"
|
||||
_COUNT_OVERLOADED_GROUP: Final = "audit-count-tokens-upstream-529"
|
||||
_COUNT_RATE_LIMITED_GROUP: Final = "audit-count-tokens-upstream-429"
|
||||
_GATEWAY: Final = "/claude_code_gateway"
|
||||
_GATEWAY_GROUPS: Final = {
|
||||
"non-stream": "audit-gateway-overloaded-non-stream",
|
||||
"stream": "audit-gateway-overloaded-stream",
|
||||
}
|
||||
_COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429}
|
||||
_COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens"
|
||||
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
||||
|
||||
def _served_id(marker: str) -> str:
|
||||
return f"msg_{_HEALTHY}_{marker}"
|
||||
|
||||
|
||||
def _respond(request: Request) -> Reply:
|
||||
body: Final = object_value(json.loads(request.body))
|
||||
backend: Final = str(body["model"])
|
||||
if unquote(request.target) == _COUNT_TOKENS_TARGET:
|
||||
status: Final = _COUNT_STATUSES[backend]
|
||||
return Reply(status=status, body=error_body(status, f"scripted {status}"))
|
||||
if backend == _OVERLOADED:
|
||||
return status_reply(529)
|
||||
if backend == _HEALTHY:
|
||||
return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT))
|
||||
return Reply(status=404, body=b'{"error": "no scripted backend"}')
|
||||
|
||||
|
||||
def _deployment(name: str, model: str, api_base: str, api_key: str, **params: JsonValue) -> dict[str, JsonValue]:
|
||||
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}}
|
||||
|
||||
|
||||
def _config(wire: Wire, directory: Path) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["model_list"] = [
|
||||
_deployment(_COOLDOWN_GROUP, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=1),
|
||||
_deployment(_COOLDOWN_GROUP, f"anthropic/{_HEALTHY}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=2),
|
||||
_deployment(
|
||||
_COUNT_DISABLED_GROUP,
|
||||
f"hosted_vllm/{_CHAT_ONLY_BACKEND}",
|
||||
_UPSTREAM_URL_PLACEHOLDER + "/v1",
|
||||
"synthetic-vllm-key",
|
||||
),
|
||||
_deployment(_COUNT_UPSTREAM_GROUP, f"anthropic/{_COUNTED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY),
|
||||
_deployment(_COUNT_OVERLOADED_GROUP, f"anthropic/{_COUNTED_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY),
|
||||
_deployment(
|
||||
_COUNT_RATE_LIMITED_GROUP, f"anthropic/{_COUNTED_RATE_LIMITED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY
|
||||
),
|
||||
*(
|
||||
_deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY)
|
||||
for group in _GATEWAY_GROUPS.values()
|
||||
),
|
||||
]
|
||||
config["router_settings"] = {
|
||||
"num_retries": 0,
|
||||
"allowed_fails_policy": {"InternalServerErrorAllowedFails": 0},
|
||||
"cooldown_time": 120,
|
||||
"redis_host": os.environ["REDIS_HOST"],
|
||||
"redis_port": int(os.environ["REDIS_PORT"]),
|
||||
}
|
||||
config["litellm_settings"]["disable_token_counter"] = True
|
||||
config["general_settings"]["enable_claude_code_gateway"] = True
|
||||
path: Final = directory / "anthropic-overloaded-cooldown.yaml"
|
||||
path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url))
|
||||
return path
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
proxy: Gateway
|
||||
wire: Wire
|
||||
|
||||
def messages(self, path: str, model: str, marker: str, **extra: JsonValue) -> httpx.Response:
|
||||
return self.proxy.request(
|
||||
"POST",
|
||||
path,
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"stream": True,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
**extra,
|
||||
},
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
)
|
||||
|
||||
def count_tokens(self, model: str, prefix: str = "") -> httpx.Response:
|
||||
return self.proxy.request(
|
||||
"POST",
|
||||
prefix + _COUNT_TOKENS_TARGET,
|
||||
{"model": model, "messages": [{"role": "user", "content": "count me"}]},
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
)
|
||||
|
||||
def targets_called(self) -> tuple[str, ...]:
|
||||
return tuple(unquote(request.target) for request in self.wire.drain())
|
||||
|
||||
def backends_called(self) -> tuple[str, ...]:
|
||||
return tuple(str(object_value(json.loads(request.body))["model"]) for request in self.wire.drain())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
||||
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-cooldown")
|
||||
with gateway_from_environment() as gateway, wire_server(_respond) as wire:
|
||||
overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _API_KEY}
|
||||
with owned_proxy(gateway, directory, overrides, config=_config(wire, directory), workers=2) as proxy:
|
||||
yield _Rig(proxy, wire)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_wire(rig: _Rig) -> None:
|
||||
rig.wire.drain()
|
||||
|
||||
|
||||
def _assert_overloaded_json(response: httpx.Response) -> None:
|
||||
assert response.status_code == 529, response.text
|
||||
payload: Final = object_value(json.loads(response.content))
|
||||
assert object_value(payload["error"])["type"] == "overloaded_error", response.text
|
||||
|
||||
|
||||
def _count_tokens_error(response: httpx.Response, status: int) -> dict[str, JsonValue]:
|
||||
assert response.status_code == status, response.text
|
||||
envelope: Final = object_value(json.loads(response.content))
|
||||
assert envelope["type"] == "error", response.text
|
||||
return object_value(envelope["error"])
|
||||
|
||||
|
||||
def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_allowed_fails_policy(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
first: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True)
|
||||
_assert_overloaded_json(first)
|
||||
assert rig.backends_called() == (_OVERLOADED,)
|
||||
second: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True)
|
||||
assert second.status_code == 200, second.text
|
||||
events: Final = parse_sse(second.text)
|
||||
assert event_types(events) == LIFECYCLE, events
|
||||
assert message_id(events) == _served_id(marker), events
|
||||
assert rig.backends_called() == (_HEALTHY,)
|
||||
|
||||
|
||||
def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP)
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "Token counting is disabled" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == ()
|
||||
|
||||
|
||||
def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP)
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "scripted 503" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
|
||||
@pytest.mark.parametrize(
|
||||
("group", "status", "error_type"),
|
||||
(
|
||||
pytest.param(_COUNT_OVERLOADED_GROUP, 529, "overloaded_error", id="529"),
|
||||
pytest.param(_COUNT_RATE_LIMITED_GROUP, 429, "rate_limit_error", id="429"),
|
||||
),
|
||||
)
|
||||
def test_count_tokens_keeps_other_upstream_statuses_and_their_error_body(
|
||||
rig: _Rig, group: str, status: int, error_type: str
|
||||
) -> None:
|
||||
response: Final = rig.count_tokens(group)
|
||||
error: Final = _count_tokens_error(response, status)
|
||||
assert error["type"] == error_type, response.text
|
||||
assert f"scripted {status}" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
|
||||
|
||||
|
||||
def test_pass_through_messages_stream_forwards_an_upstream_529_verbatim(rig: _Rig) -> None:
|
||||
response: Final = rig.messages("/anthropic/v1/messages", _OVERLOADED, uuid.uuid4().hex)
|
||||
assert response.status_code == 529, response.text
|
||||
assert response.content == error_body(529, "scripted 529"), response.text
|
||||
assert rig.backends_called() == (_OVERLOADED,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", _GATEWAY_GROUPS.keys())
|
||||
def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overloaded_error(
|
||||
rig: _Rig, shape: str
|
||||
) -> None:
|
||||
response: Final = rig.messages(
|
||||
f"{_GATEWAY}/v1/messages",
|
||||
_GATEWAY_GROUPS[shape],
|
||||
uuid.uuid4().hex,
|
||||
stream=shape == "stream",
|
||||
num_retries=0,
|
||||
)
|
||||
_assert_overloaded_json(response)
|
||||
assert rig.backends_called() == (_OVERLOADED,)
|
||||
|
||||
|
||||
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None:
|
||||
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY)
|
||||
error: Final = _count_tokens_error(response, 503)
|
||||
assert error["type"] == "api_error", response.text
|
||||
assert "scripted 503" in string_value(error["message"]), response.text
|
||||
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
|
||||
|
|
@ -45,10 +45,19 @@ _FALLBACK_GROUP: Final = "audit-fallback"
|
|||
_CW_FALLBACK_GROUP: Final = "audit-cw-fallback"
|
||||
_LONELY: Final = "audit-lonely"
|
||||
_POLICY: Final = "audit-policy"
|
||||
_SERVICE_UNAVAILABLE_RETRY_POLICY: Final = "audit-service-unavailable-retry-policy"
|
||||
_INTERNAL_RETRY_POLICY: Final = "audit-internal-retry-policy"
|
||||
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
|
||||
|
||||
Behavior = Literal[
|
||||
"drop-once", "drop-always", "hold-ping", "drop-then-too-long", "drop-then-401", "overloaded-frames-fallback-503"
|
||||
"drop-once",
|
||||
"drop-always",
|
||||
"hold-ping",
|
||||
"drop-then-too-long",
|
||||
"drop-then-401",
|
||||
"overloaded-frames-fallback-529",
|
||||
"overloaded-frames",
|
||||
"overloaded-once",
|
||||
]
|
||||
|
||||
pytestmark = pytest.mark.timeout(240)
|
||||
|
|
@ -81,13 +90,15 @@ def _primary_reply(
|
|||
return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum"))
|
||||
case "drop-then-401":
|
||||
return dropped if attempt == 1 else status_reply(401)
|
||||
case "overloaded-frames-fallback-503":
|
||||
case "overloaded-frames-fallback-529" | "overloaded-frames":
|
||||
return stream_reply((error_frame(529, "scripted overloaded"),))
|
||||
case "overloaded-once":
|
||||
return status_reply(529) if attempt == 1 else stream_reply(full)
|
||||
raise AssertionError(behavior)
|
||||
|
||||
|
||||
def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply:
|
||||
if behavior == "overloaded-frames-fallback-503":
|
||||
if behavior == "overloaded-frames-fallback-529":
|
||||
return status_reply(503)
|
||||
return stream_reply(full)
|
||||
|
||||
|
|
@ -141,13 +152,19 @@ def _config(wire: Wire, directory: Path) -> Path:
|
|||
_deployment(_CW_FALLBACK_GROUP, _CONTEXT_WINDOW),
|
||||
_deployment(_LONELY, _PRIMARY, num_retries=1),
|
||||
_deployment(_POLICY, _PRIMARY),
|
||||
_deployment(_SERVICE_UNAVAILABLE_RETRY_POLICY, _PRIMARY),
|
||||
_deployment(_INTERNAL_RETRY_POLICY, _PRIMARY),
|
||||
]
|
||||
config["router_settings"] = {
|
||||
"num_retries": 1,
|
||||
"disable_cooldowns": True,
|
||||
"fallbacks": [{_WITH_FALLBACKS: [_FALLBACK_GROUP]}],
|
||||
"context_window_fallbacks": [{_WITH_FALLBACKS: [_CW_FALLBACK_GROUP]}],
|
||||
"model_group_retry_policy": {_POLICY: {"DefaultRetries": 2}},
|
||||
"model_group_retry_policy": {
|
||||
_POLICY: {"DefaultRetries": 2},
|
||||
_SERVICE_UNAVAILABLE_RETRY_POLICY: {"ServiceUnavailableErrorRetries": 1, "InternalServerErrorRetries": 0},
|
||||
_INTERNAL_RETRY_POLICY: {"InternalServerErrorRetries": 1, "ServiceUnavailableErrorRetries": 0},
|
||||
},
|
||||
}
|
||||
path: Final = directory / "messages-retry-budget-sources.yaml"
|
||||
path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url))
|
||||
|
|
@ -290,10 +307,34 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha
|
|||
assert rig.attempts_on(_PRIMARY, marker) == 2
|
||||
|
||||
|
||||
def test_overloaded_frames_whose_fallback_fails_answer_the_mapped_internal_server_error(rig: _Rig) -> None:
|
||||
def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker)
|
||||
assert response.status_code == 500, response.text
|
||||
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker)
|
||||
assert response.status_code == 529, response.text
|
||||
assert "overloaded_error" in response.text, response.text
|
||||
assert "content_block_delta" not in response.text, response.text
|
||||
assert rig.attempts_on(_PRIMARY, marker) == 2
|
||||
assert rig.attempts_on(_FALLBACK, marker) == 2
|
||||
|
||||
|
||||
def test_overloaded_frames_that_spend_the_budget_without_a_fallback_relay_the_last_attempts_frame(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = rig.stream(_LONELY, "overloaded-frames", marker)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.text == error_frame(529, "scripted overloaded").decode(), response.text
|
||||
assert rig.attempts_on(_PRIMARY, marker) == 2
|
||||
|
||||
|
||||
def test_a_529_is_retried_under_the_internal_server_error_retry_policy(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
_assert_completed(rig.stream(_INTERNAL_RETRY_POLICY, "overloaded-once", marker), _served_id(_PRIMARY, marker, 2))
|
||||
assert rig.attempts_on(_PRIMARY, marker) == 2
|
||||
|
||||
|
||||
def test_a_529_is_not_retried_under_the_service_unavailable_error_retry_policy(rig: _Rig) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
response: Final = rig.stream(_SERVICE_UNAVAILABLE_RETRY_POLICY, "overloaded-once", marker)
|
||||
assert response.status_code == 529, response.text
|
||||
assert "overloaded_error" in response.text, response.text
|
||||
assert "content_block_delta" not in response.text, response.text
|
||||
assert rig.attempts_on(_PRIMARY, marker) == 1
|
||||
|
|
|
|||
|
|
@ -848,60 +848,36 @@ def test_wrong_azure_key_surfaces_azures_401(rig: _Rig, scenario: Scenario) -> N
|
|||
assert len(_upstream(rig, deployment).model_calls) == 1
|
||||
|
||||
|
||||
def _tool_call(call: Mapping[str, JsonValue]) -> tuple[str, Mapping[str, JsonValue]]:
|
||||
function: Final = object_value(call["function"])
|
||||
return string_value(function["name"]), iv.JSON_OBJECT.validate_json(string_value(function["arguments"]))
|
||||
def _assert_failed_once(call_id: str, *, spend: float) -> None:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows('SELECT status, spend FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)),
|
||||
lambda found: len(found) >= 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert [row["status"] for row in rows] == ["failure"], rows
|
||||
assert float(rows[0]["spend"]) == pytest.approx(spend), rows
|
||||
|
||||
|
||||
def _stream_choices(stream: _Stream) -> Iterator[Mapping[str, JsonValue]]:
|
||||
for frame in stream.frames:
|
||||
yield from rv.ITEMS.validate_python(frame.get("choices") or [])
|
||||
|
||||
|
||||
def _streamed_tool_calls(stream: _Stream) -> Iterator[tuple[str, Mapping[str, JsonValue]]]:
|
||||
for choice in _stream_choices(stream):
|
||||
yield from map(
|
||||
_tool_call, rv.ITEMS.validate_python(object_value(choice.get("delta") or {}).get("tool_calls") or [])
|
||||
)
|
||||
|
||||
|
||||
def _handed_back(deployment: _Deployment) -> list[tuple[str, Mapping[str, JsonValue]]]:
|
||||
return [(iv.SEARCH_TOOL, {"query": iv.query(deployment.name)})]
|
||||
|
||||
|
||||
def _assert_succeeded_once(model: str, identity: str) -> None:
|
||||
rows: Final = _spend_rows(model, 1)
|
||||
assert [string_value(row["request_id"]) for row in rows if row["status"] == "success"] == [identity], rows
|
||||
|
||||
|
||||
def test_failed_follow_up_call_hands_the_search_tool_call_back_with_its_usage(rig: _Rig, scenario: Scenario) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "followup-fails")
|
||||
stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)}))
|
||||
assert stream.status == 200 and stream.content == "", stream.text
|
||||
assert list(_streamed_tool_calls(stream)) == _handed_back(deployment), stream.text
|
||||
finished: Final = [choice["finish_reason"] for choice in _stream_choices(stream) if choice.get("finish_reason")]
|
||||
assert finished == ["tool_calls"], stream.text
|
||||
assert [_totals(usage) for usage in stream.usages] == [iv.FIRST_USAGE["total_tokens"]], stream.text
|
||||
(identity,) = stream.ids
|
||||
@pytest.mark.parametrize("stream", [True, False], ids=["stream", "json"])
|
||||
def test_failed_follow_up_call_answers_with_the_providers_error_instead_of_the_search_tool_call(
|
||||
rig: _Rig, scenario: Scenario, stream: bool
|
||||
) -> None:
|
||||
deployment: Final = _deploy(
|
||||
rig, scenario, "followup-fails", priced=True
|
||||
)
|
||||
extra: Final = {**_NO_CACHE, "stream_options": dict(_USAGE)} if stream else _NO_CACHE
|
||||
body: Final = _chat_body(deployment, stream=stream, extra=extra)
|
||||
response: Final = rig.gateway.request("POST", "/v1/chat/completions", body)
|
||||
assert response.status_code == 500 and iv.FOLLOWUP_FAILURE in response.text, response.text
|
||||
assert iv.SEARCH_TOOL not in response.text, response.text
|
||||
_assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1)
|
||||
_assert_succeeded_once(deployment.model, identity)
|
||||
_assert_failed_once(
|
||||
response.headers["x-litellm-call-id"],
|
||||
spend=iv.FIRST_USAGE["prompt_tokens"] * 0.001 + iv.FIRST_USAGE["completion_tokens"] * 0.002,
|
||||
)
|
||||
assert rig.gateway.chat(_deploy(rig, scenario, "plain").model)["object"] == "chat.completion"
|
||||
|
||||
|
||||
def test_failed_follow_up_call_on_a_json_request_hands_the_search_tool_call_back(rig: _Rig, scenario: Scenario) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "followup-fails")
|
||||
response: Final = rig.gateway.request("POST", "/v1/chat/completions", _chat_body(deployment, stream=False))
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = iv.JSON_OBJECT.validate_json(response.content)
|
||||
(choice,) = rv.ITEMS.validate_python(body["choices"])
|
||||
calls: Final = rv.ITEMS.validate_python(object_value(choice["message"])["tool_calls"])
|
||||
assert [_tool_call(call) for call in calls] == _handed_back(deployment), body
|
||||
assert choice["finish_reason"] == "tool_calls", body
|
||||
assert _totals(object_value(body["usage"])) == iv.FIRST_USAGE["total_tokens"], body
|
||||
_assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1)
|
||||
_assert_succeeded_once(deployment.model, string_value(body["id"]))
|
||||
|
||||
|
||||
def test_search_outage_still_restreams_the_follow_up_answer(rig: _Rig, scenario: Scenario) -> None:
|
||||
deployment: Final = _deploy(rig, scenario, "search-down")
|
||||
stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)}))
|
||||
|
|
|
|||
|
|
@ -859,7 +859,7 @@ async def test_batch_logging_azure_credentials_regression():
|
|||
"timeout": 600,
|
||||
}
|
||||
|
||||
batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n'
|
||||
batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4.1", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n'
|
||||
|
||||
print("\n1. Testing credential extraction...")
|
||||
|
||||
|
|
|
|||
|
|
@ -5,16 +5,24 @@ All sandbox dependencies are injected: a FakeSandbox stands in for the real e2b
|
|||
config and records how it is called.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.integrations.code_interpreter_interception.handler import (
|
||||
_INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY,
|
||||
)
|
||||
from litellm.integrations.code_interpreter_interception.handler import (
|
||||
_SANDBOX_KEY,
|
||||
_SANDBOX_LOST_RESULT_OUTPUT,
|
||||
_SANDBOX_UNAVAILABLE_OUTPUT,
|
||||
_SESSION_SCOPED_KEY,
|
||||
LITELLM_CODE_EXECUTION_TOOL_NAME,
|
||||
CodeInterpreterInterceptionLogger,
|
||||
|
|
@ -23,6 +31,7 @@ from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult
|
|||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
CODE_INTERPRETER_STREAM_OPTIONS_KEY,
|
||||
AgenticLoopPlan,
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
is_interception_internal_key,
|
||||
)
|
||||
|
|
@ -831,11 +840,7 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_plan_deletes_sandbox_when_execution_raises():
|
||||
"""If sandbox execution raises before a plan is built (e.g. E2B aborts
|
||||
output over its cap), the cached sandbox must be deleted before re-raising,
|
||||
otherwise a caller can leak paid containers until the prune TTL."""
|
||||
|
||||
async def test_a_failed_code_run_still_deletes_its_sandbox_once_the_loop_ends():
|
||||
class RaisingSandbox(FakeSandbox):
|
||||
async def arun_code(self, *, container, code, **kwargs):
|
||||
raise ValueError("output exceeded cap")
|
||||
|
|
@ -843,13 +848,11 @@ async def test_build_plan_deletes_sandbox_when_execution_raises():
|
|||
sandbox = RaisingSandbox()
|
||||
logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
|
||||
|
||||
with pytest.raises(ValueError, match="exceeded cap"):
|
||||
await _build_plan(logger, sandbox, call_id="k1")
|
||||
plan = await _build_plan(logger, sandbox, call_id="k1")
|
||||
await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={})
|
||||
|
||||
assert len(sandbox.create_calls) == 1, "the sandbox must have been created"
|
||||
assert len(sandbox.delete_calls) == 1, (
|
||||
"a build failure must delete the cached sandbox so it does not keep running and billing"
|
||||
)
|
||||
assert len(sandbox.create_calls) == 1
|
||||
assert len(sandbox.delete_calls) == 1
|
||||
assert "sbxkey1" not in logger._container_cache
|
||||
|
||||
|
||||
|
|
@ -1448,3 +1451,215 @@ async def test_per_identity_cap_evicts_lru_session():
|
|||
"the entry with the oldest last_accessed timestamp must be evicted first (LRU)"
|
||||
)
|
||||
assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache"
|
||||
|
||||
|
||||
class _SandboxThatCannotStart(FakeSandbox):
|
||||
async def acreate_sandbox(self, **kwargs):
|
||||
raise ConnectionError("sandbox provider unreachable")
|
||||
|
||||
|
||||
class _SandboxThatCannotRunCode(FakeSandbox):
|
||||
async def arun_code(self, *, container, code, **kwargs):
|
||||
raise TimeoutError("sandbox run timed out")
|
||||
|
||||
|
||||
_SANDBOX_FAILURES: Final = (
|
||||
pytest.param(None, _SANDBOX_UNAVAILABLE_OUTPUT, id="no_sandbox"),
|
||||
pytest.param(_SandboxThatCannotStart(), _SANDBOX_UNAVAILABLE_OUTPUT, id="create_fails"),
|
||||
pytest.param(_SandboxThatCannotRunCode(), _SANDBOX_LOST_RESULT_OUTPUT, id="run_fails"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES)
|
||||
async def test_a_sandbox_failure_answers_the_responses_tool_call_with_a_fixed_failure(sandbox, expected_output):
|
||||
plan: Final = await _build_plan(CodeInterpreterInterceptionLogger(sandbox_config=sandbox), sandbox)
|
||||
|
||||
outputs: Final = [message for message in _iter_messages(plan) if message.get("type") == "function_call_output"]
|
||||
assert plan.run_agentic_loop is True
|
||||
assert [(output["call_id"], output["output"]) for output in outputs] == [("c1", expected_output)]
|
||||
assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES)
|
||||
async def test_a_sandbox_failure_answers_the_chat_tool_call_with_a_fixed_failure(sandbox, expected_output):
|
||||
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
|
||||
|
||||
plan: Final = await logger.async_build_agentic_loop_plan(
|
||||
tools={
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "name": LITELLM_CODE_EXECUTION_TOOL_NAME, "arguments": '{"code":"print(40 + 2)"}'}
|
||||
]
|
||||
},
|
||||
model="gpt-5",
|
||||
messages=[{"role": "user", "content": "x"}],
|
||||
response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]},
|
||||
anthropic_messages_provider_config=None,
|
||||
anthropic_messages_optional_request_params={"tools": []},
|
||||
logging_obj=FakeLogging(litellm_call_id="k1"),
|
||||
stream=False,
|
||||
kwargs={
|
||||
"litellm_call_id": "k1",
|
||||
_ACTIVE_KEY: True,
|
||||
_SANDBOX_KEY: "sbxkey1",
|
||||
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
},
|
||||
)
|
||||
|
||||
tool_messages: Final = [message for message in _iter_messages(plan) if message.get("role") == "tool"]
|
||||
assert plan.run_agentic_loop is True
|
||||
assert [(message["tool_call_id"], message["content"]) for message in tool_messages] == [
|
||||
("call_1", expected_output)
|
||||
]
|
||||
assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"]
|
||||
|
||||
|
||||
_SESSION_PLAN_SURFACES: Final = (
|
||||
pytest.param({}, FakeResponse(output=[_function_call_item()]), id="responses"),
|
||||
pytest.param(
|
||||
{_ACTIVE_KEY: True, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE},
|
||||
{"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]},
|
||||
id="chat",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _build_session_plan(
|
||||
logger: CodeInterpreterInterceptionLogger,
|
||||
request_id: str,
|
||||
surface_kwargs: Mapping[str, object] | None = None,
|
||||
response: object | None = None,
|
||||
) -> AgenticLoopPlan:
|
||||
return await logger.async_build_agentic_loop_plan(
|
||||
tools={
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"call_id": "c1",
|
||||
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
|
||||
"arguments": '{"code":"print(1)"}',
|
||||
}
|
||||
]
|
||||
},
|
||||
model="gpt-5",
|
||||
messages=[{"role": "user", "content": "x"}],
|
||||
response=response or FakeResponse(output=[_function_call_item()]),
|
||||
anthropic_messages_provider_config=None,
|
||||
anthropic_messages_optional_request_params={"tools": []},
|
||||
logging_obj=FakeLogging(litellm_call_id=request_id),
|
||||
stream=False,
|
||||
kwargs={
|
||||
"litellm_call_id": request_id,
|
||||
_SANDBOX_KEY: "conv-1",
|
||||
_SESSION_SCOPED_KEY: True,
|
||||
**(surface_kwargs or {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES)
|
||||
async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request(
|
||||
surface_kwargs: Mapping[str, object], response: object
|
||||
) -> None:
|
||||
sandbox: Final = _SandboxThatCannotRunCode()
|
||||
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
|
||||
|
||||
await _build_session_plan(logger, "req1", surface_kwargs, response)
|
||||
await _build_session_plan(logger, "req2", surface_kwargs, response)
|
||||
|
||||
assert len(sandbox.create_calls) == 2
|
||||
assert len(sandbox.delete_calls) == 2
|
||||
|
||||
|
||||
class _SandboxWhoseFirstContainerDies(FakeSandbox):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.handles: list[FakeHandle] = []
|
||||
self.first_run_started = asyncio.Event()
|
||||
self.first_run_may_finish = asyncio.Event()
|
||||
|
||||
async def acreate_sandbox(self, **kwargs: object) -> FakeHandle:
|
||||
self.create_calls.append(kwargs)
|
||||
handle: Final = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}")
|
||||
self.handles.append(handle)
|
||||
return handle
|
||||
|
||||
async def arun_code(self, *, container: FakeHandle, code: str, **kwargs: object) -> CodeExecutionResult:
|
||||
self.run_calls.append({"container": container, "code": code})
|
||||
if len(self.run_calls) == 1:
|
||||
self.first_run_started.set()
|
||||
await self.first_run_may_finish.wait()
|
||||
if container is self.handles[0]:
|
||||
raise TimeoutError("sandbox expired")
|
||||
return CodeExecutionResult(stdout=self.stdout)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement() -> None:
|
||||
sandbox: Final = _SandboxWhoseFirstContainerDies()
|
||||
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
|
||||
|
||||
slow_request: Final = asyncio.create_task(_build_session_plan(logger, "req-slow"))
|
||||
await sandbox.first_run_started.wait()
|
||||
await _build_session_plan(logger, "req-fails-fast")
|
||||
await _build_session_plan(logger, "req-gets-replacement")
|
||||
sandbox.first_run_may_finish.set()
|
||||
await slow_request
|
||||
await _build_session_plan(logger, "req-after")
|
||||
|
||||
assert [call["container"] for call in sandbox.delete_calls] == [sandbox.handles[0]]
|
||||
assert [call["container"] for call in sandbox.run_calls[-2:]] == [sandbox.handles[1], sandbox.handles[1]]
|
||||
|
||||
|
||||
def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]:
|
||||
return {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": content,
|
||||
"stop_reason": stop_reason,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_request_bridged_to_claude_runs_the_code_and_ends_without_the_internal_call(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
sandbox: Final = FakeSandbox(stdout="42")
|
||||
monkeypatch.setattr(litellm, "callbacks", [CodeInterpreterInterceptionLogger(sandbox_config=sandbox)])
|
||||
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
|
||||
side_effect=[
|
||||
httpx.Response(
|
||||
200,
|
||||
json=_anthropic_message(
|
||||
[
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_1",
|
||||
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
|
||||
"input": {"code": "print(40 + 2)"},
|
||||
}
|
||||
],
|
||||
"tool_use",
|
||||
),
|
||||
),
|
||||
httpx.Response(200, json=_anthropic_message([{"type": "text", "text": "The answer is 42."}], "end_turn")),
|
||||
]
|
||||
)
|
||||
|
||||
response: Final = await litellm.aresponses(
|
||||
model="anthropic/claude-sonnet-5-5",
|
||||
input="compute 40 + 2",
|
||||
api_key="sk-test",
|
||||
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
|
||||
)
|
||||
|
||||
assert provider.call_count == 2
|
||||
assert [run["code"] for run in sandbox.run_calls] == ["print(40 + 2)"]
|
||||
assert not [item for item in response.output if getattr(item, "type", None) == "function_call"]
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ config.yaml through to the settings the loop actually reads.
|
|||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -24,6 +25,8 @@ from litellm.integrations.websearch_interception.handler import (
|
|||
WebSearchInterceptionLogger,
|
||||
)
|
||||
from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
|
|
@ -31,6 +34,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
|||
from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.secret_managers.main import get_secret
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
|
|
@ -38,6 +42,7 @@ from litellm.types.integrations.custom_logger import (
|
|||
)
|
||||
|
||||
INTERNAL_TOOL_NAME = "litellm_web_search"
|
||||
CODE_EXECUTION_TOOL_NAME = "litellm_code_execution"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -99,6 +104,29 @@ def _tool_use_names(response: dict) -> list[str]:
|
|||
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
|
||||
|
||||
|
||||
def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=0,
|
||||
status="completed",
|
||||
model="claude-sonnet-4-5",
|
||||
object="response",
|
||||
output=[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}],
|
||||
},
|
||||
*(
|
||||
{"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"}
|
||||
for call_id, name in function_calls
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class _InterceptingCallback(CustomLogger):
|
||||
"""
|
||||
Stands in for the websearch interceptor: asks for another loop whenever the
|
||||
|
|
@ -152,14 +180,46 @@ class _InterceptingCallback(CustomLogger):
|
|||
return response
|
||||
|
||||
|
||||
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
|
||||
class _ResponsesInterceptingCallback(_InterceptingCallback):
|
||||
async def async_should_run_agentic_loop(
|
||||
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
|
||||
):
|
||||
calls = [
|
||||
{"id": item.call_id, "call_id": item.call_id, "name": item.name}
|
||||
for item in response.output
|
||||
if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME
|
||||
]
|
||||
return bool(calls), {"tool_calls": calls}
|
||||
|
||||
|
||||
def _logging_obj(*callbacks: CustomLogger, converted_stream: bool = False) -> MagicMock:
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
|
||||
logging_obj.dynamic_success_callbacks = [callback]
|
||||
logging_obj.dynamic_success_callbacks = list(callbacks)
|
||||
logging_obj.litellm_call_id = "call-abc"
|
||||
return logging_obj
|
||||
|
||||
|
||||
class _CodeExecutionGate(CustomLogger):
|
||||
async def async_should_run_agentic_loop(
|
||||
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
|
||||
):
|
||||
calls = (
|
||||
[
|
||||
{"id": block["id"], "name": block["name"]}
|
||||
for block in response["content"]
|
||||
if block.get("type") == "tool_use" and block.get("name") == CODE_EXECUTION_TOOL_NAME
|
||||
]
|
||||
if isinstance(response, dict)
|
||||
else [
|
||||
{"call_id": item.call_id, "name": item.name}
|
||||
for item in response.output
|
||||
if item.type == "function_call" and item.name == CODE_EXECUTION_TOOL_NAME
|
||||
]
|
||||
)
|
||||
return bool(calls), {"tool_calls": calls}
|
||||
|
||||
|
||||
async def _run_hooks(
|
||||
handler: BaseLLMHTTPHandler,
|
||||
callback: CustomLogger,
|
||||
|
|
@ -314,6 +374,48 @@ class TestCappedLoopReturnsTerminalResponse:
|
|||
assert _tool_use_names(result) == ["get_weather"]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_hooks_internal_call_is_dropped_and_the_clients_survives(self):
|
||||
response = _response_asking_for_another_search()
|
||||
response["content"] += [
|
||||
{"id": "toolu_code_1", "type": "tool_use", "name": CODE_EXECUTION_TOOL_NAME, "input": {}},
|
||||
{"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}},
|
||||
]
|
||||
|
||||
result = await self.handler._call_agentic_completion_hooks(
|
||||
response=response,
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=_logging_obj(self.callback, _CodeExecutionGate()),
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
)
|
||||
|
||||
assert _tool_use_names(result) == ["get_weather"]
|
||||
assert self.callback.plan_calls == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_surface_drops_every_hooks_internal_call_and_keeps_the_clients(self):
|
||||
result = await self.handler._call_agentic_completion_hooks(
|
||||
response=_responses_turn(
|
||||
("call_search", INTERNAL_TOOL_NAME), ("call_code", CODE_EXECUTION_TOOL_NAME), ("call_client", "lookup")
|
||||
),
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=_logging_obj(_ResponsesInterceptingCallback(), _CodeExecutionGate()),
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_client"]
|
||||
|
||||
def test_only_the_refused_tool_calls_are_dropped(self):
|
||||
"""
|
||||
A block is matched on the id the rail refused, not on the tool name, so a
|
||||
|
|
@ -347,6 +449,18 @@ class TestCappedLoopReturnsTerminalResponse:
|
|||
assert _tool_use_names(result) == []
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self):
|
||||
response = _response_asking_for_another_search()
|
||||
response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}}
|
||||
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]},
|
||||
)
|
||||
|
||||
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
|
||||
"""
|
||||
|
|
@ -365,19 +479,34 @@ class TestCappedLoopReturnsTerminalResponse:
|
|||
assert self.callback.plan_calls == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_surface_is_left_to_its_existing_behavior(self):
|
||||
"""
|
||||
The responses surface carries a pydantic model rather than the anthropic
|
||||
dict this finalizer rewrites, so it keeps raising instead of being handed
|
||||
a response that was never actually finalized.
|
||||
"""
|
||||
with pytest.raises(AgenticLoopSafetyError):
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
api_surface="responses",
|
||||
)
|
||||
async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self):
|
||||
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup"))
|
||||
callback = _ResponsesInterceptingCallback()
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=response,
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
assert isinstance(result, ResponsesAPIResponse)
|
||||
assert [(item.type, getattr(item, "name", None)) for item in result.output] == [
|
||||
("message", None),
|
||||
("function_call", "lookup"),
|
||||
]
|
||||
assert callback.plan_calls == 0
|
||||
|
||||
def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self):
|
||||
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME))
|
||||
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]},
|
||||
)
|
||||
|
||||
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_dict_response_is_returned_untouched(self):
|
||||
|
|
@ -509,6 +638,156 @@ def _response_asking_for_searches(*queries: str) -> dict:
|
|||
}
|
||||
|
||||
|
||||
def _request_logging_obj(call_type: str, *callbacks: CustomLogger) -> Logging:
|
||||
logging_obj = Logging(
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
stream=False,
|
||||
call_type=call_type,
|
||||
start_time=time.time(),
|
||||
litellm_call_id="call-abc",
|
||||
function_id="f",
|
||||
dynamic_success_callbacks=list(callbacks),
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="claude-sonnet-4-5", user="", optional_params={}, litellm_params={}, custom_llm_provider="anthropic"
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _assert_first_call_billed(logging_obj: Logging, *, input_tokens: int, output_tokens: int) -> None:
|
||||
price = litellm.model_cost["claude-sonnet-4-5"]
|
||||
assert logging_obj.model_call_details["combined_usage_object"].total_tokens == input_tokens + output_tokens
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
|
||||
input_tokens * price["input_cost_per_token"] + output_tokens * price["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
class TestFailedFollowUpCallReachesTheClient:
|
||||
def setup_method(self):
|
||||
self.handler = BaseLLMHTTPHandler()
|
||||
self.logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"])
|
||||
|
||||
@staticmethod
|
||||
async def _rate_limited_follow_up(**call_kwargs):
|
||||
raise RateLimitError(
|
||||
"UserByModelByMinuteUncachedInputTokens", llm_provider="anthropic", model="claude-sonnet-4-5"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _search(query, kwargs=None, rich=None):
|
||||
found = SearchResult(title="Result", url="https://example.com", snippet="A result.", date=None)
|
||||
return ("Title: Result\nURL: https://example.com", SearchResponse(results=[found]))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("converted_stream", (False, True), ids=("non_streaming", "streaming_client"))
|
||||
async def test_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch, converted_stream):
|
||||
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up)
|
||||
|
||||
with (
|
||||
patch.object(self.logger, "_execute_search", side_effect=self._search),
|
||||
pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"),
|
||||
):
|
||||
await self.handler._call_agentic_completion_hooks(
|
||||
response=_response_asking_for_searches("who won the world cup"),
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
|
||||
logging_obj=_logging_obj(self.logger, converted_stream=converted_stream),
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch):
|
||||
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up)
|
||||
logging_obj = _request_logging_obj("anthropic_messages", self.logger)
|
||||
|
||||
with (
|
||||
patch.object(self.logger, "_execute_search", side_effect=self._search),
|
||||
pytest.raises(RateLimitError),
|
||||
):
|
||||
await self.handler._call_agentic_completion_hooks(
|
||||
response=_response_asking_for_searches("who won the world cup"),
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
|
||||
logging_obj=logging_obj,
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
|
||||
)
|
||||
|
||||
_assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_crashed_plan_build_is_not_retried_and_bills_the_first_call(self):
|
||||
logging_obj = _request_logging_obj("anthropic_messages", self.logger)
|
||||
|
||||
with (
|
||||
patch.object(self.logger, "async_build_agentic_loop_plan", side_effect=ValueError("the plan crashed")),
|
||||
pytest.raises(ValueError, match="the plan crashed") as raised,
|
||||
):
|
||||
await self.handler._call_agentic_completion_hooks(
|
||||
response=_response_asking_for_searches("who won the world cup"),
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
|
||||
logging_obj=logging_obj,
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
|
||||
)
|
||||
|
||||
assert is_agentic_followup_failure(raised.value)
|
||||
_assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up)
|
||||
callback = _ResponsesInterceptingCallback()
|
||||
logging_obj = _request_logging_obj("aresponses", callback)
|
||||
first_call = _responses_turn(("call_search", INTERNAL_TOOL_NAME)).model_copy(
|
||||
update={"usage": {"input_tokens": 13, "output_tokens": 6, "total_tokens": 19}}
|
||||
)
|
||||
|
||||
with pytest.raises(RateLimitError):
|
||||
await self.handler._call_agentic_completion_hooks(
|
||||
response=first_call,
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=logging_obj,
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3},
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
_assert_first_call_billed(logging_obj, input_tokens=13, output_tokens=6)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up)
|
||||
callback = _ResponsesInterceptingCallback()
|
||||
|
||||
with pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"):
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
callback,
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3},
|
||||
response=_responses_turn(("call_search", INTERNAL_TOOL_NAME)),
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
assert callback.plan_calls == 1
|
||||
|
||||
|
||||
class TestFailedSearchEndsTheTurn:
|
||||
"""
|
||||
A search that failed used to come back to the client as an empty successful
|
||||
|
|
|
|||
|
|
@ -371,8 +371,7 @@ class TestLoggingObjExcludedFromFollowUp:
|
|||
|
||||
class TestFollowUpErrorScenarios:
|
||||
"""Regression tests: the agentic loop must surface errors properly and
|
||||
not silently swallow them (except at the _call_agentic_completion_hooks
|
||||
level which intentionally catches to return the initial response)."""
|
||||
not silently swallow them."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_followup_400_raises(self):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import litellm
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
TIER_MODEL: Final = "tier-priced-test-model"
|
||||
TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
||||
|
|
@ -48,6 +48,190 @@ AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_tokens", (60, 100))
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "priority_rate", "image_rate"),
|
||||
((None, 11e-06, 7e-06), ("priority", 11e-06, 11e-06), ("flex", 11e-06, 7e-06), ("priority", 0.0, 0.0)),
|
||||
)
|
||||
def test_completion_cost_uses_priority_image_output_rate(
|
||||
local_model_cost_map: None, service_tier: str | None, priority_rate: float, image_rate: float, image_tokens: int
|
||||
) -> None:
|
||||
model: Final = "gemini-synthetic-image-priority"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_image_token": 7e-06,
|
||||
"output_cost_per_image_token_priority": priority_rate,
|
||||
}
|
||||
}
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model=model,
|
||||
usage=Usage(
|
||||
prompt_tokens=20,
|
||||
completion_tokens=100,
|
||||
total_tokens=120,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=image_tokens),
|
||||
),
|
||||
)
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier
|
||||
)
|
||||
assert cost == pytest.approx(20 * 3e-06 + (100 - image_tokens) * 5e-06 + image_tokens * image_rate)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_tokens", (199_999, 200_000, 200_001))
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "flat_rate", "context_rate"),
|
||||
((None, 7e-06, 11e-06), ("priority", 13e-06, 17e-06), ("flex", 19e-06, 23e-06)),
|
||||
)
|
||||
def test_completion_cost_uses_total_prompt_for_audio_context_tiers(
|
||||
local_model_cost_map: None, prompt_tokens: int, service_tier: str | None, flat_rate: float, context_rate: float
|
||||
) -> None:
|
||||
model: Final = "gemini-synthetic-audio-context"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 5e-06,
|
||||
"input_cost_per_audio_token": 7e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens": 11e-06,
|
||||
"input_cost_per_audio_token_priority": 13e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens_priority": 17e-06,
|
||||
"input_cost_per_audio_token_flex": 19e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens_flex": 23e-06,
|
||||
}
|
||||
}
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model=model,
|
||||
usage=Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=10,
|
||||
total_tokens=prompt_tokens + 10,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
|
||||
),
|
||||
)
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier
|
||||
)
|
||||
audio_rate: Final = context_rate if prompt_tokens > 200_000 else flat_rate
|
||||
assert cost == pytest.approx((prompt_tokens - 100) * 3e-06 + 100 * audio_rate + 10 * 5e-06)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001))
|
||||
def test_batch_cost_uses_audio_context_and_image_output_tiers(local_model_cost_map: None, prompt_tokens: int) -> None:
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
|
||||
model: Final = "gemini-synthetic-batch-audio-context"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 6e-06,
|
||||
"output_cost_per_token": 10e-06,
|
||||
"input_cost_per_token_batches": 3e-06,
|
||||
"output_cost_per_token_batches": 5e-06,
|
||||
"input_cost_per_token_above_200k_tokens_batches": 13e-06,
|
||||
"output_cost_per_token_above_200k_tokens_batches": 17e-06,
|
||||
"output_cost_per_image_token_batches": 19e-06,
|
||||
"input_cost_per_audio_token_batches": 7e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens_batches": 11e-06,
|
||||
}
|
||||
}
|
||||
)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=20,
|
||||
total_tokens=prompt_tokens + 20,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=10),
|
||||
)
|
||||
cost: Final = batch_cost_calculator(model=model, custom_llm_provider="vertex_ai", usage=usage)
|
||||
audio_rate: Final = 11e-06 if prompt_tokens > 200_000 else 7e-06
|
||||
input_rate: Final = 13e-06 if prompt_tokens > 200_000 else 3e-06
|
||||
output_rate: Final = 17e-06 if prompt_tokens > 200_000 else 5e-06
|
||||
assert sum(cost) == pytest.approx(
|
||||
(prompt_tokens - 100) * input_rate + 100 * audio_rate + 10 * output_rate + 10 * 19e-06
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001))
|
||||
def test_completion_cost_uses_flex_context_input_and_output_rates(
|
||||
local_model_cost_map: None, prompt_tokens: int
|
||||
) -> None:
|
||||
model: Final = "gemini-synthetic-flex-context"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 6e-06,
|
||||
"output_cost_per_token": 10e-06,
|
||||
"input_cost_per_token_flex": 3e-06,
|
||||
"output_cost_per_token_flex": 5e-06,
|
||||
"input_cost_per_token_above_200k_tokens_flex": 7e-06,
|
||||
"output_cost_per_token_above_200k_tokens_flex": 11e-06,
|
||||
}
|
||||
}
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model=model, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=10, total_tokens=prompt_tokens + 10)
|
||||
)
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="flex"
|
||||
)
|
||||
input_rate: Final = 7e-06 if prompt_tokens > 200_000 else 3e-06
|
||||
output_rate: Final = 11e-06 if prompt_tokens > 200_000 else 5e-06
|
||||
assert cost == pytest.approx(prompt_tokens * input_rate + 10 * output_rate)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("audio_fields", "audio_rate"),
|
||||
(
|
||||
({}, 3e-06),
|
||||
({"input_cost_per_audio_token": 7e-06}, 7e-06),
|
||||
({"input_cost_per_audio_token_above_200k_tokens": 11e-06}, 11e-06),
|
||||
({"input_cost_per_audio_token_above_200k_tokens_priority": 0.0}, 0.0),
|
||||
),
|
||||
)
|
||||
def test_audio_context_rates_preserve_fallbacks_and_explicit_zero(
|
||||
local_model_cost_map: None, audio_fields: Mapping[str, float], audio_rate: float
|
||||
) -> None:
|
||||
model: Final = "gemini-synthetic-audio-fallback"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 5e-06,
|
||||
**audio_fields,
|
||||
}
|
||||
}
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model=model,
|
||||
usage=Usage(
|
||||
prompt_tokens=200_001,
|
||||
completion_tokens=10,
|
||||
total_tokens=200_011,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
|
||||
),
|
||||
)
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="priority"
|
||||
)
|
||||
assert cost == pytest.approx(199_901 * 3e-06 + 100 * audio_rate + 10 * 5e-06)
|
||||
|
||||
|
||||
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
|
||||
return (
|
||||
(PROMPT_TOKENS - CACHED_TOKENS) * prices[f"input_cost_per_token{column_suffix}"]
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
schema_has_unresolvable_ref,
|
||||
schema_property_count,
|
||||
split_concatenated_json_objects,
|
||||
split_embedding_media_for_tpm,
|
||||
split_media_for_tpm,
|
||||
strip_encrypted_reasoning_from_messages,
|
||||
strip_litellm_response_only_fields,
|
||||
|
|
@ -2709,3 +2710,22 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt
|
|||
|
||||
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"])
|
||||
def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None:
|
||||
import gc
|
||||
import tracemalloc
|
||||
|
||||
uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii")
|
||||
split_embedding_media_for_tpm(f"data:{mime};base64,AAAA")
|
||||
gc.collect()
|
||||
tracemalloc.start()
|
||||
try:
|
||||
_, media = split_embedding_media_for_tpm(uri)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
|
||||
assert media
|
||||
assert peak < 1.5 * len(uri)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ they stay out of the body even without it.
|
|||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Dict, Final, List, Optional, Tuple
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -28,11 +30,13 @@ import pytest
|
|||
from openai._legacy_response import LegacyAPIResponse
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.integrations.code_interpreter_interception.handler import (
|
||||
CodeInterpreterInterceptionLogger,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
|
||||
_execute_chat_completion_agentic_plan,
|
||||
maybe_run_chat_completion_agentic_loop,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
|
@ -43,6 +47,7 @@ from litellm.types.integrations.custom_logger import (
|
|||
WEBSEARCH_INTERCEPTION_PREFIX,
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
|
|
@ -50,6 +55,7 @@ from litellm.types.utils import (
|
|||
Function,
|
||||
Message,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
|
||||
# The internal control fields that must never reach a provider request body.
|
||||
|
|
@ -260,8 +266,6 @@ class _GateOnlyLogger(CustomLogger):
|
|||
|
||||
|
||||
class _RecordingGateLogger(_GateOnlyLogger):
|
||||
"""Also keeps every follow-up response the loop hands to the post hook."""
|
||||
|
||||
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
|
||||
super().__init__(plan=plan, tool_calls=tool_calls)
|
||||
self.followups: tuple[ModelResponse, ...] = ()
|
||||
|
|
@ -441,72 +445,544 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ
|
|||
assert call_kwargs["api_base"] == "https://a"
|
||||
|
||||
|
||||
_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
|
||||
_REFUSING_RAILS: Final = (
|
||||
{"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
{
|
||||
"_agentic_loop_depth": 0,
|
||||
"max_agentic_loops": 3,
|
||||
"_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall:
|
||||
return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}"))
|
||||
|
||||
|
||||
async def _run_refused_rerun(
|
||||
response: ModelResponse,
|
||||
rail_kwargs: Dict[str, object],
|
||||
stream: bool = False,
|
||||
gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS,
|
||||
other_hooks_tool_calls: tuple[Dict[str, object], ...] = (),
|
||||
) -> Any:
|
||||
litellm.callbacks = [
|
||||
_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=tool_calls)
|
||||
for tool_calls in (gate_tool_calls, *other_hooks_tool_calls)
|
||||
]
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
try:
|
||||
return await maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs=dict(rail_kwargs),
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=stream,
|
||||
)
|
||||
finally:
|
||||
acompletion_mock.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops(
|
||||
restore_callbacks,
|
||||
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
|
||||
async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call(
|
||||
restore_callbacks, rail_kwargs: Dict[str, object]
|
||||
):
|
||||
"""depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops,
|
||||
before any follow-up call is attempted."""
|
||||
logger = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(run_agentic_loop=True),
|
||||
result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs)
|
||||
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert not result.choices[0].message.tool_calls
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks):
|
||||
response = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content="Running it.",
|
||||
tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = await _run_refused_rerun(response, _REFUSING_RAILS[0])
|
||||
|
||||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
assert result.choices[0].finish_reason == "tool_calls"
|
||||
assert result.choices[0].message.content == "Running it."
|
||||
|
||||
|
||||
def _indexed_tool_call(call_id: str, name: str, index: int) -> ChatCompletionMessageToolCall:
|
||||
return ChatCompletionMessageToolCall(
|
||||
id=call_id, type="function", function=Function(name=name, arguments="{}"), index=index
|
||||
)
|
||||
|
||||
|
||||
def _turn_with_the_internal_call_between_the_clients() -> ModelResponse:
|
||||
return ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
_indexed_tool_call("call_weather", "get_weather", 0),
|
||||
_indexed_tool_call("call_abc", "litellm_code_execution", 1),
|
||||
_indexed_tool_call("call_time", "get_time", 2),
|
||||
],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
|
||||
async def test_refused_rerun_renumbers_the_kept_tool_calls_from_zero(restore_callbacks, rail_kwargs: Dict[str, object]):
|
||||
result = await _run_refused_rerun(_turn_with_the_internal_call_between_the_clients(), rail_kwargs)
|
||||
|
||||
assert [(call.id, call.index) for call in result.choices[0].message.tool_calls] == [
|
||||
("call_weather", 0),
|
||||
("call_time", 1),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_on_a_converted_stream_replays_the_kept_tool_calls_from_index_zero(
|
||||
restore_callbacks, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"callbacks",
|
||||
[_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=_REFUSED_GATE_TOOL_CALLS)],
|
||||
)
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
litellm_call_id="call-refused-stream",
|
||||
function_id="fn-refused-stream",
|
||||
)
|
||||
|
||||
result: Final = await maybe_run_chat_completion_agentic_loop(
|
||||
response=_turn_with_the_internal_call_between_the_clients(),
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs={**_REFUSING_RAILS[1], f"{CODE_INTERPRETER_INTERCEPTION_PREFIX}_converted_stream": True},
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
)
|
||||
|
||||
assert isinstance(result, CustomStreamWrapper)
|
||||
deltas: Final = [chunk.choices[0].delta async for chunk in result if chunk.choices]
|
||||
streamed_calls: Final = [call for delta in deltas for call in delta.tool_calls or []]
|
||||
assert [(call.id, call.index) for call in streamed_calls] == [("call_weather", 0), ("call_time", 1)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clients(restore_callbacks):
|
||||
response = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
_tool_call("call_abc", "litellm_code_execution"),
|
||||
_tool_call("call_search", "litellm_web_search"),
|
||||
_tool_call("call_client", "lookup"),
|
||||
],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = await _run_refused_rerun(
|
||||
response,
|
||||
_REFUSING_RAILS[0],
|
||||
other_hooks_tool_calls=({"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]},),
|
||||
)
|
||||
|
||||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks):
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
_tool_call("call_abc", "litellm_code_execution"),
|
||||
_tool_call("call_search", "litellm_web_search"),
|
||||
_tool_call("call_client", "lookup"),
|
||||
],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result: Final = await _run_refused_rerun(
|
||||
response,
|
||||
_REFUSING_RAILS[0],
|
||||
gate_tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},),
|
||||
)
|
||||
|
||||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response, gate_tool_calls",
|
||||
[(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)],
|
||||
ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"],
|
||||
)
|
||||
async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged(
|
||||
restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]]
|
||||
):
|
||||
result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls)
|
||||
|
||||
assert result.choices[0].model_dump() == response.choices[0].model_dump()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks):
|
||||
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
|
||||
await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_surfaces_a_failed_follow_up_instead_of_the_tool_call_turn():
|
||||
logger: Final = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(
|
||||
messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"}
|
||||
),
|
||||
),
|
||||
tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await _dispatch_tool_call_turn(_tool_call_model_response(), logger)
|
||||
|
||||
assert logger.cleanup_calls == 1
|
||||
|
||||
|
||||
class _PlanRecordingLogger(_GateOnlyLogger):
|
||||
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
|
||||
super().__init__(plan=plan, tool_calls=tool_calls)
|
||||
self.logging_objs: List[Any] = []
|
||||
|
||||
async def async_build_agentic_loop_plan(
|
||||
self,
|
||||
tools: Dict[str, Any],
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: Any,
|
||||
anthropic_messages_provider_config: Any,
|
||||
anthropic_messages_optional_request_params: Dict[str, Any],
|
||||
logging_obj: Any,
|
||||
stream: bool,
|
||||
kwargs: Dict[str, Any],
|
||||
) -> AgenticLoopPlan:
|
||||
self.logging_objs.append(logging_obj)
|
||||
return self._plan
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(restore_callbacks):
|
||||
logger: Final = _PlanRecordingLogger(
|
||||
plan=AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(
|
||||
messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"}
|
||||
),
|
||||
),
|
||||
tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
)
|
||||
litellm.callbacks = [logger]
|
||||
first_call: Final = ModelResponse(
|
||||
choices=_tool_call_model_response().choices, usage=Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
|
||||
)
|
||||
mock_client: Final = MagicMock()
|
||||
mock_client.post = AsyncMock(return_value=_raw_response_for(first_call))
|
||||
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
with pytest.raises(ValueError, match="max_agentic_loops"):
|
||||
await maybe_run_chat_completion_agentic_loop(
|
||||
response=_tool_call_model_response(),
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
)
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await litellm.acompletion(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
api_key="sk-test",
|
||||
client=mock_client,
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
acompletion_mock.assert_not_awaited()
|
||||
(logging_obj,) = logger.logging_objs
|
||||
price: Final = litellm.model_cost["gpt-4o-mini"]
|
||||
assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
|
||||
5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
async def _agentic_turn_with_follow_up(
|
||||
logger: _PlanRecordingLogger, monkeypatch: pytest.MonkeyPatch
|
||||
) -> tuple[ModelResponse, Logging]:
|
||||
monkeypatch.setattr(litellm, "callbacks", [logger])
|
||||
mock_client: Final = MagicMock()
|
||||
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_model_response()))
|
||||
response: Final = await litellm.acompletion(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
api_key="sk-test",
|
||||
client=mock_client,
|
||||
num_retries=0,
|
||||
)
|
||||
assert isinstance(response, ModelResponse)
|
||||
(logging_obj,) = logger.logging_objs
|
||||
return response, logging_obj
|
||||
|
||||
|
||||
def _follow_up_logger() -> _PlanRecordingLogger:
|
||||
return _PlanRecordingLogger(
|
||||
plan=AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": "42"}),
|
||||
),
|
||||
tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks):
|
||||
"""A tool_calls fingerprint already present in _agentic_loop_fingerprints ->
|
||||
ValueError about the repeated fingerprint (cycle guard), with no follow-up
|
||||
call."""
|
||||
import json
|
||||
async def test_acompletion_records_the_cost_breakdown_of_a_successful_follow_up(monkeypatch):
|
||||
response, logging_obj = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch)
|
||||
|
||||
# The dispatcher fingerprints the whole value the gate returns as its second
|
||||
# tuple element, so the seeded fingerprint must mirror that dict exactly.
|
||||
gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
|
||||
fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str)
|
||||
response_cost: Final = response._hidden_params["response_cost"]
|
||||
assert response_cost > 0
|
||||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["total_cost"] == pytest.approx(response_cost)
|
||||
|
||||
logger = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(run_agentic_loop=True),
|
||||
tool_calls=gate_tool_calls,
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_keeps_a_cached_follow_up_free(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type="local"))
|
||||
await litellm.acompletion(model="openai/gpt-4o-mini", messages=_patched_messages(), mock_response="42")
|
||||
|
||||
response, _ = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch)
|
||||
|
||||
assert response._hidden_params["cache_hit"] is True
|
||||
assert response._hidden_params["response_cost"] == 0
|
||||
|
||||
|
||||
def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
model="gpt-4o-mini",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]),
|
||||
)
|
||||
],
|
||||
usage=usage,
|
||||
)
|
||||
litellm.callbacks = [logger]
|
||||
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
with pytest.raises(ValueError, match="fingerprint"):
|
||||
await maybe_run_chat_completion_agentic_loop(
|
||||
response=_tool_call_model_response(),
|
||||
|
||||
class _TwoRoundLogger(CustomLogger):
|
||||
def __init__(self, middle_round: ModelResponse) -> None:
|
||||
super().__init__()
|
||||
self._follow_ups: Final = (middle_round, "litellm.RateLimitError")
|
||||
self.logging_objs: tuple[object, ...] = ()
|
||||
|
||||
async def async_should_run_agentic_loop(
|
||||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
assert isinstance(response, ModelResponse)
|
||||
calls: Final = response.choices[0].message.tool_calls or []
|
||||
return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]}
|
||||
|
||||
async def async_build_agentic_loop_plan(
|
||||
self,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
anthropic_messages_provider_config: object,
|
||||
anthropic_messages_optional_request_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> AgenticLoopPlan:
|
||||
follow_up: Final = self._follow_ups[len(self.logging_objs)]
|
||||
self.logging_objs = (*self.logging_objs, logging_obj)
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails(
|
||||
restore_callbacks, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
|
||||
middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15)
|
||||
logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage))
|
||||
monkeypatch.setattr(litellm, "callbacks", [logger])
|
||||
mock_client: Final = MagicMock()
|
||||
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage)))
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await litellm.acompletion(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
api_key="sk-test",
|
||||
client=mock_client,
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
first_logging_obj, _ = logger.logging_objs
|
||||
assert isinstance(first_logging_obj, Logging)
|
||||
price: Final = litellm.model_cost["gpt-4o-mini"]
|
||||
billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"]
|
||||
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4)
|
||||
assert first_logging_obj.model_call_details["response_cost"] == pytest.approx(
|
||||
(5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool):
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
|
||||
|
||||
sandbox_error: Final = RuntimeError("sandbox crashed")
|
||||
|
||||
async def failing_follow_up() -> ModelResponse:
|
||||
raise sandbox_error
|
||||
|
||||
with pytest.raises(RuntimeError) as raised:
|
||||
await bill_first_round_on_failure(
|
||||
failing_follow_up(),
|
||||
logging_obj=Logging(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs={
|
||||
"_agentic_loop_depth": 0,
|
||||
"max_agentic_loops": 3,
|
||||
"_agentic_loop_fingerprints": [fingerprint],
|
||||
},
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
)
|
||||
stream=stream,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
litellm_call_id="call-builtin-error",
|
||||
function_id="fn-builtin-error",
|
||||
),
|
||||
response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
acompletion_mock.assert_not_awaited()
|
||||
assert raised.value is sandbox_error
|
||||
|
||||
|
||||
class _RunOnlyLoopLogger(CustomLogger):
|
||||
def __init__(self, result: ModelResponse) -> None:
|
||||
super().__init__()
|
||||
self._result = result
|
||||
|
||||
async def async_should_run_agentic_loop(
|
||||
self,
|
||||
response: Any,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
tools: Optional[List[Dict[str, Any]]],
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Dict[str, Any],
|
||||
) -> Tuple[bool, Dict[str, Any]]:
|
||||
return True, {"tool_calls": [{"id": "call_abc"}]}
|
||||
|
||||
async def async_run_agentic_loop(
|
||||
self,
|
||||
tools: Dict[str, Any],
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: Any,
|
||||
anthropic_messages_provider_config: Any,
|
||||
anthropic_messages_optional_request_params: Dict[str, Any],
|
||||
logging_obj: Any,
|
||||
stream: bool,
|
||||
kwargs: Dict[str, Any],
|
||||
) -> Any:
|
||||
return self._result
|
||||
|
||||
|
||||
class _LoggingWithCallbacks(_LoggingStub):
|
||||
def __init__(self, callbacks: Tuple[CustomLogger, ...]) -> None:
|
||||
self.dynamic_success_callbacks = list(callbacks)
|
||||
|
||||
|
||||
async def _dispatch_tool_call_turn(
|
||||
response: ModelResponse, *callbacks: CustomLogger
|
||||
) -> ModelResponse | CustomStreamWrapper | None:
|
||||
return await maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
optional_params={},
|
||||
kwargs={},
|
||||
logging_obj=_LoggingWithCallbacks(callbacks),
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_returns_the_tool_call_turn_when_the_plan_terminates():
|
||||
response: Final = _tool_call_model_response()
|
||||
terminating: Final = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(terminate=True), tool_calls={"tool_calls": [{"id": "call_abc"}]}
|
||||
)
|
||||
|
||||
assert await _dispatch_tool_call_turn(response, terminating) is response
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_moves_past_a_plan_that_declines_the_loop():
|
||||
override: Final = _plain_model_response("from the next callback")
|
||||
tool_calls: Final = {"tool_calls": [{"id": "call_abc"}]}
|
||||
declined: Final = _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=False), tool_calls=tool_calls)
|
||||
overriding: Final = _GateOnlyLogger(plan=AgenticLoopPlan(response_override=override), tool_calls=tool_calls)
|
||||
|
||||
assert await _dispatch_tool_call_turn(_tool_call_model_response(), declined, overriding) is override
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_returns_a_run_only_loops_result():
|
||||
loop_result: Final = _plain_model_response()
|
||||
|
||||
assert await _dispatch_tool_call_turn(_tool_call_model_response(), _RunOnlyLoopLogger(loop_result)) is loop_result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -562,3 +1038,55 @@ async def test_converted_stream_replays_the_final_answer_with_the_requested_usag
|
|||
if include_usage
|
||||
else []
|
||||
)
|
||||
|
||||
|
||||
class TestAgenticFollowUpKeepsTheProviderPrefix:
|
||||
@staticmethod
|
||||
async def _run_followup(
|
||||
model: str, custom_llm_provider: str, patched_model: str | None = None
|
||||
) -> ModelResponse:
|
||||
response: Final = await _execute_chat_completion_agentic_plan(
|
||||
plan=AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(
|
||||
messages=[{"role": "user", "content": "hi"}], model=patched_model
|
||||
),
|
||||
),
|
||||
callback=CustomLogger(),
|
||||
model=model,
|
||||
optional_params={"mock_response": "ok from followup"},
|
||||
kwargs={},
|
||||
logging_obj=None,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=0,
|
||||
max_loops=2,
|
||||
fingerprints=[],
|
||||
fingerprint="fp",
|
||||
)
|
||||
assert isinstance(response, ModelResponse)
|
||||
return response
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_sub_path_model_still_resolves_a_provider(self) -> None:
|
||||
response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock")
|
||||
|
||||
assert response.choices[0].message.content == "ok from followup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_ordinary_model_still_resolves_a_provider(self) -> None:
|
||||
response: Final = await self._run_followup("gpt-4o-mini", "openai")
|
||||
|
||||
assert response.choices[0].message.content == "ok from followup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None:
|
||||
response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5")
|
||||
|
||||
assert response.model == "claude-sonnet-4-5"
|
||||
assert response._hidden_params["custom_llm_provider"] == "anthropic"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None:
|
||||
response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o")
|
||||
|
||||
assert response._hidden_params["custom_llm_provider"] == "openrouter"
|
||||
|
|
|
|||
|
|
@ -217,6 +217,22 @@ def test_unknown_exception_falls_back_to_status_then_unclassified() -> None:
|
|||
assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED"
|
||||
|
||||
|
||||
def test_a_529_the_anthropic_routes_answered_is_an_overload() -> None:
|
||||
paused = _proxy_exc("Model is paused", "service_unavailable_error", 529)
|
||||
assert StandardLoggingPayloadSetup.get_error_information(paused)["normalized_error"] == "503_PROVIDER_OVERLOADED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ["", "<html>bad gateway</html>", '{"type":"error","error":{"type":"api_error"}}'])
|
||||
def test_an_internal_server_error_answered_as_529_is_an_overload_whatever_its_body_says(body: str) -> None:
|
||||
overloaded = litellm.InternalServerError(body, llm_provider="anthropic", model="claude", provider_status_code=529)
|
||||
overloaded.status_code = 529
|
||||
internal = litellm.InternalServerError(body, llm_provider="anthropic", model="claude")
|
||||
assert (
|
||||
StandardLoggingPayloadSetup.get_error_information(overloaded)["normalized_error"],
|
||||
StandardLoggingPayloadSetup.get_error_information(internal)["normalized_error"],
|
||||
) == ("503_PROVIDER_OVERLOADED", "500_PROVIDER_INTERNAL_ERROR")
|
||||
|
||||
|
||||
def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None:
|
||||
exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key")
|
||||
assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED"
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue