Merge remote-tracking branch 'origin/main' into litellm_azure_create_no_blind_retry

# Conflicts:
#	litellm/router_python_core.py
This commit is contained in:
mateo-berri 2026-11-03 18:21:41 -08:00
commit 4de8bfea56
188 changed files with 14933 additions and 3532 deletions

View file

@ -5,6 +5,7 @@ on:
branches:
- main
- "litellm_**"
- "rc/**"
schedule:
- cron: "23 6 * * *"
workflow_dispatch:

View file

@ -151,6 +151,8 @@ pub enum Uplift {
Residency(Residency),
/// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
NonGlobalEndpoint,
/// OpenAI FedRAMP endpoint (gov.api.openai.com), from `fedramp_uplift_multiplier`
FedrampEndpoint,
/// Anthropic `speed: fast`, from `provider_specific_entry.fast`
Speed(Speed),
/// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main

View file

@ -104,6 +104,7 @@ impl From<&EntryWire> for Adjustments {
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
(Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
(Uplift::FedrampEndpoint, wire.fedramp_uplift),
]
.into_iter()
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
@ -746,7 +747,8 @@ mod tests {
fn uplifts_are_adjustments_not_prices() {
let wire = wire(json!({
"regional_endpoint_uplift_multiplier": 1.1,
"regional_processing_uplift_multiplier_eu": "1.2"
"regional_processing_uplift_multiplier_eu": "1.2",
"fedramp_uplift_multiplier": 1.1
}));
let adjustments = Adjustments::from(&wire);
@ -764,6 +766,12 @@ mod tests {
Some(1.2)
);
assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
assert_eq!(
adjustments
.get(&Uplift::FedrampEndpoint)
.map(Multiplier::get),
Some(1.1)
);
}
#[rstest]

View file

@ -96,6 +96,7 @@ pub struct ModelInfoFields {
pub output_cost_per_token_ultrafast: Option<f64>,
pub regional_processing_uplift_multiplier_eu: Option<f64>,
pub regional_processing_uplift_multiplier_us: Option<f64>,
pub fedramp_uplift_multiplier: Option<f64>,
pub regional_endpoint_uplift_multiplier: Option<f64>,
pub output_cost_per_audio_token: Option<f64>,
pub output_cost_per_character: Option<f64>,

View file

@ -29,6 +29,8 @@ struct PriceBlocks {
regional_processing_uplift_multiplier_eu: Option<Multiplier>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
regional_processing_uplift_multiplier_us: Option<Multiplier>,
/// Multiplier applied to all token costs when served from the OpenAI FedRAMP endpoint (e.g. 1.10 = +10%).
fedramp_uplift_multiplier: Option<Multiplier>,
/// USD cost per web search query, keyed by search context size.
search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.

View file

@ -95,6 +95,8 @@ attributes! {
source: String,
/// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
successor_model: String,
/// URL of the provider page or feed `deprecation_date` was read from.
deprecation_source: String,
/// Audio container formats the model can return.
supported_audio_formats: Vec<AudioFormat>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
@ -307,6 +309,8 @@ mod tests {
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
#[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
#[case::deprecation_source("deprecation_source", json!("https://developers.openai.com/api/docs/deprecations"), true)]
#[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]

View file

@ -24,6 +24,7 @@ pub struct EntryWire {
pub regional_endpoint_uplift: Option<Multiplier>,
pub residency_uplift_eu: Option<Multiplier>,
pub residency_uplift_us: Option<Multiplier>,
pub fedramp_uplift: Option<Multiplier>,
pub unknown: Fields,
pub problems: Vec<Error>,
}
@ -57,6 +58,7 @@ impl EntryWire {
"regional_processing_uplift_multiplier_us" => {
self.residency_uplift_us = reader.value(path(), value)
}
"fedramp_uplift_multiplier" => self.fedramp_uplift = reader.value(path(), value),
_ => self.read_open_key(reader, key, value),
}
}

View file

@ -688,6 +688,7 @@ darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
streamlake_models: Final[Set[str]] = set()
wally_models: Final[Set[str]] = set()
zerogpu_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
@ -984,6 +985,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
kyma_models.add(key)
elif value.get("litellm_provider") == "streamlake":
streamlake_models.add(key)
elif value.get("litellm_provider") == "wally":
wally_models.add(key)
elif value.get("litellm_provider") == "zerogpu":
zerogpu_models.add(key)
elif value.get("litellm_provider") == "v0":
@ -1071,7 +1074,6 @@ _populate_provider_model_sets(model_cost)
# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting
azure_llms = {
"gpt-35-turbo": "azure/gpt-35-turbo",
"gpt-35-turbo-16k": "azure/gpt-35-turbo-16k",
"gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct",
"azure/gpt-41": "gpt-4.1",
"azure/gpt-41-mini": "gpt-4.1-mini",
@ -1166,6 +1168,7 @@ model_list = list(
| openinfer_models
| kyma_models
| streamlake_models
| wally_models
| zerogpu_models
| v0_models
| morph_models
@ -1299,6 +1302,7 @@ def _build_models_by_provider() -> dict:
"openinfer": openinfer_models,
"kyma": kyma_models,
"streamlake": streamlake_models,
"wally": wally_models,
"zerogpu": zerogpu_models,
"v0": v0_models,
"morph": morph_models,

View file

@ -1,10 +1,588 @@
{
"posts": [
{
"title": "Realtime WebRTC HTTP Endpoints",
"description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.",
"date": "2026-03-12",
"url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints"
"title": "Release cadence changes and an LTS version pilot",
"description": "Starting the week of November 2, 2026, release candidates and stable releases ship on Tuesday mornings, and LiteLLM is piloting a long-term support line on 1.108 for one month.",
"date": "2026-11-02",
"url": "https://docs.litellm.ai/blog/release-cadence-and-lts-pilot",
"tags": [
"release",
"support"
]
},
{
"title": "How to build an agent with grounding on LiteLLM",
"description": "Build a grounded chat agent in four steps with LiteLLM's AI Gateway. Connect your evidence, call a grounding guardrail, and show its verdict in chat.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/grounded-agent",
"tags": [
"engineering",
"guardrails",
"agents"
]
},
{
"title": "Introducing LiteLLM’s Unified Cost Accounting Engine",
"description": "Consistent cost reporting, independently scalable accounting, and spend analysis by team, user, model, or custom dimensions.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/cost-infrastructure-makeover",
"tags": [
"engineering",
"reliability"
]
},
{
"title": "What we are changing after October 21",
"description": "Why the controls built after March did not stop the v1.107.0 release, and the identity, credential and detection changes we are making in response.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/security-changes-after-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Use Hex to analyze LiteLLM costs, then set budgets and routing",
"description": "Connect Hex to LiteLLM’s Postgres data, import a cost dashboard, then use the LiteLLM UI to set a budget or change model routing.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/hex-litellm-roi",
"tags": [
"observability",
"cost tracking",
"ai-gateway"
]
},
{
"title": "Introducing the LiteLLM Private Registry",
"description": "LiteLLM is committing to a private registry to provide early access to fixes for the most severe security vulnerabilities, ahead of public release.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/litellm-private-registry",
"tags": [
"security"
]
},
{
"title": "Morph open-weight models on LiteLLM",
"description": "Use Morph's Kimi, GLM, and DeepSeek models through LiteLLM with tool calling, structured output, streaming, and prompt-cache cost tracking.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/morph-provider",
"tags": []
},
{
"title": "Abliterated models on LiteLLM",
"description": "Call abliterated models from Refuseless and Abliteration through LiteLLM, with gateway access controls, streaming, tool calls, and cached-token pricing.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/abliterated-models",
"tags": [
"engineering"
]
},
{
"title": "GPT-6.1 Terra API Pricing and Day 0 Support on LiteLLM",
"description": "GPT-6.1 Terra costs $1 per 1M input tokens and $5 per 1M output tokens, half the price of GPT-6.1 Sol. LiteLLM supports it on day 0.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/gpt-6-1-terra",
"tags": [
"openai",
"gpt-6.1",
"gpt-6.1-terra",
"pricing",
"day 0 support",
"card_title:GPT-6.1 Terra",
"card_summary:Half the price of GPT-6.1 Sol at $1 / $5 per 1M tokens, 50.3% on Terminal-Bench 4.0, 1.05M context."
]
},
{
"title": "Rust Migration: /v1/messages end-to-end in Rust by Dec 1",
"description": "By December 1st, the proxy will handle Anthropic messages requests 20x faster",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/rust-migration-rerouted",
"tags": [
"rust",
"rust-migration",
"ai-gateway",
"engineering"
]
},
{
"title": "OpenCode Go is now a LiteLLM provider",
"description": "Use your OpenCode Go subscription through LiteLLM. Connect once, route models to the right API, preserve sessions, and track estimated usage.",
"date": "2026-10-24",
"url": "https://docs.litellm.ai/blog/opencode-go",
"tags": [
"providers",
"tutorial"
]
},
{
"title": "The attack chain behind the v1.107.0 LiteLLM release",
"description": "How an internal key committed to a public repository in May led to three stolen GitHub tokens and the compromised litellm v1.107.0 release.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Tokenize text across vLLM and Watsonx",
"description": "Choose token counting for prompt budgets or /tokenize to inspect tokens. Use one request format across vLLM and Watsonx through LiteLLM.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/unified-tokenize",
"tags": [
"engineering"
]
},
{
"title": "October Townhall Updates: 426 Bug Fixes, 88.4% Coverage, and the Decisions API",
"description": "A recap of the October LiteLLM town hall: security changes that need action before you upgrade, 426 bug fixes, test coverage at 88.4%, the Decisions API, Lens, Moyai, and LiteLLM Relay.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/october-townhall-updates",
"tags": [
"townhall",
"security",
"reliability",
"product"
]
},
{
"title": "Contained: Oct 21, 2026 LiteLLM release v1.107.0 Compromised",
"description": "LiteLLM release v1.107.0 was compromised by a malicious actor.",
"date": "2026-10-21",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "JevRank: decision models through LiteLLM’s rerank API",
"description": "Preview JevRank for low-cost reranking with Perplexity and TypeSafe decision models. Explore SciFact results, reproducible data and LiteLLM setup.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/jevrank",
"tags": [
"engineering",
"reranking",
"decisions"
]
},
{
"title": "Cache-Aware Routing for Self-Hosted Models",
"description": "Route each agent session to the same vLLM pod on Kubernetes to reuse cached prompt prefixes.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/cache-aware-routing",
"tags": [
"routing",
"kubernetes",
"vllm",
"ai-gateway"
]
},
{
"title": "Day 0 Support: TypeSafe Jev 1.14: Adds images, box questions, and long context",
"description": "TypeSafe Jev 1.14 reads images, answers box questions with coordinates and takes long-context requests. LiteLLM serves all three on /v1/systemone and /v1/decisions behind TypeSafe's opt-in headers.",
"date": "2026-10-15",
"url": "https://docs.litellm.ai/blog/typesafe_jev_images",
"tags": [
"typesafe",
"jev",
"decisions",
"day 0 support",
"card_title:Jev 1.14",
"card_summary:Jev 1.14 reads PNG and JPEG images, answers box questions with coordinates, and takes long-context requests on LiteLLM."
]
},
{
"title": "Decision models on LiteLLM: /v1/decisions and /v1/systemone",
"description": "Decision models return probabilities instead of text, which makes routing, triage and guardrail checks cheap and deterministic. LiteLLM serves nine providers of them behind one request shape.",
"date": "2026-10-14",
"url": "https://docs.litellm.ai/blog/decision-models",
"tags": [
"openai",
"typesafe",
"decisions",
"day 0 support",
"card_title:Decision Models",
"card_summary:Typed yes/no, choice and score answers as probabilities, from 9 providers on /v1/decisions and /v1/systemone."
]
},
{
"title": "Workshop: Build a self-hosted cloud agent.",
"description": "Build a self-hosted cloud agent with the LiteLLM team. A two-week remote workshop, October 19–30, to build a working v0 and deploy an internal beta on your infrastructure.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/cloud-agent-workshop",
"tags": [
"agents",
"infrastructure",
"announcement"
]
},
{
"title": "Gemini 4 Argon API Pricing and Day 0 Support on LiteLLM",
"description": "Gemini 4 Argon costs $2.00 per 1M input tokens and $10.00 per 1M output tokens until Nov 30. LiteLLM supports it on day 0 on AI Studio and Google Cloud.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/gemini-4-argon",
"tags": [
"gemini",
"gemini 4",
"gemini-4-argon",
"pricing",
"day 0 support",
"card_title:Gemini 4 Argon",
"card_summary:Google's new flagship: 77.9% on DeepSWE v1.1, 1M context, thinking levels, $2 / $10 per 1M tokens until Nov 30."
]
},
{
"title": "October Townhall: Product + Roadmap Updates",
"description": "Join the LiteLLM October townhall on Friday, 23 October at 7:30 AM PT to learn about LiteLLM's product updates and roadmap.",
"date": "2026-10-10",
"url": "https://docs.litellm.ai/blog/october-townhall-announcement",
"tags": [
"announcement",
"townhall"
]
},
{
"title": "Introducing Microsoft 365 Copilot in LiteLLM",
"description": "Use Microsoft 365 Copilot from your applications through LiteLLM, with access to your Microsoft 365 work data.",
"date": "2026-10-09",
"url": "https://docs.litellm.ai/blog/microsoft-365-copilot",
"tags": [
"announcement",
"ai-gateway"
]
},
{
"title": "Introducing litellm-core: the first step toward a leaner SDK",
"description": "Ongoing work to reduce LiteLLM's dependencies and installation footprint, improve runtime loading, and make imports faster",
"date": "2026-10-08",
"url": "https://docs.litellm.ai/blog/introducing-litellm-core",
"tags": []
},
{
"title": "Claude Haiku 5.5 API Pricing and Day 0 Support on LiteLLM",
"description": "Claude Haiku 5.5 costs $0.10 per 1M input tokens and $0.50 per 1M output tokens. LiteLLM supports it on day 0 on Anthropic, AWS, Google Cloud and Azure.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/claude-haiku-5-5",
"tags": [
"anthropic",
"claude",
"claude haiku",
"haiku 5.5",
"pricing",
"day 0 support",
"card_title:Claude Haiku 5.5",
"card_summary:90% cheaper than Haiku 4.5 on short prompts, 1M context, 72.4% on OSWorld 2.1, $0.10 / $0.50 per 1M tokens."
]
},
{
"title": "Open Sourcing Moyai: Self-Hosted Cloud Coding Agent",
"description": "Run cloud coding agents on your own infrastructure with Moyai. Choose your agent and models through LiteLLM, track costs, and follow the setup guide.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/moyai-open-source",
"tags": [
"agents",
"open-source",
"infrastructure"
]
},
{
"title": "Day 0 Support: Nano Banana 2.1",
"description": "Day 0 support for Gemini Nano Banana 2.1 on LiteLLM, on Google AI Studio and Gemini Enterprise Agent Platform, with image output at half Nano Banana 2's price.",
"date": "2026-10-06",
"url": "https://docs.litellm.ai/blog/nano_banana_2_1",
"tags": [
"gemini",
"gemini enterprise agent platform",
"nano banana",
"image generation",
"day 0 support",
"card_title:Nano Banana 2.1",
"card_summary:Gemini image model with image output at half Nano Banana 2's price: $30 / MTok image out, $1.50 text in."
]
},
{
"title": "How LiteLLM Lens finds repeated failures across 1,000s of agent traces",
"description": "Inside Lens: parallel trace review, Python tools for large traces, and investigations that connect repeated failures to source evidence.",
"date": "2026-10-05",
"url": "https://docs.litellm.ai/blog/lens-failure-patterns",
"tags": [
"lens",
"agent-tracing",
"engineering"
]
},
{
"title": "How we cut time to first byte by 94% for long prompts",
"description": "A 440k-token benchmark went from 553 ms to 35 ms median time to first byte. We removed unnecessary prompt-cache routing work before the model call.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/long-prompt-latency",
"tags": [
"performance",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we built our own internal Devin in 2 days",
"description": "How we built Moyai Devin with Render, Modal, Hermes, Temporal, and LiteLLM: durable sessions, parallel agents, Slack, and shared organization connections.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/internal-devin-two-days",
"tags": [
"engineering",
"agents",
"infrastructure",
"slack"
]
},
{
"title": "Adding Self-hosted Auto Router Classifiers: Laya & Nimble",
"description": "Use Laya or Bespoke Nimble to classify Auto Router requests on your own infrastructure. Control where classification runs, which model you serve, and how you provision it.",
"date": "2026-10-02",
"url": "https://docs.litellm.ai/blog/laya-nimble-classifiers",
"tags": [
"auto-router",
"product",
"ai-gateway"
]
},
{
"title": "Launching LiteLLM Lens",
"description": "LiteLLM Lens turns the traces flowing through your gateway into findings your agents can act on. Built for agent swarms generating 200K+ traces.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/litellm-lens-launch",
"tags": [
"lens",
"agent-tracing"
]
},
{
"title": "How we cut LiteLLM's Redis round trips per request by 64%",
"description": "A governed request to the LiteLLM AI Gateway waited on Redis 22 times. It now waits 8 times: one pipeline per Redis backend before the model call, one after.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/redis-request-pipelines",
"tags": [
"performance",
"redis",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we made the LiteLLM Usage page 120x faster",
"description": "From six minutes to 3.2 seconds: how moving aggregation into Postgres made LiteLLM's Usage page roughly 120x faster in our benchmark.",
"date": "2026-09-30",
"url": "https://docs.litellm.ai/blog/usage-page-bounded-loading",
"tags": [
"performance",
"admin-ui",
"postgres",
"engineering",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6.1 Sol",
"description": "Day 0 support for GPT-6.1 Sol on LiteLLM, with cached input at half GPT-6 Sol's price.",
"date": "2026-09-29",
"url": "https://docs.litellm.ai/blog/gpt_6_1_sol",
"tags": [
"openai",
"gpt-6",
"gpt-6.1-sol",
"completion",
"day 0 support",
"card_title:GPT-6.1 Sol",
"card_summary:GPT-6 Sol successor with cached input at half GPT-6 Sol's price, same request family and params."
]
},
{
"title": "Learn the LiteLLM gateway with a guided course",
"description": "Follow a request through the LiteLLM gateway, Router, and SDK. A guided course for teams deploying the gateway and contributors making code changes.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/litellm-gateway-course",
"tags": [
"proxy",
"product"
]
},
{
"title": "Day 0 Support: Claude Sonnet 5.5",
"description": "Day 0 support for Claude Sonnet 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/claude_sonnet_5_5",
"tags": [
"anthropic",
"claude",
"sonnet 5.5",
"day 0 support",
"card_title:Claude Sonnet 5.5",
"card_summary:Same $2 / $10 price as Sonnet 5, 30%+ faster, 70.6% on Terminal-Bench 4.0 (Sonnet 5: 10.3%), 1M context."
]
},
{
"title": "Introducing the LiteLLM ROI Calculator",
"description": "Compare each engineer's LiteLLM gateway spend with the estimated effort of their merged pull requests. Open source and self-hosted.",
"date": "2026-09-26",
"url": "https://docs.litellm.ai/blog/litellm-roi-calculator",
"tags": [
"cost",
"product",
"announcement"
]
},
{
"title": "Introducing LiteAgents",
"description": "Switch agent harnesses without rewriting your agent. Keep your tools and model configuration, use native harness controls, and add Temporal when you need durable runs.",
"date": "2026-09-25",
"url": "https://docs.litellm.ai/blog/liteagents-sdk",
"tags": [
"agents",
"sdk"
]
},
{
"title": "September Townhall Updates: 583 Bug Fixes, OCR on Rust, and 83.5% Coverage",
"description": "A recap of the September LiteLLM town hall: security and stability updates, OCR running on Rust by default, test coverage past the 80% target, and new Fusion and liteagents launches.",
"date": "2026-09-24",
"url": "https://docs.litellm.ai/blog/september-townhall-updates",
"tags": [
"townhall",
"reliability",
"product"
]
},
{
"title": "Day 0 support: Gemini 3.8 Flash TTS and Flash-Lite TTS",
"description": "Day 0 support for Gemini 3.8 Flash TTS and Flash-Lite TTS on LiteLLM's /v1/audio/speech, with launch pricing tracked through 2026.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/gemini_3_8_flash_tts",
"tags": [
"gemini",
"gemini-3.8-flash-tts",
"text to speech",
"day 0 support",
"card_title:Gemini 3.8 Flash TTS and Flash-Lite TTS",
"card_summary:Gemini speech models on /v1/audio/speech, Flash and Flash-Lite, with launch pricing tracked through 2026."
]
},
{
"title": "Introducing LiteAdmin MCP",
"description": "Give your agent tools to create keys, add models, and manage budgets with LiteAdmin MCP. Use the same connector through LiteAdmin, our Slack admin agent.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/litellm-admin-mcp",
"tags": [
"mcp",
"agents",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6 Sol and GPT-6 Luna",
"description": "Day 0 support for GPT-6 Sol and GPT-6 Luna on LiteLLM, at half the price of their GPT-5.6 counterparts.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/gpt_6_sol_luna",
"tags": [
"openai",
"gpt-6",
"gpt-6-sol",
"gpt-6-luna",
"completion",
"day 0 support",
"card_title:GPT-6 Sol and GPT-6 Luna",
"card_summary:Two GPT-6 variants at half the price of their GPT-5.6 counterparts, routed as the GPT-5 request family."
]
},
{
"title": "Day 0 Support: Claude Opus 5.5",
"description": "Day 0 support for Claude Opus 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/claude_opus_5_5",
"tags": [
"anthropic",
"claude",
"opus 5.5",
"day 0 support",
"card_title:Claude Opus 5.5",
"card_summary:20% cheaper than Opus 5 at $4 / $20 per 1M, 30% faster output, Terminal-Bench 4.0 66.4%, thinking always on."
]
},
{
"title": "Getting Started with Fireworks AI on LiteLLM",
"description": "A beginner tutorial: run an open model on Fireworks AI behind a LiteLLM gateway, call it with the OpenAI SDK you already use, then add a second model and a fallback.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/fireworks-getting-started",
"tags": [
"tutorial",
"providers",
"fireworks",
"getting-started"
]
},
{
"title": "Day 0 Support: Xiaomi MiMo V2.6",
"description": "Day 0 support for Xiaomi MiMo V2.6 Pro and Flash on LiteLLM, priced on the native route for the first time.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/mimo_v2_6",
"tags": [
"xiaomi",
"mimo",
"mimo-v2.6",
"day 0 support",
"card_title:Xiaomi MiMo V2.6",
"card_summary:Xiaomi's Pro ($0.435 / $0.87 per 1M) and Flash ($0.14 / $0.28), same rates as V2.5, priced on the native route."
]
},
{
"title": "Claude Code server-side auto mode through LiteLLM",
"description": "Anthropic is moving Claude Code auto mode's safety classifier server-side. LiteLLM's native /v1/messages route now forwards the safeguards contract to the Anthropic API, Bedrock InvokeModel, Bedrock Mantle and Vertex AI. Here is the contract, what changed, which releases carry it, and how to verify it.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/claude-code-server-side-auto-mode",
"tags": [
"announcement",
"claude-code",
"anthropic",
"ai-gateway"
]
},
{
"title": "Day 0 Support: Grok 4.7",
"description": "Day 0 support for Grok 4.7 on LiteLLM, at the same price as Grok 4.6.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/grok_4_7",
"tags": [
"xai",
"grok",
"grok-4.7",
"day 0 support",
"card_title:Grok 4.7",
"card_summary:SpaceXAI's update to Grok 4.6 at the same price and limits, better at checking its own work and long context."
]
},
{
"title": "JEV Classifier: 5.43x as Fast as Haiku, 96% Lower Cost",
"description": "JEV classified requests 5.43x as fast as Haiku by median latency in our AI Gateway benchmark. Explore the setup, cost savings and methodology.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/jev-auto-router-benchmark",
"tags": [
"routing",
"complexity-router",
"engineering",
"ai-gateway"
]
},
{
"title": "TypeSafe Jev on LiteLLM",
"description": "TypeSafe AI's Jev lands in LiteLLM v1.103.0-rc: call it through the proxy with logging and cost tracking.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/typesafe_jev",
"tags": [
"typesafe",
"jev",
"product"
]
}
]
}

View file

@ -135,6 +135,7 @@ ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
DEFAULT_MAX_REQUEST_HEADER_SIZE: Final = 16 * 1024
budget_reservation_disabled_info_emitted = False
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)
DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512)
@ -1092,6 +1093,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.scx.ai/v1",
"https://api.prisminference.com/v1",
"https://api.reka.ai/v1",
"https://inference.runanywhere.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
@ -1179,6 +1181,7 @@ openai_compatible_providers: Final[list[str]] = [
"sail",
"reka",
"streamlake",
"wally",
]
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
@ -2067,6 +2070,7 @@ SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30
SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000
SPEND_LOG_KEY_METADATA_QUERY_TIMEOUT_MS: Final = 5000
SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS: Final = 30
SPEND_LOG_KEY_METADATA_ROWS_PER_PROBE: Final = 100
# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
# callers from forcing a DB query per request for unknown names, while bounding

View file

@ -49,6 +49,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_batch_cost_rates,
get_billable_input_tokens,
get_cost_per_unit,
get_modality_token_cost_key,
get_regional_uplift_multiplier,
get_service_tier_cost_key,
get_token_type_cost_breakdown,
@ -442,6 +444,14 @@ def _per_second_pricing_cost(
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds)
def _per_second_pricing_cost_from_info(
model_info: ModelInfoBase,
response_time_ms: float | None,
audio_seconds: float = 0.0,
) -> tuple[float, float] | None:
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
cost_per_second: Final = model_info.get("cost_per_second")
@ -464,7 +474,7 @@ def _per_second_pricing_cost(
)
verbose_logger.debug(
"For model=%s - cost_per_second: %s; response time: %s",
model,
model_info.get("key"),
resolved_cost_per_second,
response_time_ms,
)
@ -487,6 +497,7 @@ def cost_per_token(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
### NUMBER OF QUERIES ###
number_of_queries: int | None = None,
### USAGE OBJECT ###
@ -598,6 +609,20 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
if requested_model_info_override is not None:
per_second: Final = _per_second_pricing_cost_from_info(
requested_model_info_override, response_time_ms, audio_transcription_file_duration
)
if per_second is not None:
return per_second
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider or "",
service_tier=service_tier,
model_info=requested_model_info_override,
)
model_cost_ref: Final = litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
@ -1747,6 +1772,7 @@ def completion_cost(
### CUSTOM PRICING ###
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
optional_params: dict | None = None,
custom_pricing: bool | None = None,
base_model: str | None = None,
@ -1800,6 +1826,7 @@ def completion_cost(
quality=quality,
n=n,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
custom_cost_per_second=custom_cost_per_second,
optional_params=optional_params,
custom_pricing=custom_pricing,
@ -2274,6 +2301,7 @@ def completion_cost(
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
prompt_characters=prompt_characters,
completion_characters=completion_characters,
cache_creation_input_tokens=cache_creation_input_tokens,
@ -2508,6 +2536,7 @@ def response_cost_calculator(
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
requested_model_info_override: ModelInfo | None = None,
) -> float:
"""
Returns
@ -2564,6 +2593,7 @@ def response_cost_calculator(
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
requested_model_info_override=requested_model_info_override,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
@ -3158,8 +3188,12 @@ def _batch_rate(
"input_cost_per_video_token_batches",
],
fallback: float,
*,
prompt_tokens: int = 0,
) -> float:
rate: Final = model_info.get(key)
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
rate: Final = model_info.get(key) if context_rate is None else context_rate
return fallback if rate is None else rate
@ -3309,7 +3343,8 @@ def _batch_prompt_cost(
)
return (
text_tokens * input_rate
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ audio_tokens
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)

View file

@ -867,6 +867,7 @@ class BadGatewayError(openai.APIStatusError):
class InternalServerError(openai.InternalServerError):
response: httpx.Response # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX response
request: httpx.Request # pyright: ignore[reportIncompatibleVariableOverride] # LiteLLM constructs an HTTPX request
provider_status_code: int | None
def __init__(
self,
@ -878,8 +879,10 @@ class InternalServerError(openai.InternalServerError):
max_retries: int | None = None,
num_retries: int | None = None,
body: object | None = None,
provider_status_code: int | None = None,
):
self.status_code = 500
self.provider_status_code = provider_status_code
self.message = f"litellm.InternalServerError: {message}"
self.llm_provider = llm_provider
self.model = model

View file

@ -10,6 +10,7 @@ import json
import time
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable
from pydantic import ValidationError
@ -73,7 +74,7 @@ class CodeInterpreterLogOutput(TypedDict):
class CodeInterpreterCall(TypedDict):
id: str
type: Literal["code_interpreter_call"]
status: Literal["completed"]
status: Literal["completed", "failed"]
code: str
container_id: str | None
outputs: list[CodeInterpreterLogOutput]
@ -141,6 +142,39 @@ class _SupportsOutput(Protocol):
_CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None]
@dataclass(frozen=True, slots=True)
class _ReadySandbox:
container: ContainerHandle
params: SandboxToolParams | None
@dataclass(frozen=True, slots=True)
class _SandboxUnavailable:
output: str
_Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable
@dataclass(frozen=True, slots=True)
class _ToolRun:
output: str
status: Literal["completed", "failed"]
_SANDBOX_UNAVAILABLE_OUTPUT: Final = "The code did not run because the code execution sandbox is unavailable."
_SANDBOX_LOST_RESULT_OUTPUT: Final = (
"The code execution sandbox failed while running the code, so the code may have run but its result is lost."
)
def _sandbox_unavailable(step: str, error: Exception, output: str = _SANDBOX_UNAVAILABLE_OUTPUT) -> _SandboxUnavailable:
verbose_logger.warning(
"CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error
)
return _SandboxUnavailable(output=output)
def _output_item_type(item: object) -> object:
if isinstance(item, dict):
item_mapping: Final[dict[str, object]] = item
@ -189,6 +223,14 @@ def _extract_identity(kwargs: Mapping[str, object]) -> str:
return identity if isinstance(identity, str) else ""
def _bridged_from_responses(kwargs: Mapping[str, object], call_type: CallTypes | None) -> bool:
route: Final = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
return call_type in (CallTypes.completion, CallTypes.acompletion) and route in (
CallTypes.responses,
CallTypes.aresponses,
)
def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None:
if sandbox_tool_name is None:
return None
@ -261,7 +303,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
async def async_pre_call_deployment_hook(
self, kwargs: dict[str, object], call_type: CallTypes | None
) -> dict | None:
if not kwargs.get("_agentic_loop_depth"):
if not kwargs.get("_agentic_loop_depth") and not _bridged_from_responses(kwargs, call_type):
kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
kwargs.pop(_SANDBOX_KEY, None)
self._strip_interception_metadata(kwargs)
@ -451,16 +493,16 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = self._container_id(container)
container_id: Final = self._sandbox_container_id(sandbox)
input_list: Final = self._normalize_messages(messages)
code_interpreter_calls: Final[list[CodeInterpreterCall]] = []
for tool_call in tool_calls:
arguments = tool_call.get("arguments", "")
code = self._parse_code(arguments)
stdout = await self._run_tool_call(container=container, params=params, arguments=arguments)
tool_run = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
input_list.append(
{
"type": "function_call",
@ -473,22 +515,23 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
{
"type": "function_call_output",
"call_id": tool_call.get("call_id"),
"output": stdout,
"output": tool_run.output,
}
)
code_interpreter_calls.append(
{
"id": f"ci_{uuid.uuid4().hex}",
"type": "code_interpreter_call",
"status": "completed",
"status": tool_run.status,
"code": code,
"container_id": container_id,
"outputs": ([{"type": "logs", "logs": stdout}] if stdout else []),
"outputs": ([{"type": "logs", "logs": tool_run.output}] if tool_run.output else []),
}
)
except Exception:
await self._delete_container_for_cache_key(sandbox_key)
raise
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
optional_params: Final = anthropic_messages_optional_request_params
request_patch: Final = AgenticLoopRequestPatch(
@ -526,14 +569,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
sandbox_key: Final = self._extract_sandbox_key(kwargs)
is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
identity: Final = _extract_identity(kwargs) if is_session else None
container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity)
try:
container_id: Final = self._container_id(container)
container_id: Final = self._sandbox_container_id(sandbox)
tool_results: Final = [
await self._build_chat_completion_tool_result(
container=container,
params=params,
sandbox=sandbox,
tool_call=tool_call,
container_id=container_id,
)
@ -544,6 +586,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
raise
tool_messages: Final = [result[0] for result in tool_results]
code_interpreter_calls: Final = [result[1] for result in tool_results]
await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls)
request_patch: Final = AgenticLoopRequestPatch(
model=model,
@ -569,9 +612,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
)
@staticmethod
def _container_id(container: ContainerHandle) -> str | None:
container_id: Final[object] = getattr(container, "id", None)
return container_id if isinstance(container_id, str) else None
def _sandbox_container_id(sandbox: _Sandbox) -> str | None:
match sandbox:
case _ReadySandbox(container=container):
container_id: Final[object] = getattr(container, "id", None)
return container_id if isinstance(container_id, str) else None
case _SandboxUnavailable():
return None
@staticmethod
def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]:
@ -588,28 +635,27 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
async def _build_chat_completion_tool_result(
self,
container: ContainerHandle,
params: SandboxToolParams | None,
sandbox: _Sandbox,
tool_call: CodeExecutionToolCall,
container_id: str | None,
) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
arguments: Final = tool_call.get("arguments", "")
code: Final = self._parse_code(arguments)
stdout: Final = await self._run_tool_call(container=container, params=params, arguments=arguments)
tool_run: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments)
tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex
return (
{
"role": "tool",
"tool_call_id": tool_call_id,
"content": stdout,
"content": tool_run.output,
},
{
"id": f"ci_{uuid.uuid4().hex}",
"type": "code_interpreter_call",
"status": "completed",
"status": tool_run.status,
"code": code,
"container_id": container_id,
"outputs": [{"type": "logs", "logs": stdout}] if stdout else [],
"outputs": [{"type": "logs", "logs": tool_run.output}] if tool_run.output else [],
},
)
@ -694,19 +740,55 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
except (json.JSONDecodeError, TypeError, AttributeError):
return ""
async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str:
async def _evict_sandbox_if_a_run_failed(
self, sandbox_key: str | None, sandbox: _Sandbox, calls: Sequence[CodeInterpreterCall]
) -> None:
if not sandbox_key or not isinstance(sandbox, _ReadySandbox):
return
if not any(call["status"] == "failed" for call in calls):
return
cached: Final = self._container_cache.get(sandbox_key)
if cached is None or cached[0] is not sandbox.container:
return
self._container_cache.pop(sandbox_key)
await self._delete_container(container=cached[0], params=cached[1])
async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun:
match sandbox:
case _SandboxUnavailable(output=output):
return _ToolRun(output=output, status="failed")
case _ReadySandbox(container=container, params=params):
match await self._run_tool_call(container=container, params=params, arguments=arguments):
case _SandboxUnavailable(output=output):
return _ToolRun(output=output, status="failed")
case str() as output:
return _ToolRun(output=output, status="completed")
async def _run_tool_call(
self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str
) -> str | _SandboxUnavailable:
try:
code: Final = json.loads(arguments).get("code", "") if arguments else ""
except (json.JSONDecodeError, TypeError):
return "[invalid tool arguments: could not parse code]"
result: Final = await self._run_code(container=container, params=params, code=code)
try:
result: Final = await self._run_code(container=container, params=params, code=code)
except Exception as e:
return _sandbox_unavailable("running the code", e, _SANDBOX_LOST_RESULT_OUTPUT)
if getattr(result, "error", None):
error: Final = result.error
message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error)
return f"[execution error] {message}"
return getattr(result, "stdout", "") or ""
async def _acquire_sandbox(self, cache_key: str | None, identity: str | None) -> _Sandbox:
try:
container, params = await self._get_or_create_container(cache_key=cache_key, identity=identity)
except Exception as e:
return _sandbox_unavailable("creating the sandbox", e)
return _ReadySandbox(container=container, params=params)
async def _get_or_create_container(
self,
cache_key: str | None,

View file

@ -0,0 +1,36 @@
from dataclasses import dataclass
from typing import Final, TypeAlias
from litellm.types.utils import Usage
ServedRound: TypeAlias = tuple[Usage, float]
_MARK_ATTRIBUTE: Final = "_litellm_agentic_followup_mark"
@dataclass(frozen=True, slots=True)
class _FollowUpMark:
served_rounds: tuple[ServedRound, ...]
def _mark_of(error: BaseException) -> _FollowUpMark | None:
mark: Final[object] = getattr(error, _MARK_ATTRIBUTE, None)
return mark if isinstance(mark, _FollowUpMark) else None
def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None:
setattr(error, _MARK_ATTRIBUTE, _FollowUpMark(served_rounds))
def is_agentic_followup_failure(error: BaseException) -> bool:
return _mark_of(error) is not None
def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]:
mark: Final = _mark_of(error)
return () if mark is None else mark.served_rounds
def carry_agentic_followup_mark(source: BaseException, mapped: BaseException) -> None:
if mapped is not source and is_agentic_followup_failure(source):
mark_agentic_followup_failure(mapped, rounds_served_before(source))

View file

@ -0,0 +1,107 @@
import asyncio
from collections.abc import Awaitable, Iterable, Mapping
from itertools import chain
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import (
ChatCompletionMessageCustomToolCall,
ChatCompletionMessageToolCall,
Choices,
ModelResponse,
)
def _strings(values: Iterable[object]) -> frozenset[str]:
return frozenset(value for value in values if isinstance(value, str))
def _claimed_calls(tool_calls: object) -> tuple[object, ...]:
wrapped: Final = message_field(tool_calls, "tool_calls")
return parts_of(tool_calls if wrapped is None else wrapped)
async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[object, ...]:
try:
should_run, tool_calls = await gate
except Exception as e:
verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed with %s", type(e).__name__)
return ()
return _claimed_calls(tool_calls) if should_run else ()
async def every_claimed_tool_call(
refused_tool_calls: object, other_gates: Iterable[Awaitable[tuple[bool, object]]]
) -> Mapping[str, tuple[object, ...]]:
claims: Final = await asyncio.gather(*(_calls_a_gate_claims(gate) for gate in other_gates))
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
def _call_ids(call: object) -> frozenset[str]:
return _strings((message_field(call, "id"), message_field(call, "call_id")))
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
calls: Final = _claimed_calls(tool_calls)
ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls))
names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call))
return ids, names
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names
def is_refused_item(
item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str]
) -> bool:
if message_field(item, "type") != item_type:
return False
return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names)
def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
kept: Final = [
item
for item in parts_of(message_field(response, "output"))
if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names)
]
return response.model_copy(update={"output": kept})
def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
name: Final = message_field(message_field(call, "function"), "name")
return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names)
def _at_position(
call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, position: int
) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall:
return call if message_field(call, "index") is None else call.model_copy(update={"index": position})
def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object:
if not isinstance(choice, Choices) or not choice.message.tool_calls:
return choice
surviving: Final = (
call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names)
)
kept: Final = [_at_position(call, position) for position, call in enumerate(surviving)]
finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason
return choice.model_copy(
update={
"message": choice.message.model_copy(update={"tool_calls": kept or None}),
"finish_reason": finish_reason,
}
)
def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
return response.model_copy(
update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]}
)

View file

@ -1,29 +1,35 @@
# this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api
import json
from collections.abc import Mapping
from collections.abc import Awaitable, Iterator, Mapping, Sequence
from itertools import chain
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
from pydantic import TypeAdapter, ValidationError
from litellm._internal_context import internal_sub_call
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopLimitError,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
converted_stream_requested,
is_interception_internal_key,
stashed_stream_options,
@ -31,6 +37,8 @@ from litellm.types.integrations.custom_logger import (
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
_DispatchResult: Final = TypeVar("_DispatchResult")
_FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
(
"acompletion",
@ -44,22 +52,41 @@ _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset(
)
async def bill_first_round_on_failure(
dispatch: Awaitable[_DispatchResult], *, logging_obj: object, response: object, stream: bool
) -> _DispatchResult:
try:
return await dispatch
except Exception as error:
if stream or not isinstance(logging_obj, LiteLLMLoggingObject):
mark_agentic_followup_failure(error, rounds_served_before(error))
raise
this_round: Final = logging_obj.served_round(response)
served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,)))
mark_agentic_followup_failure(error, served_rounds)
logging_obj.record_served_rounds_for_failure(served_rounds)
raise
def _gate_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_should_run_agentic_loop
func: Final = type(callback).async_should_run_agentic_loop
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return type(callback).async_should_run_agentic_loop is not CustomLogger.async_should_run_agentic_loop
def _build_plan_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_build_agentic_loop_plan
func: Final = type(callback).async_build_agentic_loop_plan
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return type(callback).async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
def _chat_build_plan_overridden(callback: CustomLogger) -> bool:
return (
type(callback).async_build_chat_completion_agentic_loop_plan
is not CustomLogger.async_build_chat_completion_agentic_loop_plan
)
def _post_hook_overridden(callback: CustomLogger) -> bool:
base: Final = CustomLogger.async_post_agentic_loop_response_hook
func: Final = type(callback).async_post_agentic_loop_response_hook
return getattr(func, "__func__", func) is not getattr(base, "__func__", base)
return (
type(callback).async_post_agentic_loop_response_hook is not CustomLogger.async_post_agentic_loop_response_hook
)
_STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
@ -76,14 +103,14 @@ def _coerce_int(value: object, default: int) -> int:
return int(value) if isinstance(value, (int, str)) else default
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, tuple[str, ...]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
fingerprints: Final = tuple(str(fp) for fp in raw_fingerprints) if isinstance(raw_fingerprints, list) else ()
return depth, max_loops, fingerprints
@ -96,16 +123,16 @@ def _fingerprint_tools(tool_calls: object) -> str:
def _check_agentic_loop_safety(
tool_calls: object,
fingerprints: list[str],
fingerprints: Sequence[str],
depth: int,
max_loops: int,
model: str,
) -> str:
fingerprint: Final = _fingerprint_tools(tool_calls)
if fingerprint in fingerprints:
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@ -134,6 +161,83 @@ def _converted_stream_replay(
)
def _end_refused_turn(
finalized: ModelResponse,
*,
kwargs: Mapping[str, object],
depth: int,
model: str,
custom_llm_provider: str,
logging_obj: object,
) -> ModelResponse | CustomStreamWrapper:
replay: Final = _converted_stream_replay(
finalized,
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
return finalized if replay is None else replay
def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]:
return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)]
def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]:
import litellm
dynamic: Final = getattr(logging_obj, "dynamic_success_callbacks", None) or []
return tuple(callback for callback in (*litellm.callbacks, *dynamic) if isinstance(callback, CustomLogger))
async def _refused_chat_turn(
callbacks: Sequence[CustomLogger],
refusing: CustomLogger,
refused_tool_calls: object,
*,
response: ModelResponse,
model: str,
messages: list[dict[str, object]],
tools: list[dict[str, object]] | None,
custom_llm_provider: str,
kwargs: Mapping[str, object],
) -> ModelResponse:
surface_kwargs: Final = {
**kwargs,
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
"custom_llm_provider": custom_llm_provider,
}
def other_gates() -> Iterator[Awaitable[tuple[bool, object]]]:
for callback in callbacks:
if callback is refusing:
continue
yield callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(kwargs),
)
if _gate_overridden(callback):
yield callback.async_should_run_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=dict(surface_kwargs),
)
claimed: Final = await every_claimed_tool_call(refused_tool_calls, other_gates())
return finalize_refused_chat_turn(response, claimed)
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
metadata: Final = kwargs_for_followup.get("litellm_metadata")
return MappingProxyType(
@ -164,6 +268,12 @@ def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]:
}
def _leave_pricing_to_the_request(response: object) -> None:
hidden_params: Final = get_hidden_params(response)
if hidden_params is not None and not served_from_cache(response):
hidden_params.pop("response_cost", None)
async def _execute_chat_completion_agentic_plan(
*,
plan: AgenticLoopPlan,
@ -175,7 +285,7 @@ async def _execute_chat_completion_agentic_plan(
custom_llm_provider: str,
depth: int,
max_loops: int,
fingerprints: list[str],
fingerprints: Sequence[str],
fingerprint: str,
) -> object:
import litellm
@ -226,6 +336,7 @@ async def _execute_chat_completion_agentic_plan(
model,
str(e),
)
_leave_pricing_to_the_request(response_followup)
replay: Final = _converted_stream_replay(
response_followup,
kwargs=kwargs,
@ -252,23 +363,18 @@ async def maybe_run_chat_completion_agentic_loop(
*,
response: ModelResponse,
model: str,
messages: list,
optional_params: dict,
messages: list[dict[str, object]],
optional_params: dict[str, object],
kwargs: dict[str, object],
logging_obj: object,
custom_llm_provider: str,
stream: bool,
) -> ModelResponse | CustomStreamWrapper | None:
import litellm
callbacks: Final = litellm.callbacks + (getattr(logging_obj, "dynamic_success_callbacks", None) or [])
callbacks: Final = _custom_loggers(logging_obj)
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
tools: Final = optional_params.get("tools", [])
tools: Final = _request_tools(optional_params)
for callback in callbacks:
if not isinstance(callback, CustomLogger):
continue
if not _gate_overridden(callback):
continue
@ -297,29 +403,39 @@ async def maybe_run_chat_completion_agentic_loop(
if not should_run:
continue
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
try:
if not _build_plan_overridden(callback):
return await callback.async_run_agentic_loop(
tools=tool_calls,
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return _end_refused_turn(
await _refused_chat_turn(
callbacks,
callback,
tool_calls,
response=response,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
),
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
plan = await callback.async_build_agentic_loop_plan(
if not _build_plan_overridden(callback):
return await callback.async_run_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
@ -331,31 +447,38 @@ async def maybe_run_chat_completion_agentic_loop(
kwargs=hook_kwargs,
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
return response
if not plan.run_agentic_loop:
continue
plan = await callback.async_build_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
response=response,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
except Exception as e:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s",
str(e),
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
return response
if not plan.run_agentic_loop:
continue
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
return _converted_stream_replay(
response,
@ -365,3 +488,113 @@ async def maybe_run_chat_completion_agentic_loop(
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
async def run_chat_completion_agentic_hooks(
*,
response: ModelResponse,
model: str,
messages: list[dict[str, object]],
optional_params: dict[str, object],
kwargs: dict[str, object],
logging_obj: LiteLLMLoggingObject,
custom_llm_provider: str,
stream: bool,
) -> object | None:
callbacks: Final = _custom_loggers(logging_obj)
depth, max_loops, fingerprints = _agentic_loop_settings(kwargs)
tools: Final = _request_tools(optional_params)
hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider}
for callback in callbacks:
try:
should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
except Exception:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop"
)
continue
if not should_run:
continue
try:
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return await _refused_chat_turn(
callbacks,
callback,
tool_calls,
response=response,
model=model,
messages=messages,
tools=tools,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
if not _chat_build_plan_overridden(callback):
return await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
plan = await callback.async_build_chat_completion_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=hook_kwargs,
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
verbose_logger.debug(
"Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason
)
return response
if not plan.run_agentic_loop:
continue
return await _execute_chat_completion_agentic_plan(
plan=plan,
callback=callback,
model=model,
optional_params=optional_params,
kwargs=hook_kwargs,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
return None

View file

@ -1,4 +1,4 @@
from collections.abc import Iterator, Mapping
from collections.abc import Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import Final
@ -25,6 +25,14 @@ def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
def deployment_backend_model(deployment: object) -> str:
return next(iter(deployment_pricing_models(deployment)), "")
def deployment_backend_models(deployments: Iterable[object]) -> tuple[str, ...]:
return tuple(map(deployment_backend_model, deployments))
def _named_models(*candidates: object) -> Iterator[str]:
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)

View file

@ -133,6 +133,8 @@ _STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType(
}
)
_OVERLOADED_STATUS_CODE: Final = "529"
_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = (
TypeError,
KeyError,
@ -201,6 +203,8 @@ def normalize_error(exc: Exception | None, status_code: str, message: str) -> st
)
if by_message is not None:
return by_message
if status_code == _OVERLOADED_STATUS_CODE:
return PROVIDER_OVERLOADED
by_class: Final = _classify_by_class(exc)
if by_class is not None:
return by_class

View file

@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import _ENABLE_SECRET_REDACTION, redact_string, verbose_logger
from litellm.litellm_core_utils.agentic_followup_failure import carry_agentic_followup_mark
from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
@ -605,6 +606,11 @@ def _map_openai_exception(
)
def _anthropic_overload_status(original_exception: _ProviderHTTPException) -> int | None:
upstream_status: Final = getattr(original_exception, "status_code", None)
return 529 if isinstance(upstream_status, int) and upstream_status >= 500 else None
def _map_anthropic_exception(
*,
model: str,
@ -630,6 +636,7 @@ def _map_anthropic_exception(
message=f"AnthropicError - {error_str}",
model=model,
llm_provider="anthropic",
provider_status_code=_anthropic_overload_status(original_exception),
)
if "Invalid API Key" in error_str:
raise AuthenticationError(
@ -694,6 +701,7 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
response=getattr(original_exception, "response", None),
provider_status_code=original_exception.status_code,
)
elif original_exception.status_code == 502:
raise BadGatewayError(
@ -2895,6 +2903,7 @@ def exception_type(
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request
)
except Exception as e:
carry_agentic_followup_mark(source=original_exception, mapped=e)
# LOGGING
exception_logging(
logger_fn=None,
@ -2920,6 +2929,7 @@ def exception_type(
model="",
)
setattr(raised_exc, "litellm_response_headers", litellm_response_headers)
carry_agentic_followup_mark(source=original_exception, mapped=raised_exc)
raise raised_exc

View file

@ -33,6 +33,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
)
PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header"
AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset(
{"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"}
)
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset(
{
"token_exchange_endpoint",
@ -94,6 +97,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
| ANTHROPIC_WIF_KWARGS_KEYS
| OPENAI_WIF_KWARGS_KEYS
| OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS
| AGENTIC_LOOP_CONTROL_KWARGS_KEYS
| frozenset(CustomPricingLiteLLMParams.model_fields)
)

View file

@ -54,6 +54,7 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
declared_cost_map_pricing,
@ -77,6 +78,7 @@ from litellm.integrations.langtrace import langtrace_trace_endpoint
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
from litellm.litellm_core_utils.classifier_logging import (
classifier_audit_fields,
classifier_input_snapshot,
@ -722,6 +724,7 @@ class Logging(LiteLLMLoggingBaseClass):
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
custom_model_info: ModelInfo | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -943,6 +946,11 @@ class Logging(LiteLLMLoggingBaseClass):
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self._stream_failure_claimed: bool = False
self._stash_owed_on_success: bool = False
self._stash_billed_by_continuation: bool = False
self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
self._failed_attempt_ids_owed_on_success: tuple[str, ...] = ()
self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = ()
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
@ -2227,6 +2235,7 @@ class Logging(LiteLLMLoggingBaseClass):
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"requested_model_info_override": self.custom_model_info if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
@ -2318,12 +2327,19 @@ class Logging(LiteLLMLoggingBaseClass):
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True)
def _price_without_recording(
self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool
) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
previous_failure_debug_info: Final = self.model_call_details.get(debug_key)
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
cost: Final = self.response_cost_calculator(
result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost
)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
@ -2423,6 +2439,10 @@ class Logging(LiteLLMLoggingBaseClass):
additional_headers=additional_headers,
)
def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None:
"""Price this call from its own full price entry instead of ``litellm.model_cost``."""
self.custom_model_info = custom_model_info
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
@ -2771,27 +2791,43 @@ class Logging(LiteLLMLoggingBaseClass):
own usage while the request's failure log still bills every attempt."""
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
billed_by_continuation: Final = self._stash_billed_by_continuation
owed_on_success: Final = self._stash_owed_on_success
self._stash_billed_by_continuation = False
self._stash_owed_on_success = False
captured_attempt_ids: Final = (
self.capture_failed_attempt_usage(usage)
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None
else ()
)
self.model_call_details.pop("_accounting_capture", None)
self.model_call_details.pop("_accounting_pricing_snapshot", None)
self.model_call_details.pop("_accounting_usage_observed", None)
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
if not isinstance(usage, Usage):
return
attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0)
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
attempt,
)
if not owed_on_success or billed_by_continuation:
return
self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt)
self._failed_attempt_ids_owed_on_success = (*self._failed_attempt_ids_owed_on_success, *captured_attempt_ids)
def capture_failed_attempt_usage(self, usage: Usage) -> None:
def mark_stash_billed_by_continuation(self) -> None:
self._stash_billed_by_continuation = True
def capture_failed_attempt_usage(self, usage: Usage) -> tuple[str, ...]:
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
capture: Final = type(self).usage_capture_callback
if capture is None:
return
return ()
attempt: Final = self.attempt_scoped_copy()
attempt.prepared_cost = None
attempt.finalized_cost = None
@ -2826,11 +2862,13 @@ class Logging(LiteLLMLoggingBaseClass):
datetime.datetime.now(datetime.timezone.utc),
False,
)
if isinstance(captured, CapturedUsage):
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
if not isinstance(captured, CapturedUsage):
return ()
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
return tuple(write.event.event_id for write in captured.writes)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
@ -2838,20 +2876,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
def _add_continued_partial_spend(self, response: object) -> Usage | None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
return own_usage if isinstance(own_usage, Usage) else None
combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage])
self.model_call_details["combined_usage_object"] = combined_usage
if not isinstance(own_cost, (int, float)):
return
return combined_usage
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
@ -2861,6 +2896,27 @@ class Logging(LiteLLMLoggingBaseClass):
"mid_stream_partial_cost": partial_cost,
},
)
return combined_usage
def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None:
own_cost: Final = self.model_call_details.get("response_cost")
if own_usage is None or not isinstance(own_cost, (int, float)):
return
failed_attempts: Final = self._failed_attempts_owed_on_success
if not failed_attempts:
return
failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts)
self.model_call_details.pop("failed_attempts_usage", None)
self.failed_attempts_billed_on_success = failed_attempts
self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
if self._failed_attempt_ids_owed_on_success:
self.model_call_details["_accounting_included_attempt_ids"] = self._failed_attempt_ids_owed_on_success
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
)
self.model_call_details["response_cost"] = own_cost + failed_attempts_cost
if self.cost_breakdown is not None:
self.cost_breakdown = _with_failed_attempts_cost(self.cost_breakdown, own_cost, failed_attempts_cost)
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
@ -2868,6 +2924,24 @@ class Logging(LiteLLMLoggingBaseClass):
self._stream_failure_claimed = True
return True
def served_round(self, response: object) -> ServedRound | None:
usage: Final = get_usage_object(completion_response=response)
if usage is None:
return None
price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False)
return usage, price.cost or 0.0
def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None:
if not served_rounds:
return
self._stash_owed_on_success = True
self.record_partial_usage_for_failure(
served_rounds[0][0]
if len(served_rounds) == 1
else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]),
sum(cost for _, cost in served_rounds),
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -3172,9 +3246,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("cache_hit") is True:
self.model_call_details["response_cost"] = 0.0
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
@ -3184,6 +3260,7 @@ class Logging(LiteLLMLoggingBaseClass):
pass
else:
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
if not build_logging_payload:
return None
@ -3747,7 +3824,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
@ -4343,7 +4420,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
@ -7836,6 +7913,20 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
)
def _with_failed_attempts_cost(breakdown: CostBreakdown, own_cost: float, failed_attempts_cost: float) -> CostBreakdown:
return breakdown | CostBreakdown(
total_cost=own_cost + failed_attempts_cost,
additional_costs={**breakdown.get("additional_costs", {}), "failed_attempts_cost": failed_attempts_cost},
)
def _cached_cost_breakdown(served: CostBreakdown | None, failed_attempts: tuple[ServedRound, ...]) -> CostBreakdown:
cached: Final = _cache_hit_cost_breakdown(served)
if not failed_attempts:
return cached
return _with_failed_attempts_cost(cached, 0.0, sum(cost for _, cost in failed_attempts))
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
finalized: Final = getattr(logging_obj, "finalized_cost", None)
@ -8029,7 +8120,7 @@ def get_standard_logging_object_payload(
if finalized_cost is not None
else cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
_cached_cost_breakdown(uncached_price.cost_breakdown, logging_obj.failed_attempts_billed_on_success)
if uncached_price is not None
else logging_obj.cost_breakdown
),

View file

@ -1203,21 +1203,55 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
)
def get_modality_token_cost_key(
model_info: ModelInfo,
cost_key: str,
prompt_tokens: int,
*,
threshold_is_inclusive: bool = False,
) -> str:
suffix: Final = next((suffix for suffix in _NON_STANDARD_THRESHOLD_SUFFIXES if cost_key.endswith(suffix)), "")
base_key: Final = cost_key.removesuffix(suffix)
threshold_key: Final = max(
(
key
for key in model_info
if key.startswith(f"{base_key}_above_")
and key.endswith(("_tokens", f"_tokens{suffix}"))
and model_info.get(key) is not None
and _prompt_exceeds_threshold(prompt_tokens, _parse_above_token_threshold(key), threshold_is_inclusive)
),
key=_parse_above_token_threshold,
default=None,
)
if threshold_key is None:
return cost_key
return f"{threshold_key.removesuffix(suffix)}{suffix}"
def _modality_token_cost(
model_info: ModelInfo,
modality_cost_key: str,
tokens: int,
prompt_base_cost: float,
service_tier: str | None,
*,
prompt_tokens: int = 0,
threshold_is_inclusive: bool = False,
) -> float:
"""
Audio, image and video prompt tokens bill at their modality rate when the model has one.
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
"""
tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
if model_info.get(cost_key) is None:
cost_key: Final = get_modality_token_cost_key(
model_info,
get_service_tier_cost_key(modality_cost_key, service_tier),
prompt_tokens,
threshold_is_inclusive=threshold_is_inclusive,
)
standard_key: Final = cost_key.removesuffix(get_service_tier_cost_key("", service_tier))
if model_info.get(cost_key) is None and model_info.get(standard_key) is None:
return float(tokens) * prompt_base_cost
return calculate_cost_component(model_info, cost_key, tokens)
@ -1230,6 +1264,9 @@ def _calculate_input_cost(
cache_creation_cost: float,
cache_creation_cost_by_ttl: Mapping[str, float],
service_tier: str | None = None,
*,
prompt_tokens: int = 0,
threshold_is_inclusive: bool = False,
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
@ -1260,6 +1297,8 @@ def _calculate_input_cost(
prompt_tokens_details["audio_tokens"],
prompt_base_cost,
service_tier,
prompt_tokens=prompt_tokens,
threshold_is_inclusive=threshold_is_inclusive,
)
### IMAGE TOKEN COST
@ -1618,6 +1657,8 @@ def generic_cost_per_token(
cache_creation_cost=base_costs.cache_creation_rate,
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
service_tier=service_tier,
prompt_tokens=usage.prompt_tokens,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
## CALCULATE OUTPUT COST
@ -1675,11 +1716,9 @@ def generic_cost_per_token(
## IMAGE COST
if not is_text_tokens_total and image_tokens and image_tokens > 0:
_output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None)
_output_cost_per_image_token = (
_output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost
)
completion_cost += float(image_tokens) * _output_cost_per_image_token
image_cost_key: Final = get_service_tier_cost_key("output_cost_per_image_token", service_tier)
image_rate: Final = get_cost_per_unit(resolved_model_info, image_cost_key, None)
completion_cost += float(image_tokens) * (completion_base_cost if image_rate is None else image_rate)
## VIDEO COST
if not is_text_tokens_total and video_tokens and video_tokens > 0:

View file

@ -0,0 +1,104 @@
"""The price entry for a single call that sets its own custom pricing, built without writing ``litellm.model_cost``."""
from collections.abc import Mapping
from typing import Final, Literal
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing
from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfo
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO: Final = TypeAdapter(ModelInfo)
def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
"""A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that
deployment's full custom pricing under that id when the deployment is added."""
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key)
if not isinstance(metadata, Mapping):
continue
model_info = _STR_MAPPING.validate_python(metadata).get("model_info")
if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str):
return _STR_MAPPING.validate_python(model_info)
return None
def per_call_model_info(
model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"]
) -> ModelInfo | None:
"""The full price entry for one call, or None when the cost map (or the Router's registered deployment entry)
already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with
the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices.
The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the
same model name is priced from the cost map again."""
deployment: Final = _router_deployment_model_info(kwargs)
registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None
custom: Final = {
field: kwargs[field]
for field in CustomPricingLiteLLMParams.model_fields
if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field])
}
if not custom:
return None
unmapped: Final[Mapping[str, object]] = {
"key": model,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": None,
"output_cost_per_token": None,
"litellm_provider": custom_llm_provider,
"mode": mode,
"supported_openai_params": None,
}
row: Final = _catalog_row(model, custom_llm_provider) or unmapped
catalog: Final = (
{**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row
)
zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None
return _MODEL_INFO.validate_python(
{
**catalog,
**_token_rates_cleared_for_per_second_overrides(custom),
**custom,
**_tiers_with_flat_overrides(catalog, custom),
**(zeroed or {}),
"mode": catalog.get("mode") or mode,
}
)
_FLAT_TOKEN_RATE_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token")
_PER_SECOND_RATE_FIELDS: Final = ("cost_per_second", "input_cost_per_second", "output_cost_per_second")
_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def _token_rates_cleared_for_per_second_overrides(custom: Mapping[str, object]) -> Mapping[str, object]:
"""Token rates in the entry make the cost calculator skip per-second pricing, so a caller who sets only a
per-second rate gets the catalog's token rates and tier table cleared for this call."""
if not any(key in custom for key in _PER_SECOND_RATE_FIELDS) or any(
key in custom for key in _FLAT_TOKEN_RATE_FIELDS
):
return {}
return {"input_cost_per_token": None, "output_cost_per_token": None, "tiered_pricing": None}
def _tiers_with_flat_overrides(catalog: Mapping[str, object], custom: Mapping[str, object]) -> Mapping[str, object]:
"""A ``tiered_pricing`` table wins over flat rates in the cost calculator, so a flat rate the caller set is written
into every tier of the table in force (the caller's own, else the inherited one) for that direction, and the other
direction keeps each tier's own rate."""
overrides: Final = {key: custom[key] for key in _FLAT_TOKEN_RATE_FIELDS if key in custom}
tiers: Final = custom.get("tiered_pricing", catalog.get("tiered_pricing"))
if not overrides or not isinstance(tiers, (list, tuple)) or not tiers:
return {}
return {"tiered_pricing": [{**tier, **overrides} for tier in _TIERS.validate_python(tiers)]}
def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
return None

View file

@ -399,6 +399,74 @@ def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, objec
return projection.content, projection.media
_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/")
def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower()
if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES):
return ()
return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},)
_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}")
def _is_base64_data_uri(value: str) -> bool:
header_end: Final = value.find(",", 0, 256)
if header_end < 0 or not value[:header_end].endswith(";base64"):
return False
payload_start: Final = header_end + 1
if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None:
return False
last_quantum: Final = value[max(payload_start, len(value) - 4) :]
return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum
def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection:
projection: Final = _tpm_file_media(part)
fields: Final = as_str_mapping(part.get("file"))
data: Final = fields.get("file_data") if fields is not None else None
if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data):
return projection
media: Final = _tpm_embedding_file(data)
return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media)
def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is not None and part.get("type") == "file":
return _tpm_embedding_file_block(part)
if not isinstance(value, str) or not value.startswith("data:"):
return _tpm_media_leaf(value)
if not _is_base64_data_uri(value):
return _TPMMediaProjection(value)
media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value)
return _TPMMediaProjection("" if media else value, media)
def _is_json_array(value: object) -> bool:
return isinstance(value, (list, tuple))
def _tpm_embedding_item(value: object) -> _TPMMediaProjection:
return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value)
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = (
_tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value)
)
return projection.content, projection.media
def embedding_input_is_all_media(value: object) -> bool:
items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array
return all(_tpm_embedding_leaf(item).media for item in items)
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
recursion; an Anthropic tool_result nests its image parts exactly one level down."""

View file

@ -610,6 +610,127 @@ def estimate_image_tokens(
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
_MEDIA_FILE_CHARS_PER_TOKEN: Final = 4
_EmbeddingInputT: Final = TypeVar("_EmbeddingInputT")
def _max_input_tokens(model: str) -> int | None:
try:
limit: Final = litellm.get_model_info(model).get("max_input_tokens")
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
return None
return limit if isinstance(limit, int) and limit > 0 else None
def _media_file_tokens(data: str, models: Sequence[str]) -> int:
text_estimate: Final = len(data) // _MEDIA_FILE_CHARS_PER_TOKEN
ceilings: Final = tuple(map(_max_input_tokens, models))
known_ceilings: Final = tuple(ceiling for ceiling in ceilings if ceiling is not None)
if not known_ceilings or len(known_ceilings) != len(ceilings):
return text_estimate
return min(text_estimate, max(known_ceilings))
def _block_str_field(block: Mapping[str, object], container: str, field: str) -> str | None:
inner: Final = as_str_mapping(block.get(container))
value: Final = inner.get(field) if inner is not None else None
return value if isinstance(value, str) else None
def estimate_media_block_tokens(block: Mapping[str, object], models: Sequence[str], fallback_model: str | None) -> int:
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
match block.get("type"):
case "input_audio":
return estimate_audio_tokens(
_block_str_field(block, "input_audio", "data"), models, fallback_model=fallback_model
)
case "file":
return _media_file_tokens(_block_str_field(block, "file", "file_data") or "", models)
case _:
detail: Final = _block_str_field(block, "image_url", "detail")
return estimate_image_tokens(
models,
_block_str_field(block, "image_url", "url") or "",
"auto" if detail is None else detail,
fallback_model=fallback_model,
)
_PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS: Final = frozenset({"vertex_ai", "gemini", "azure_ai"})
_DataUriMediaScope: TypeAlias = Literal["per_item", "whole_batch"]
def _is_pdf_block(block: Mapping[str, object]) -> bool:
return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf")
def _bedrock_data_uri_media_scope(
resolved_model: str, media: Sequence[Mapping[str, object]]
) -> _DataUriMediaScope | None:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.utils import supports_embedding_image_input
base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/"))
if not supports_embedding_image_input(f"bedrock/{base_model}"):
return None
match BaseAWSLLM.get_bedrock_embedding_provider(base_model):
case "cohere":
return "whole_batch"
case "twelvelabs":
return None
case "nova" if any(map(_is_pdf_block, media)):
return None
case _:
return "per_item"
def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> _DataUriMediaScope | None:
from litellm.utils import supports_embedding_image_input
try:
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model)
except litellm.exceptions.BadRequestError:
return None
if provider == "bedrock":
return _bedrock_data_uri_media_scope(resolved_model, media)
if not supports_embedding_image_input(model):
return None
if provider == "cohere":
return "whole_batch"
return "per_item" if provider in _PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS else None
def split_embedding_input_tokens(
value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None
) -> tuple[_EmbeddingInputT, int]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
embedding_input_is_all_media,
split_embedding_media_for_tpm,
)
text, media = split_embedding_media_for_tpm(value)
if not media:
return value, 0
models: Final = tuple(media_models())
scopes: Final = frozenset(_data_uri_media_scope(model, media) for model in models)
if not models or None in scopes or ("whole_batch" in scopes and not embedding_input_is_all_media(value)):
return value, 0
media_tokens: Final = sum(estimate_media_block_tokens(block, models, fallback_model) for block in media)
return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape
def embedding_aware_token_count(
messages: Sequence[Mapping[str, str]] | None,
text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str]
media_models: Callable[[], Sequence[str]],
) -> int:
embedding_text, media_tokens = split_embedding_input_tokens(text, media_models)
return litellm.token_counter(messages=messages, text=embedding_text) + media_tokens
TokenCounterFunction = Callable[[str], int]
"""
Type for a function that counts tokens in a string.

View file

@ -28,7 +28,11 @@ from litellm.constants import (
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.asyncify import is_event_loop_running
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
drops_unsupported_params,
map_finish_reason,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
invalid_type_message,
@ -581,12 +585,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
@classmethod
def get_config(cls, *, model: str | None = None):
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
config: Final = super().get_config()
# anthropic requires a default value for max_tokens
if config.get("max_tokens") is None:
config["max_tokens"] = cls.get_max_tokens_for_model(model)
config["max_tokens"] = (
configured_max_output_tokens
if configured_max_output_tokens is not None
else cls.get_max_tokens_for_model(model)
)
return config
@ -2539,7 +2547,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
optional_params["tools"] = tools
## Load Config
config: Final[Mapping[str, object]] = MappingProxyType(litellm.AnthropicConfig.get_config(model=model))
model_info: Final[object] = cast( # cast-ok: request parameters are a Mapping
Mapping[str, object], litellm_params
).get("model_info")
configured_max_output_tokens: Final = (
coerce_token_limit(
cast(Mapping[str, object], model_info).get("max_output_tokens") # cast-ok: runtime Mapping check
)
if isinstance(model_info, Mapping)
else None
)
config: Final[Mapping[str, object]] = MappingProxyType(
litellm.AnthropicConfig.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
)
for k, v in config.items():
if (
k not in optional_params

View file

@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -35,10 +36,16 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
b'data: {"type": "error", "error": {"type": "api_error", "message": '
b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
)
AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = (
b"event: error\n"
b'data: {"type": "error", "error": {"type": "api_error", "message": '
b'"The agentic loop reached its limit before this turn finished. '
b'Raise max_agentic_loops to allow more loops."}}\n\n'
)
def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
return chunk in (SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES)
async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
@ -216,6 +223,7 @@ class AgenticAnthropicStreamingIterator:
self._follow_up_chunk_task: asyncio.Task | None = None
self._replay_index = 0
self._error_emitted = False
self._loop_limit_reached = False
@property
def has_buffered_provider_output(self) -> bool:
@ -288,6 +296,8 @@ class AgenticAnthropicStreamingIterator:
"tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
self._model,
)
if self._loop_limit_reached:
return AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES
return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
if self._replay_index < len(self._collected_bytes):
@ -393,14 +403,9 @@ class AgenticAnthropicStreamingIterator:
"AgenticStreamingIterator: Unexpected result type from hooks: %s",
type(result).__name__,
)
except Exception as e:
_call_id: Final = getattr(self._logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"AgenticStreamingIterator: Error in agentic hook processing [call_id=%s model=%s]: %s",
_call_id,
self._model,
str(e),
)
except AgenticLoopSafetyError as e:
self._loop_limit_reached = isinstance(e, AgenticLoopLimitError)
verbose_logger.warning("AgenticStreamingIterator: agentic loop refused [rail=%s]", type(e).__name__)
@staticmethod
def _rebuild_anthropic_response_from_sse(

View file

@ -111,10 +111,4 @@ class AzureAIRerankConfig(CohereRerankConfig):
return rerank_response
def _get_base_model(self, azure_model_group: str | None) -> str | None:
if azure_model_group is None:
return None
if azure_model_group == "offer-cohere-rerank-mul-paygo":
return "azure_ai/cohere-rerank-v3-multilingual"
if azure_model_group == "offer-cohere-rerank-eng-paygo":
return "azure_ai/cohere-rerank-v3-english"
return azure_model_group

View file

@ -965,8 +965,6 @@ class BaseAWSLLM(SignsRequestsWithAWS):
self,
aws_role_name: str,
ssl_verify: bool | str | None = None,
aws_region_name: str | None = None,
aws_sts_endpoint: str | None = None,
) -> bool:
"""
Check if the current environment is already running as the target IAM role.
@ -1000,14 +998,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
import boto3
with tracer.trace("boto3.client(sts).get_caller_identity"):
sts_client: Final = boto3.client(
"sts",
**self._build_sts_client_kwargs(
aws_sts_endpoint=aws_sts_endpoint,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
),
)
sts_client: Final = boto3.client("sts", verify=self._get_ssl_verify(ssl_verify))
identity: Final = sts_client.get_caller_identity()
caller_arn: Final = identity.get("Arn", "")
@ -1298,12 +1289,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
caller's key skips both. ``aws_session_name`` defaults inside this fetcher rather than in
``get_credentials`` so the cache key stays stable when the caller does not supply one.
"""
if self._is_already_running_as_role(
aws_role_name,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
):
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
@ -1429,12 +1415,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
# partition, and role name). This avoids silently using the
# wrong identity when there is a genuine trust-policy or
# permission misconfiguration.
if self._is_already_running_as_role(
aws_role_name,
ssl_verify=ssl_verify,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
):
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
verbose_logger.warning(
"AssumeRole failed for %s (%s). "
"Caller is already running as this role; "

View file

@ -109,8 +109,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
self._read_configured_max_tokens: Final = read_configured_max_tokens
@classmethod
def get_config(cls, *, model: str | None = None) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict
return {key: value for key, value in super().get_config(model=model).items() if key != "max_tokens"}
def get_config(
cls, *, model: str | None = None, configured_max_output_tokens: int | None = None
) -> dict[str, object]: # mutable-ok: overrides BaseConfig's dict
return {
key: value
for key, value in super()
.get_config(model=model, configured_max_output_tokens=configured_max_output_tokens)
.items()
if key != "max_tokens"
}
@property
def custom_llm_provider(self) -> str | None:

View file

@ -9,6 +9,7 @@ import httpx
from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import header_value
from litellm.types.llms.bedrock import (
CohereEmbeddingRequest,
CohereEmbeddingRequestWithModel,
@ -23,6 +24,11 @@ class _SupportsEncode(Protocol):
def encode(self, text: str, /) -> Sized: ...
def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None:
count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count")
return int(count) if count is not None and count.isdigit() else None
class CohereEmbeddingConfig:
"""
Reference: https://docs.cohere.com/v2/reference/embed
@ -76,32 +82,30 @@ class CohereEmbeddingConfig:
) -> CohereEmbeddingRequestWithModel:
return self._transform_request(model, input, inference_params)
def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage:
input_tokens = 0
def _calculate_usage(
self, input: list[str], encoding: _SupportsEncode, meta: dict, billed_input_tokens: int | None = None
) -> Usage:
text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens")
image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images")
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
if image_tokens is None and text_tokens is None:
for text in input:
input_tokens += len(encoding.encode(text))
else:
prompt_tokens_details = PromptTokensDetailsWrapper(
image_tokens=image_tokens,
text_tokens=text_tokens,
counted_tokens: Final = (
billed_input_tokens
if billed_input_tokens is not None
else sum(len(encoding.encode(text)) for text in input)
)
if image_tokens:
input_tokens += image_tokens
if text_tokens:
input_tokens += text_tokens
return Usage(prompt_tokens=counted_tokens, completion_tokens=0, total_tokens=counted_tokens)
input_tokens: Final = (image_tokens or 0) + (text_tokens or 0)
return Usage(
prompt_tokens=input_tokens,
completion_tokens=0,
total_tokens=input_tokens,
prompt_tokens_details=prompt_tokens_details,
prompt_tokens_details=PromptTokensDetailsWrapper(
image_tokens=image_tokens,
text_tokens=text_tokens,
),
)
def _transform_response(
@ -129,6 +133,7 @@ class CohereEmbeddingConfig:
model=model,
encoding=encoding,
input=input,
billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None,
)
def transform_response(
@ -151,6 +156,7 @@ class CohereEmbeddingConfig:
model: str,
encoding: _SupportsEncode,
input: list,
billed_input_tokens: int | None = None,
) -> EmbeddingResponse:
"""
Parse a Cohere embed response body into an OpenAI-style EmbeddingResponse.
@ -195,7 +201,7 @@ class CohereEmbeddingConfig:
setattr(
model_response,
"usage",
self._calculate_usage(input, encoding, response_json.get("meta", {})),
self._calculate_usage(input, encoding, response_json.get("meta", {}), billed_input_tokens),
)
return model_response

View file

@ -47,6 +47,12 @@ from litellm.anthropic_beta_headers_manager import (
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import (
every_claimed_tool_call,
finalize_refused_responses_turn,
is_refused_item,
refused_tool_call_identifiers,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
@ -56,11 +62,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
SUBTITLE_RESPONSE_FORMATS,
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
qualify_agentic_followup_model,
set_provider_response_headers_in_hidden_params,
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
bill_first_round_on_failure,
run_chat_completion_agentic_hooks,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
@ -150,6 +156,7 @@ from litellm.types.extraction import ExtractionResponse
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
from litellm.types.integrations.custom_logger import (
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopLimitError,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
@ -6005,7 +6012,7 @@ class BaseLLMHTTPHandler:
@staticmethod
def _check_agentic_loop_safety(
tool_calls: object,
fingerprints: list[str],
fingerprints: Sequence[str],
depth: int,
max_loops: int,
model: str,
@ -6024,7 +6031,7 @@ class BaseLLMHTTPHandler:
if fingerprint in fingerprints:
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@staticmethod
@ -6034,26 +6041,6 @@ class BaseLLMHTTPHandler:
except Exception:
return str(tools)
@staticmethod
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
"""
Collect the ids and names of the tool calls a safety rail just refused.
Callbacks hand back either a bare list of tool calls or a dict wrapping
that list under ``tool_calls``, and both the anthropic and responses
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
"""
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
if not isinstance(calls, list):
return frozenset(), frozenset()
dict_calls: Final = (call for call in calls if isinstance(call, dict))
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
ids: Final = frozenset(
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
)
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
return ids, names
@staticmethod
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
"""
@ -6064,22 +6051,13 @@ class BaseLLMHTTPHandler:
share a name with a refused one. The name is only consulted for tool call
shapes that arrive without an id.
"""
if not isinstance(block, dict) or block.get("type") != "tool_use":
return False
block_id: Final = block.get("id")
if isinstance(block_id, str) and refused_ids:
return block_id in refused_ids
return block.get("name") in refused_names
return is_refused_item(block, "tool_use", "id", refused_ids, refused_names)
@staticmethod
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
"""
Whether a refused rerun can still be answered with a finalized turn.
Only the anthropic messages surface can. The responses surface carries a
pydantic model the finalizer does not rewrite, so it keeps raising, which
is what every surface did before this path learned to end the turn.
The messages and responses call sites pass ``stream=False``, because
interception converts an intercepted stream to non-streaming before the
loop runs and rebuilds the SSE stream from the finalized turn
@ -6088,7 +6066,7 @@ class BaseLLMHTTPHandler:
finalized turn would reach the client as a second message rather than
as a replacement.
"""
return not stream and api_surface == "anthropic_messages"
return not stream and api_surface in ("anthropic_messages", "responses")
@staticmethod
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
@ -6105,10 +6083,12 @@ class BaseLLMHTTPHandler:
one is still in the response the stop reason stays ``tool_use`` so the
client knows to answer it.
"""
if isinstance(response, ResponsesAPIResponse):
return finalize_refused_responses_turn(response, tool_calls)
if not isinstance(response, dict):
return response
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
finalized: Final = dict(response)
content: Final = finalized.get("content")
if isinstance(content, list):
@ -6135,7 +6115,7 @@ class BaseLLMHTTPHandler:
kwargs: dict,
depth: int,
max_loops: int,
fingerprints: list[str],
fingerprints: Sequence[str],
fingerprint: str,
stream: bool = False,
callback: Optional["CustomLogger"] = None,
@ -6176,7 +6156,7 @@ class BaseLLMHTTPHandler:
kwargs_for_followup.update(patch.kwargs)
kwargs_for_followup["_agentic_loop_depth"] = depth + 1
kwargs_for_followup["max_agentic_loops"] = max_loops
kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint]
response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate(
**{
@ -6215,7 +6195,7 @@ class BaseLLMHTTPHandler:
kwargs: dict,
depth: int,
max_loops: int,
fingerprints: list[str],
fingerprints: Sequence[str],
fingerprint: str,
callback: Optional["CustomLogger"] = None,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
@ -6336,67 +6316,6 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
)
async def _execute_chat_completion_agentic_plan(
self,
plan: AgenticLoopPlan,
model: str,
messages: list[dict],
optional_params: dict,
kwargs: dict,
custom_llm_provider: str,
depth: int,
max_loops: int,
fingerprints: list[str],
fingerprint: str,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
) -> ModelResponse | CustomStreamWrapper:
patch: Final = plan.request_patch or AgenticLoopRequestPatch()
if patch.messages is None:
raise ValueError("Agentic loop plan missing patched messages")
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
optional_params_for_followup: Final = dict(optional_params)
optional_params_for_followup.update(patch.optional_params)
if patch.tools is not None:
optional_params_for_followup["tools"] = patch.tools
internal_params: Final = {
"_websearch_interception",
"acompletion",
"litellm_logging_obj",
"custom_llm_provider",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
kwargs_for_followup: Final = build_agentic_followup_kwargs(
request_kwargs=MappingProxyType(
{
k: v
for k, v in kwargs.items()
if not k.startswith("_websearch_interception")
and not k.startswith("_compression_interception")
and k not in internal_params
}
),
patch_kwargs=patch.kwargs,
request_params=frozenset((*optional_params_for_followup, "model", "messages")),
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
with internal_sub_call(logging_obj.litellm_call_id if logging_obj is not None else None):
response: Final = await litellm.acompletion(
model=full_model_name,
messages=patch.messages,
**optional_params_for_followup,
**kwargs_for_followup,
)
return response
def _maybe_wrap_in_fake_stream(
self,
response: _ResponseT,
@ -6513,26 +6432,38 @@ class BaseLLMHTTPHandler:
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
logging_obj.litellm_call_id,
model,
str(e),
)
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__)
return self._maybe_wrap_in_fake_stream(
self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
self._finalize_refused_agentic_response(
response=response,
tool_calls=await every_claimed_tool_call(
tool_calls,
(
other.async_should_run_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=hook_kwargs,
)
for other in callbacks
if isinstance(other, CustomLogger) and other is not callback
),
),
),
logging_obj,
api_surface,
)
try:
kwargs_with_provider = hook_kwargs.copy()
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
build_plan_overridden = (
callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
)
if not build_plan_overridden:
agentic_result: object = await callback.async_run_agentic_loop(
kwargs_with_provider = {**hook_kwargs, "custom_llm_provider": custom_llm_provider}
build_plan_overridden = (
callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan
)
if not build_plan_overridden:
agentic_result: object = await bill_first_round_on_failure(
callback.async_run_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
@ -6542,10 +6473,15 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface)
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface)
plan = await callback.async_build_agentic_loop_plan(
plan = await bill_first_round_on_failure(
callback.async_build_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
@ -6555,30 +6491,35 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
if plan.response_override is not None:
return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface)
if plan.terminate:
verbose_logger.debug(
"Agentic loop terminated by callback=%s reason=%s",
callback.__class__.__name__,
plan.stop_reason,
)
return self._maybe_wrap_in_fake_stream(
await callback.async_post_agentic_loop_response_hook(
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
plan=plan,
kwargs=kwargs_with_provider,
),
logging_obj,
api_surface,
)
if not plan.run_agentic_loop:
continue
if plan.response_override is not None:
return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface)
if plan.terminate:
verbose_logger.debug(
"Agentic loop terminated by callback=%s reason=%s",
callback.__class__.__name__,
plan.stop_reason,
)
return self._maybe_wrap_in_fake_stream(
await callback.async_post_agentic_loop_response_hook(
response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
plan=plan,
kwargs=kwargs_with_provider,
),
logging_obj,
api_surface,
)
if not plan.run_agentic_loop:
continue
if api_surface == "responses":
return await self._execute_responses_agentic_plan(
if api_surface == "responses":
return await bill_first_round_on_failure(
self._execute_responses_agentic_plan(
plan=plan,
model=model,
response_api_optional_request_params=anthropic_messages_optional_request_params,
@ -6589,10 +6530,15 @@ class BaseLLMHTTPHandler:
fingerprints=fingerprints,
fingerprint=fingerprint,
callback=callback,
)
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
return self._maybe_wrap_in_fake_stream(
await self._execute_anthropic_agentic_plan(
return self._maybe_wrap_in_fake_stream(
await bill_first_round_on_failure(
self._execute_anthropic_agentic_plan(
plan=plan,
model=model,
messages=messages,
@ -6606,17 +6552,13 @@ class BaseLLMHTTPHandler:
stream=stream,
callback=callback,
),
logging_obj,
api_surface,
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in agentic completion hooks [call_id=%s model=%s]: %s",
_call_id,
model,
str(e),
)
logging_obj=logging_obj,
response=response,
stream=stream,
),
logging_obj,
api_surface,
)
# Check if we need to convert response to fake stream
# This happens when:
@ -6648,111 +6590,23 @@ class BaseLLMHTTPHandler:
Returns the response from agentic loop, or None if no hook runs.
"""
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
tools: Final = optional_params.get("tools", [])
depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
for callback in callbacks:
if not isinstance(callback, CustomLogger):
continue
if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
continue
should_run: bool = False
tool_calls: object = None
try:
(
should_run,
tool_calls,
) = await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
except Exception as e:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s",
str(e),
)
continue
if not should_run:
continue
# Safety guards must run OUTSIDE the callback try/except — they are
# bounded-loop / cycle-break rails that must propagate to the caller.
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
agentic_response: Final = await bill_first_round_on_failure(
run_chat_completion_agentic_hooks(
response=response,
model=model,
)
try:
kwargs_with_provider = kwargs.copy() if kwargs else {}
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
build_plan_overridden = (
callback.__class__.async_build_chat_completion_agentic_loop_plan
is not CustomLogger.async_build_chat_completion_agentic_loop_plan
)
if not build_plan_overridden:
return await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
plan = await callback.async_build_chat_completion_agentic_loop_plan(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
if plan.response_override is not None:
return plan.response_override
if plan.terminate:
verbose_logger.debug(
"Agentic chat loop terminated by callback=%s reason=%s",
callback.__class__.__name__,
plan.stop_reason,
)
return response
if not plan.run_agentic_loop:
continue
return await self._execute_chat_completion_agentic_plan(
plan=plan,
model=model,
messages=messages,
optional_params=optional_params,
kwargs=kwargs_with_provider,
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
depth=depth,
max_loops=max_loops,
fingerprints=fingerprints,
fingerprint=fingerprint,
)
except Exception as e:
verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e)
messages=messages,
optional_params=optional_params,
kwargs=kwargs or {},
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
stream=stream,
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
if agentic_response is not None:
return agentic_response
# Check if we need to convert response to fake stream for chat completions
# This happens when:

View file

@ -395,7 +395,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
return "databricks"
@classmethod
def get_config(cls, *, model: str | None = None):
def get_config(cls, *, model: str | None = None, configured_max_output_tokens: int | None = None):
return super().get_config()
def get_json_schema_from_pydantic_object(

View file

@ -30,6 +30,10 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.exceptions import UnsupportedParamsError
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
bill_first_round_on_failure,
run_chat_completion_agentic_hooks,
)
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
@ -665,67 +669,23 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
Returns the response from agentic loop, or None if no hook runs.
"""
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
# Avoid logging full callback objects to prevent leaking sensitive data
verbose_logger.debug("LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks))
tools: Final = optional_params.get("tools", [])
# Avoid logging full tools payloads; they may contain sensitive parameters
verbose_logger.debug(
"LiteLLM.AgenticHooks: tools_count=%s",
len(tools) if isinstance(tools, list) else 1 if tools else 0,
if not isinstance(response, ModelResponse):
return None
return await bill_first_round_on_failure(
run_chat_completion_agentic_hooks(
response=response,
model=model,
messages=messages,
optional_params=optional_params,
kwargs=litellm_params or {},
logging_obj=logging_obj,
custom_llm_provider=litellm_params.get("custom_llm_provider", "openai"),
stream=stream,
),
logging_obj=logging_obj,
response=response,
stream=stream,
)
# Get custom_llm_provider from litellm_params
custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai")
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
# Check if the callback has the chat completion agentic loop methods
if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
continue
# First: Check if agentic loop should run (using chat completion method)
(
should_run,
tool_calls,
) = await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=litellm_params,
)
if should_run:
# Second: Execute agentic loop
kwargs_with_provider = litellm_params.copy() if litellm_params else {}
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
# For OpenAI Chat Completions, use the chat completion agentic loop method
agentic_response: object = await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
# First hook that runs agentic loop wins
return agentic_response
except Exception as e:
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e
)
return None
def mock_streaming(
self,
@ -1076,8 +1036,28 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
hidden_params={"headers": headers},
_response_headers=headers,
)
except openai.UnprocessableEntityError as e:
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
except Exception as e:
exception_response: object = getattr(e, "response", None)
status_code = getattr(e, "status_code", 500)
exception_body = getattr(e, "body", None)
error_headers = getattr(e, "headers", None)
if error_headers is None and exception_response:
error_headers = getattr(exception_response, "headers", None)
message = getattr(e, "message", str(e))
# Call agentic completion hooks (e.g., for websearch_interception)
raise OpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
else:
agentic_response = await self._call_agentic_completion_hooks_openai(
response=final_response_obj,
model=model,
@ -1100,29 +1080,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
return final_response_obj
except openai.UnprocessableEntityError as e:
## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
if litellm.drop_params is True or drop_params is True:
data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True)
else:
raise e
# e.message
except Exception as e:
exception_response: object = getattr(e, "response", None)
status_code = getattr(e, "status_code", 500)
exception_body = getattr(e, "body", None)
error_headers = getattr(e, "headers", None)
if error_headers is None and exception_response:
error_headers = getattr(exception_response, "headers", None)
message = getattr(e, "message", str(e))
raise OpenAIError(
status_code=status_code,
message=message,
response=upstream_error_response(e),
headers=error_headers,
body=exception_body,
)
def streaming(
self,

View file

@ -302,5 +302,14 @@
"preserve_cache_control": true
},
"supported_endpoints": ["/v1/chat/completions"]
},
"wally": {
"base_url": "https://inference.runanywhere.ai/v1",
"api_key_env": "WALLY_API_KEY",
"api_base_env": "WALLY_API_BASE",
"provider_info": {
"supports_function_calling": true,
"supports_reasoning": true
}
}
}

View file

@ -81,11 +81,13 @@ from litellm.litellm_core_utils.audio_utils.utils import (
get_audio_file_for_health_check,
)
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
bill_first_round_on_failure,
maybe_run_chat_completion_agentic_loop,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
AWS_CREDENTIAL_KWARGS_KEYS,
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@ -111,6 +113,7 @@ from litellm.litellm_core_utils.mock_functions import (
mock_embedding,
mock_image_generation,
)
from litellm.litellm_core_utils.per_call_pricing import per_call_model_info
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export
)
@ -167,7 +170,6 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s
from litellm.types.utils import (
AgenticLoopParams,
BaseUrlRoutingHint,
CustomPricingLiteLLMParams,
ModelResponseStream,
ProviderRoutingHints,
RawRequestTypedDict,
@ -954,26 +956,31 @@ async def acompletion(
# no equivalent: every provider already funnels through one shared
# handler where the loop is dispatched.
if isinstance(response, litellm.ModelResponse):
looped: Final = await maybe_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
optional_params={
k: v
for k, v in completion_kwargs.items()
if v is not None
and k
not in (
"model",
"messages",
"stream",
"acompletion",
"deployment_id",
)
},
kwargs=kwargs,
looped: Final = await bill_first_round_on_failure(
maybe_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
optional_params={
k: v
for k, v in completion_kwargs.items()
if v is not None
and k
not in (
"model",
"messages",
"stream",
"acompletion",
"deployment_id",
)
},
kwargs=kwargs,
logging_obj=kwargs.get("litellm_logging_obj"),
custom_llm_provider=custom_llm_provider,
stream=bool(stream),
),
logging_obj=kwargs.get("litellm_logging_obj"),
custom_llm_provider=custom_llm_provider,
response=response,
stream=bool(stream),
)
if looped is not None:
@ -1323,7 +1330,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
)
from litellm.responses.bridge_check import (
bridge_reasoning_effort,
keep_registered_mode,
str_keyed_mapping,
)
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
@ -1429,46 +1435,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
return values
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None = None,
) -> dict:
"""Build a complete model cost entry from kwargs and model_info.
Collects all CustomPricingLiteLLMParams fields present in kwargs and
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
so that register_model() receives the full pricing configuration.
"""
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
for field_name in CustomPricingLiteLLMParams.model_fields:
value = kwargs.get(field_name)
if value is not None:
entry[field_name] = value
if model_info and isinstance(model_info, dict):
for key in ("mode", "supports_prompt_caching", "max_tokens"):
if key in model_info and model_info[key] is not None:
entry.setdefault(key, model_info[key])
return entry
def _get_router_deployment_id(kwargs: dict) -> str | None:
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key) or {}
if not isinstance(metadata, dict):
continue
deployment_model_info = metadata.get("model_info") or {}
if not isinstance(deployment_model_info, dict):
continue
deployment_id = deployment_model_info.get("id")
if deployment_id is not None:
return str(deployment_id)
return None
def _record_agentic_loop_params(
logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool
) -> None:
@ -1479,53 +1445,6 @@ def _record_agentic_loop_params(
)
def _register_custom_pricing_for_request(
model: str,
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None,
) -> None:
"""Register per-request custom pricing in litellm.model_cost.
Router-originated requests (identified by the deployment id the router puts
in metadata) get their full pricing registered under that unique id only;
the shared ``{provider}/{model}`` key receives the entry with pricing fields
stripped, mirroring Router._create_deployment. This keeps one deployment's
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
return ctx.metadata
@ -5787,9 +5706,9 @@ def completion(
### CUSTOM MODEL COST ###
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
### CUSTOM PROMPT TEMPLATE ###
initial_prompt_value: Final = kwargs.get("initial_prompt_value", None)
roles: Final = kwargs.get("roles", None)
@ -5987,19 +5906,7 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6269,6 +6176,7 @@ def completion(
*ANTHROPIC_WIF_KWARGS_KEYS,
*OPENAI_WIF_KWARGS_KEYS,
*OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
*AGENTIC_LOOP_CONTROL_KWARGS_KEYS,
"github_copilot_auth_type",
"github_copilot_host",
"github_copilot_user_session",
@ -7006,12 +6914,6 @@ def embedding(
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
### CUSTOM MODEL COST ###
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
openai_params: Final = [
"user",
"dimensions",
@ -7058,21 +6960,8 @@ def embedding(
**non_default_params,
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
logging.update_environment_variables(
model=model,
user=user,

File diff suppressed because it is too large Load diff

View file

@ -5250,6 +5250,32 @@
"generateContent": false
}
},
"wally": {
"display_name": "Wally (`wally`)",
"url": "https://docs.litellm.ai/docs/providers/wally",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"empiriolabs": {
"display_name": "EmpirioLabs (`empiriolabs`)",
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",

View file

@ -4796,11 +4796,14 @@ class UnregisteredJWTClientBehavior(str, enum.Enum):
- auto_register: Automatically create a new virtual key and mapping on first
encounter. The new key has no budget/model restrictions; admins can tighten
it later via /jwt_client/update.
- link_only: Map an eligible existing user key or return HTTP 403, without
creating a key. Proxy admins authenticate without a mapped key.
"""
FALLBACK_TEAM_MAPPING = "fallback_team_mapping"
REJECT = "reject"
AUTO_REGISTER = "auto_register"
LINK_ONLY = "link_only"
class JWTIssuerConfig(LiteLLMBaseModel):
@ -5038,7 +5041,12 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
"What to do when virtual_key_claim_field is set but the JWT claim value "
"has no registered mapping. 'fallback_team_mapping' (default): fall through "
"to team-based JWT auth. 'reject': return HTTP 403. "
"'auto_register': auto-create a virtual key and mapping on first encounter."
"'auto_register': auto-create a virtual key and mapping on first encounter. "
"'link_only': map an eligible existing user key or return HTTP 403, without creating a key. "
"link_only ignores auto_register_map_existing_key; user provisioning still follows user_id_upsert. "
"Existing-key selection ignores allowed_routes; JWTs inherit the selected key's permissions, "
"including management routes, with request-time key restrictions still enforced. "
"Unmapped proxy admins authenticate directly in auto_register and link_only modes."
),
)
auto_register_map_existing_key: bool = Field(
@ -5048,8 +5056,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
"field is the user_id_jwt_field or user_email_jwt_field, the JWT claim is mapped to a virtual key the "
"JWT-resolved user already owns instead of minting a new one. If the user owns several, the most recently created key in the "
"JWT-resolved team (or with no team when the JWT resolves none) is chosen among keys that never "
"expire, are not blocked, are not Admin UI session keys, were not minted by auto_register, and "
"have no allowed_routes or include llm_api_routes. Otherwise a new key is minted as usual."
"expire, are not blocked, are not Admin UI session keys, and were not minted by auto_register. "
"allowed_routes does not affect key selection. JWTs inherit the selected key's permissions, "
"including management routes; request-time key restrictions still apply. "
"Otherwise a new key is minted as usual."
),
)
routing_overrides: list[JWTRoutingOverride] | None = Field(

View file

@ -2,31 +2,22 @@ import asyncio
import json
from collections.abc import Sequence
from datetime import datetime
from typing import Final, Protocol
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
from litellm.proxy._types import LiteLLMRoutes
from litellm.proxy.auth.authorization import ReadScope, read_scope_bind_params
from litellm.proxy.utils import query_raw_within_statement_timeout
from litellm.types.llms.base import LiteLLMBaseModel
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
UNKNOWN_CALL_TYPE: Final = "Unknown"
INFO_ROUTES_JSON: Final = json.dumps(LiteLLMRoutes.info_routes.value)
class _SupportsQueryRaw(Protocol):
"""The single database operation the cache-activity queries issue."""
async def query_raw(self, query: str, *args: object) -> Sequence[object]: ...
class _SupportsRawQueryDb(Protocol):
"""A prisma client handle, narrowed to the raw-query surface used here."""
@property
def db(self) -> _SupportsQueryRaw: ...
class CacheActivityGroup(LiteLLMBaseModel):
call_type: str
api_requests: int
@ -174,7 +165,7 @@ def compute_totals(groups: Sequence[CacheActivityGroup]) -> CacheActivityTotals:
async def get_cache_activity(
prisma_client: _SupportsRawQueryDb,
prisma_client: "PrismaClient",
read_scope: ReadScope,
start_date: datetime,
end_date: datetime,
@ -185,14 +176,46 @@ async def get_cache_activity(
models_json: Final = json.dumps(list(models))
scope_params: Final = read_scope_bind_params(read_scope)
group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather(
prisma_client.db.query_raw(
GROUPS_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params
query_raw_within_statement_timeout(
prisma_client,
GROUPS_SQL,
start_date,
end_date,
key_aliases_json,
models_json,
INFO_ROUTES_JSON,
*scope_params,
what="Cache activity",
),
prisma_client.db.query_raw(
ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json, INFO_ROUTES_JSON, *scope_params
query_raw_within_statement_timeout(
prisma_client,
ERROR_BREAKDOWN_SQL,
start_date,
end_date,
key_aliases_json,
models_json,
INFO_ROUTES_JSON,
*scope_params,
what="Cache activity",
),
query_raw_within_statement_timeout(
prisma_client,
KEY_ALIAS_OPTIONS_SQL,
start_date,
end_date,
INFO_ROUTES_JSON,
*scope_params,
what="Cache activity",
),
query_raw_within_statement_timeout(
prisma_client,
MODEL_OPTIONS_SQL,
start_date,
end_date,
INFO_ROUTES_JSON,
*scope_params,
what="Cache activity",
),
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params),
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date, INFO_ROUTES_JSON, *scope_params),
)
groups: Final = _groups_adapter.validate_python(group_rows or [])
return CacheActivityResponse(

View file

@ -0,0 +1,30 @@
from typing import Final, TypeVar
from litellm.exceptions import InternalServerError, MidStreamFallbackError
_Raised = TypeVar("_Raised", bound=Exception)
def _provider_status_code(exc: Exception) -> int | None:
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
unwrapped = unwrapped.original_exception
match unwrapped:
case InternalServerError(provider_status_code=int() as provider_status):
return provider_status
case _:
return None
def _copy_carrying(exc: _Raised, attribute: str, value: int | str) -> _Raised:
answered: Final = type(exc).__new__(type(exc), *exc.args)
answered.__dict__.update({**exc.__dict__, attribute: value})
answered.__cause__ = exc.__cause__
answered.__context__ = exc.__context__
answered.__suppress_context__ = exc.__suppress_context__
return answered.with_traceback(exc.__traceback__)
def with_answered_status(exc: Exception) -> Exception:
provider_status_code: Final = _provider_status_code(exc)
return exc if provider_status_code is None else _copy_carrying(exc, "status_code", provider_status_code)

View file

@ -16,12 +16,14 @@ from litellm.anthropic_interface.exceptions import (
AnthropicExceptionMapping,
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
)
from litellm.proxy._types import *
from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
@ -48,6 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
)
from litellm.proxy.common_utils.token_count_log import rejected_token_count
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
from litellm.proxy.litellm_pre_call_utils import (
forwarded_anthropic_oauth_token,
should_auto_drop_params_for_agentic_cli,
@ -225,15 +228,18 @@ async def anthropic_response(
request_id=request.headers.get("x-request-id"),
)
return JSONResponse(status_code=e.status_code, content=body)
except Exception as e:
except Exception as raised:
answered: Final = with_answered_status(raised)
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data
user_api_key_dict=user_api_key_dict,
original_exception=answered,
request_data=base_llm_response_processor.data,
)
log_llm_api_exception(e, base_llm_response_processor.litellm_call_id)
log_llm_api_exception(answered, base_llm_response_processor.litellm_call_id)
if isinstance(e, ProxyException):
if isinstance(answered, ProxyException):
return _anthropic_error_json_response(
with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request
with_litellm_call_id(answered, base_llm_response_processor.litellm_call_id), request
)
# Extract model_id from request metadata (same as success path)
@ -247,23 +253,25 @@ async def anthropic_response(
call_id=base_llm_response_processor.litellm_call_id,
model_id=model_id,
version=version,
response_cost=0,
response_cost=(
failure_row_response_cost(base_llm_response_processor.data) if is_agentic_followup_failure(e) else 0
),
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
request_data=data,
timeout=getattr(e, "timeout", None),
timeout=getattr(answered, "timeout", None),
litellm_logging_obj=None,
)
if isinstance(e, HTTPException):
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
if isinstance(answered, HTTPException):
return _anthropic_error_json_response(proxy_exception_from_http_exception(answered, headers), request)
error_msg: Final = f"{e}"
error_msg: Final = f"{answered}"
return _anthropic_error_json_response(
ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
code=error_status_code(e, 500),
message=getattr(answered, "message", error_msg),
type=openai_error_type(answered, error_status_code(answered, 500)),
param=openai_error_param(answered),
code=error_status_code(answered, 500),
headers=headers,
),
request,
@ -383,8 +391,9 @@ async def count_tokens(
except ProxyException as e:
return _anthropic_error_json_response(e, request)
except Exception as e:
log_llm_api_exception(e, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
answered: Final = with_answered_status(e)
log_llm_api_exception(answered, litellm_call_id)
return _anthropic_error_json_response(proxy_exception_from_route_error(answered), request)
else:
return {
"input_tokens": token_response.total_tokens,

View file

@ -13,6 +13,7 @@ import re
import secrets
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from enum import Enum
from types import MappingProxyType
from typing import Final, NamedTuple, Protocol, Union, cast
@ -1576,10 +1577,14 @@ _JWT_AUTH_DISABLED_HINT = (
)
class _JWTKeyMappingFailure(Enum):
NO_ELIGIBLE_KEY = "JWT Key Mapping: No eligible existing virtual key for this identity. Access denied."
class _PendingAutoRegister(NamedTuple):
"""
Signal returned by ``_resolve_jwt_to_virtual_key`` when the JWT's claim is
unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER.
unmapped and ``unregistered_jwt_client_behavior`` is AUTO_REGISTER or LINK_ONLY.
The caller MUST run standard ``JWTAuthManager.auth_builder`` to apply RBAC,
scope mappings, ``custom_validate``, and ``user_allowed_email_domain``
@ -1593,23 +1598,22 @@ class _PendingAutoRegister(NamedTuple):
claim_value: str
cache_key: str
jwt_issuer: str | None = None
behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER
def _claim_identifies_user(jwt_handler: JWTHandler, claim_field: str, jwt_issuer: str | None) -> bool:
if not jwt_handler.litellm_jwtauth.auto_register_map_existing_key:
return False
if jwt_handler.litellm_jwtauth.is_user_identity_claim(claim_field, jwt_issuer):
return True
verbose_proxy_logger.warning(
"JWT Key Mapping (auto_register_map_existing_key): claim '%s' is not the user_id or user_email JWT field "
"and may be shared by several users, so a new key is minted instead of reusing one the user owns.",
"JWT Key Mapping: claim '%s' is not the user_id or user_email JWT field "
"and may be shared by several users, so existing user keys cannot be linked automatically.",
claim_field,
)
return False
async def _reusable_key_hash_for_user(prisma_client: PrismaClient, user_id: str, team_id: str | None) -> str | None:
key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_llm_api_key(user_id, team_id)
key: Final = await VerificationTokenRepository(prisma_client).find_newest_reusable_key(user_id, team_id)
return None if key is None else key.token
@ -1628,11 +1632,13 @@ async def _auto_register_jwt_mapping(
org_id: str | None = None,
end_user_id: str | None = None,
agent_id: str | None = None,
) -> UserAPIKeyAuth | None:
behavior: UnregisteredJWTClientBehavior = UnregisteredJWTClientBehavior.AUTO_REGISTER,
) -> UserAPIKeyAuth | _JWTKeyMappingFailure | None:
"""
Auto-register: create a new virtual key + mapping for an unrecognised JWT
claim value, or point the mapping at a key the resolved user already owns
when ``auto_register_map_existing_key`` is set. ``team_id`` and ``user_id``
when ``auto_register_map_existing_key`` is set. LINK_ONLY always searches
for an existing key and rejects when none is eligible. ``team_id`` and ``user_id``
must come from a successful ``JWTAuthManager.auth_builder`` run — they
encode the JWT identity AFTER
RBAC/scope/custom_validate/email-domain policy has been enforced. The key
@ -1650,11 +1656,21 @@ async def _auto_register_jwt_mapping(
generate_key_helper_fn,
)
link_only: Final = behavior == UnregisteredJWTClientBehavior.LINK_ONLY
reuse_existing: Final = link_only or jwt_handler.litellm_jwtauth.auto_register_map_existing_key
existing_token_hash: Final = (
await _reusable_key_hash_for_user(prisma_client, user_id, team_id)
if user_id is not None and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer)
await _reusable_key_hash_for_user(
prisma_client,
user_id,
team_id,
)
if reuse_existing
and user_id is not None
and _claim_identifies_user(jwt_handler, virtual_key_claim_field, jwt_issuer)
else None
)
if link_only and existing_token_hash is None:
return _JWTKeyMappingFailure.NO_ELIGIBLE_KEY
minted: Final = existing_token_hash is None
if existing_token_hash is not None:
token_hash = existing_token_hash
@ -1691,8 +1707,8 @@ async def _auto_register_jwt_mapping(
"jwt_claim_name": virtual_key_claim_field,
"jwt_claim_value": claim_value,
"token": token_hash,
"created_by": "auto_register",
"updated_by": "auto_register",
"created_by": behavior.value,
"updated_by": behavior.value,
}
)
except Exception as e:
@ -1704,8 +1720,8 @@ async def _auto_register_jwt_mapping(
# the DB and the cleartext is in memory on this request. Delete it
# so orphans don't accumulate under sustained concurrency.
verbose_proxy_logger.debug(
"JWT Key Mapping (auto_register): unique conflict on create — "
"deleting orphaned virtual key and fetching winner's mapping for %s='%s'.",
"JWT Key Mapping (%s): unique conflict on create, fetching winner's mapping for %s='%s'.",
behavior.value,
virtual_key_claim_field,
claim_value,
)
@ -1735,7 +1751,7 @@ async def _auto_register_jwt_mapping(
raise HTTPException(
status_code=503,
detail=(
"JWT Key Mapping: AUTO_REGISTER race resolution failed — "
f"JWT Key Mapping: {behavior.value.upper()} race resolution failed — "
"winner's mapping was concurrently removed. Retry the request."
),
)
@ -1749,7 +1765,8 @@ async def _auto_register_jwt_mapping(
)
verbose_proxy_logger.info(
"JWT Key Mapping (auto_register): %s virtual key for %s='%s'.",
"JWT Key Mapping (%s): %s virtual key for %s='%s'.",
behavior.value,
"created new" if minted else "mapped existing",
virtual_key_claim_field,
claim_value,
@ -1821,7 +1838,7 @@ async def _resolve_jwt_to_virtual_key(
- ``UserAPIKeyAuth``: a resolved virtual key (cache hit or DB hit). The
caller may use this directly; JWT policy has been enforced previously
(at key-creation time or, for cached results, before caching).
- ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER.
- ``_PendingAutoRegister``: claim is unmapped and behavior is AUTO_REGISTER or LINK_ONLY.
The caller MUST run ``JWTAuthManager.auth_builder`` to enforce JWT
policy (RBAC, scope, custom_validate, email-domain), then invoke
``_auto_register_jwt_mapping`` with the validated team_id/user_id.
@ -1829,7 +1846,7 @@ async def _resolve_jwt_to_virtual_key(
The caller falls through to standard team-based JWT auth (which itself
enforces full JWT policy via auth_builder).
- Raises HTTPException: REJECT policy hit, missing claim under
REJECT/AUTO_REGISTER, or other policy violations.
REJECT/AUTO_REGISTER/LINK_ONLY, or other policy violations.
"""
raw_issuer: Final = jwt_claims.get(JWTHandler.LITELLM_JWT_ISSUER_CLAIM)
normalized_issuer: Final = raw_issuer if isinstance(raw_issuer, str) else None
@ -1856,6 +1873,7 @@ async def _resolve_jwt_to_virtual_key(
if behavior in (
UnregisteredJWTClientBehavior.REJECT,
UnregisteredJWTClientBehavior.AUTO_REGISTER,
UnregisteredJWTClientBehavior.LINK_ONLY,
):
raise HTTPException(
status_code=403,
@ -1887,7 +1905,7 @@ async def _resolve_jwt_to_virtual_key(
status_code=403,
detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.",
)
if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER:
if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY):
# Stale sentinel written under a prior fallback_team_mapping config —
# evict it and defer auto-register to after auth_builder runs. Raise
# the same 500 as the fresh-path AUTO_REGISTER branch when there is
@ -1897,7 +1915,7 @@ async def _resolve_jwt_to_virtual_key(
raise HTTPException(
status_code=500,
detail=(
"JWT Key Mapping: AUTO_REGISTER requires a database connection. "
f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. "
"Configure a database or change unregistered_jwt_client_behavior."
),
)
@ -1907,6 +1925,7 @@ async def _resolve_jwt_to_virtual_key(
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
behavior=behavior,
)
return None
elif cached_mapping is not None:
@ -1967,12 +1986,12 @@ async def _resolve_jwt_to_virtual_key(
detail=f"JWT Key Mapping: No registered mapping for {virtual_key_claim_field}='{claim_value}'. Access denied.",
)
if behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER:
if behavior in (UnregisteredJWTClientBehavior.AUTO_REGISTER, UnregisteredJWTClientBehavior.LINK_ONLY):
if prisma_client is None:
raise HTTPException(
status_code=500,
detail=(
"JWT Key Mapping: AUTO_REGISTER requires a database connection. "
f"JWT Key Mapping: {behavior.value.upper()} requires a database connection. "
"Configure a database or change unregistered_jwt_client_behavior."
),
)
@ -1985,6 +2004,7 @@ async def _resolve_jwt_to_virtual_key(
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
behavior=behavior,
)
# FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the
@ -2477,10 +2497,13 @@ async def user_api_key_auth_builder(
org_id=org_id,
end_user_id=end_user_id,
agent_id=agent_id,
behavior=pending_auto_register.behavior,
)
if pending_auto_register is not None and prisma_client is not None
else None
)
if isinstance(auto_registered, _JWTKeyMappingFailure):
raise HTTPException(status_code=403, detail=auto_registered.value)
if auto_registered is not None:
auto_registered.jwt_claims = jwt_claims
auto_registered.user_email = user_email
@ -2496,7 +2519,13 @@ async def user_api_key_auth_builder(
falls_through_to_key_checks: Final = (
auto_registered is not None
and jwt_handler.litellm_jwtauth.auto_register_map_existing_key
and (
jwt_handler.litellm_jwtauth.auto_register_map_existing_key
or (
pending_auto_register is not None
and pending_auto_register.behavior == UnregisteredJWTClientBehavior.LINK_ONLY
)
)
and master_key is not None
)
if not falls_through_to_key_checks:

View file

@ -55,6 +55,7 @@ from litellm.constants import (
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
@ -132,6 +133,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
endpoint_unsupported_by_model,
@ -272,6 +274,7 @@ if TYPE_CHECKING:
ProxyConfig = _ProxyConfig
else:
ProxyConfig = Any
from litellm.proxy.anthropic_endpoints.answered_status import with_answered_status
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
AnthropicStreamModelRestamper,
)
@ -1272,6 +1275,21 @@ def _sse_error_frames(error_obj: Mapping[str, object]) -> tuple[str, str]:
return f"data: {json.dumps({'error': error_obj}, default=str)}\n\n", "data: [DONE]\n\n"
def _openai_late_failure_frames(exc: Exception) -> tuple[str, ...]:
_, error_obj = sse_error_payload(exc)
return _sse_error_frames(error_obj)
def _anthropic_late_failure_frames(exc: Exception) -> tuple[str, ...]:
answered: Final = proxy_exception_from_route_error(with_answered_status(exc))
return (
anthropic_error_sse_frame(
status_code=error_status_code(answered, status.HTTP_500_INTERNAL_SERVER_ERROR),
raw_message=answered.message,
),
)
def _sse_stream_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
"""`headers` plus the two that stop reverse proxies from buffering SSE (issue #28384)."""
return MappingProxyType({**headers, **_TTFT_KEEPALIVE_HEADERS})
@ -1520,6 +1538,7 @@ async def open_sse_before_first_byte(
ping_interval_seconds: float | str | None,
media_type: str = "text/event-stream",
on_late_failure: Callable[[Exception], Awaitable[HTTPException | None]] | None = None,
late_failure_frames: Callable[[Exception], tuple[str, ...]] = _openai_late_failure_frames,
) -> _LateResponseT | StreamingResponse:
"""Write SSE keepalive comments while the upstream LLM call is still in flight.
@ -1565,8 +1584,7 @@ async def open_sse_before_first_byte(
# would never fire and the failure would go unaudited. The hook
# also gets to sanitize what reaches the client, by returning or
# raising a replacement, so its answer decides the frame.
_, error_obj = sse_error_payload(await _sanitized_late_failure(exc, on_late_failure))
for frame in _sse_error_frames(error_obj):
for frame in late_failure_frames(await _sanitized_late_failure(exc, on_late_failure)):
yield frame.encode()
return
async for chunk in _relay_late_response(produced):
@ -2982,13 +3000,14 @@ class ProxyBaseLLMRequestProcessing:
idle. See ``open_sse_before_first_byte``; unwrapped unless an operator sets
``litellm_settings.sse_keepalive_ping_interval_seconds``.
"""
answers_as_anthropic: Final = route_type == "anthropic_messages"
async def _audit_late_failure(exc: Exception) -> HTTPException | None:
# Once a keepalive is on the wire this can no longer raise, so the
# caller's `except` never runs its own post_call_failure_hook.
return await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
original_exception=with_answered_status(exc) if answers_as_anthropic else exc,
request_data=self.data,
dispatched_call_type=self.route_type,
)
@ -3017,6 +3036,7 @@ class ProxyBaseLLMRequestProcessing:
),
ping_interval_seconds=ttft_keepalive_interval(self.data, llm_router),
on_late_failure=_audit_late_failure,
late_failure_frames=_anthropic_late_failure_frames if answers_as_anthropic else _openai_late_failure_frames,
)
async def _process_llm_request(
@ -4384,6 +4404,7 @@ class ProxyBaseLLMRequestProcessing:
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
log_llm_api_exception(e, self.litellm_call_id)
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
@ -4420,7 +4441,9 @@ class ProxyBaseLLMRequestProcessing:
model_id=model_id,
version=version,
response_cost=(
_litellm_logging_obj.finalized_cost.amount
failure_row_response_cost(self.data)
if failed_in_agentic_follow_up
else _litellm_logging_obj.finalized_cost.amount
if isinstance(_litellm_logging_obj, LiteLLMLoggingObj)
and _litellm_logging_obj.finalized_cost is not None
else 0
@ -4641,6 +4664,7 @@ class ProxyBaseLLMRequestProcessing:
request: Request | None = None,
flush_tail: Callable[[], bytes] | None = None,
seal_open_frame: Callable[[bytes], str] | None = None,
answer_status: Callable[[Exception], Exception] | None = None,
) -> AsyncGenerator[str, None]:
"""
Shared streaming data generator: runs proxy iterator hook, per-chunk hook,
@ -4656,6 +4680,9 @@ class ProxyBaseLLMRequestProcessing:
relays raw upstream bytes, so an upstream that hangs up mid-frame leaves the
client inside an open frame, where an error frame would be swallowed or
misparsed instead of raised.
``answer_status`` gives a failure the status the route answers it with
before the failure hook records it.
"""
verbose_proxy_logger.debug("inside generator")
# Resolve per-stream (not per-chunk) whether the heavy per-chunk path
@ -4752,6 +4779,7 @@ class ProxyBaseLLMRequestProcessing:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e
)
answered: Final = e if answer_status is None else answer_status(e)
already_reported: Final = (
isinstance(stream_logging_obj, LiteLLMLoggingObj)
and not stream_logging_obj.claim_stream_failure_report()
@ -4761,24 +4789,25 @@ class ProxyBaseLLMRequestProcessing:
if already_reported
else await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
original_exception=answered,
request_data=request_data,
)
)
if transformed_exception is not None:
e = transformed_exception
failure: Final = answered if transformed_exception is None else transformed_exception
verbose_proxy_logger.debug(
"An error occurred: %s",
e,
failure,
)
if isinstance(e, HTTPException):
raise e
stream_error_status: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR)
if isinstance(failure, HTTPException):
if failure is e:
raise
raise failure from e
stream_error_status: Final = error_status_code(failure, status.HTTP_500_INTERNAL_SERVER_ERROR)
proxy_exception: Final = ProxyException(
message=redact_internal_details_from_client_message(getattr(e, "message", str(e))),
type=openai_error_type(e, stream_error_status),
param=openai_error_param(e),
message=redact_internal_details_from_client_message(getattr(failure, "message", str(failure))),
type=openai_error_type(failure, stream_error_status),
param=openai_error_param(failure),
code=stream_error_status,
)
stream_completed = True
@ -4834,6 +4863,7 @@ class ProxyBaseLLMRequestProcessing:
request=request,
flush_tail=None if restamper is None else restamper.flush,
seal_open_frame=seal_open_sse_frame,
answer_status=with_answered_status,
)
@overload

View file

@ -0,0 +1,90 @@
from collections.abc import Mapping
from typing import Final
from litellm.secret_managers.main import str_to_bool
FIPS_MODE_SETTING: Final = "fips_mode"
_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
class FipsModeError(Exception):
pass
def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None:
"""general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including
a key with no value."""
if FIPS_MODE_SETTING not in general_settings:
return False
value: Final = general_settings[FIPS_MODE_SETTING]
if isinstance(value, bool):
return value
return str_to_bool(value) if isinstance(value, str) else None
def openssl_enforces_fips() -> bool:
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5."""
try:
import _hashlib
if _hashlib.get_fips_mode() != 1:
return False
except (ImportError, AttributeError, ValueError):
return False
try:
_hashlib.new("md5", b"", usedforsecurity=True)
except ValueError:
return True
return False
def cryptography_enforces_fips() -> bool:
"""The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256."""
try:
from cryptography.exceptions import InternalError, UnsupportedAlgorithm
from cryptography.hazmat.bindings._rust import openssl as rust_openssl
from cryptography.hazmat.primitives import hashes
except ImportError:
return False
try:
if not rust_openssl.is_fips_enabled():
return False
hashes.Hash(hashes.SHA256()).finalize()
except (AttributeError, InternalError, UnsupportedAlgorithm):
return False
return True
def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None:
"""Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the
cryptography library's OpenSSL does not enforce FIPS. None when it may start."""
setting: Final = _fips_mode_setting(general_settings)
if setting is None:
value: Final = general_settings.get(FIPS_MODE_SETTING)
if value is None:
return (
f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. "
"Set it to true or false, or remove it."
)
return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it."
if not setting:
return None
if not openssl_enforces_fips():
return (
f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS "
f"provider active and default_properties = fips=yes, or turn {_SETTING} off."
)
if not cryptography_enforces_fips():
return (
f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it "
"against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn "
f"{_SETTING} off."
)
return None
def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
"""Raise FipsModeError with the fips_mode_refusal reason, if there is one."""
refusal: Final = fips_mode_refusal(general_settings)
if refusal is not None:
raise FipsModeError(refusal)

View file

@ -32,6 +32,7 @@ class _ResolvedDeprecation:
litellm_model: str | None
litellm_provider: str | None
copied_from_another_row: bool = False
successor_model: str | None = None
def _parse_deprecation_date(raw_value: object) -> date | None:
@ -58,10 +59,11 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
entry: Final = litellm.model_cost.get(model_key)
if not isinstance(entry, Mapping):
return None
parsed: Final = _parse_deprecation_date(entry.get("deprecation_date"))
fields: Final[Mapping[str, object]] = entry
parsed: Final = _parse_deprecation_date(fields.get("deprecation_date"))
if parsed is None:
return None
provider: Final = entry.get("litellm_provider")
provider: Final = fields.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=parsed,
litellm_model=model_key,
@ -69,6 +71,7 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
copied_from_another_row=isinstance(entry, AliasedCostEntry)
and "deprecation_date" not in entry.registered_fields
and model_key not in catalog,
successor_model=_successor_model(fields.get("successor_model")),
)
@ -127,15 +130,6 @@ def _resolve_deployment_deprecation(
litellm_params: Final = _mapping_field(deployment, "litellm_params")
model_key: Final = _as_model_key(litellm_params.get("model"))
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
if override is not None:
override_provider: Final = model_info.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=override,
litellm_model=model_key,
litellm_provider=override_provider if isinstance(override_provider, str) else None,
)
provider: Final = _deployment_provider(litellm_params)
base_model: Final = _as_model_key(model_info.get("base_model"))
model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key
@ -150,7 +144,18 @@ def _resolve_deployment_deprecation(
if pick is not None
)
own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row)
return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
from_catalog: Final = min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
if override is None:
return from_catalog
override_provider: Final = model_info.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=override,
litellm_model=model_key,
litellm_provider=override_provider if isinstance(override_provider, str) else None,
successor_model=from_catalog.successor_model if from_catalog is not None else None,
)
def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus:
@ -182,7 +187,7 @@ def _build_info(
today: date,
warn_within_days: int,
catalog: Mapping[str, object],
) -> ModelDeprecationInfo | None:
) -> _DeploymentInfo | None:
model_name: Final = deployment.get("model_name")
if not isinstance(model_name, str) or not model_name:
return None
@ -192,21 +197,40 @@ def _build_info(
return None
days_until: Final = (resolved.deprecation_date - today).days
return ModelDeprecationInfo(
model_name=model_name,
litellm_model=resolved.litellm_model,
deprecation_date=resolved.deprecation_date,
days_until_deprecation=days_until,
status=_classify(days_until, warn_within_days),
litellm_provider=resolved.litellm_provider,
successor_model=_checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router),
configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
return _DeploymentInfo(
info=ModelDeprecationInfo(
model_name=model_name,
litellm_model=resolved.litellm_model,
deprecation_date=resolved.deprecation_date,
days_until_deprecation=days_until,
status=_classify(days_until, warn_within_days),
litellm_provider=resolved.litellm_provider,
successor_model=configured or resolved.successor_model,
),
configured_successor=configured is not None,
)
def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names"""
first: Final = group[0]
successors: Final = tuple(model.successor_model for model in group if model.successor_model is not None)
@dataclass(frozen=True, slots=True)
class _DeploymentInfo:
info: ModelDeprecationInfo
configured_successor: bool
def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names,
a successor configured in model_info ahead of one the cost map supplies"""
first: Final = group[0].info
tiers: Final = tuple(
tuple(
deployment.info.successor_model
for deployment in group
if deployment.configured_successor is configured and deployment.info.successor_model is not None
)
for configured in (True, False)
)
successors: Final = next((tier for tier in tiers if tier), ())
if not successors:
return first
if len(frozenset(successors)) > 1:
@ -220,13 +244,13 @@ def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
def _dedupe(
models: Sequence[ModelDeprecationInfo],
models: Sequence[_DeploymentInfo],
) -> tuple[ModelDeprecationInfo, ...]:
"""Report a model group carrying the same date on several deployments once"""
ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date))
ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date))
return tuple(
_merge_group(tuple(group))
for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date))
for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date))
)

View file

@ -920,8 +920,6 @@ def update_db_credential(
# update model info
if encrypted_credential.credential_info:
"""Update credential info"""
if "credential_info" not in merged_credential.credential_info:
merged_credential.credential_info = {}
merged_credential.credential_info.update(encrypted_credential.credential_info)
return merged_credential

View file

@ -34,6 +34,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014"
_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout"
def exception_chain(e: BaseException) -> Iterator[BaseException]:
@ -302,6 +304,17 @@ class PrismaDBExceptionHandler:
return True
return _LOCK_TIMEOUT_PHRASE in str(e).lower()
@staticmethod
def is_statement_timeout_error(e: Exception) -> bool:
"""True iff Postgres cancelled a statement for exceeding ``statement_timeout`` (SQLSTATE 57014)."""
import prisma
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if PrismaDBExceptionHandler.postgres_sqlstate(e) == _STATEMENT_TIMEOUT_SQLSTATE:
return True
return _STATEMENT_TIMEOUT_PHRASE in str(e).lower()
@staticmethod
def postgres_sqlstate(e: Exception) -> str | None:
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""

View file

@ -22,7 +22,7 @@ import json
import time
from collections import Counter, OrderedDict
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeGuard
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard
from urllib.parse import urlparse
import httpx
@ -507,6 +507,8 @@ class _CompressionResult:
class CompresrGuardrail(CustomGuardrail):
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({COMPRESR_RETRIEVE_TOOL_NAME})
def __init__(
self,
api_base: str | None = None,

View file

@ -55,7 +55,6 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
@ -64,7 +63,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.litellm_core_utils.token_counter import (
estimate_media_block_tokens,
offload_token_count,
split_embedding_input_tokens,
)
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -156,6 +159,12 @@ def _resolve_media_models(router: "Router | None", model: str, team_id: str | No
return tuple(dict.fromkeys(router.registry().names(model, kind="litellm_base_models", team_id=team_id))) or (model,)
def _media_models(
resolver: Callable[[str, str | None], tuple[str, ...]] | None, model: str, team_id: str | None
) -> tuple[str, ...]:
return resolver(model, team_id) if resolver is not None else (model,)
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
from litellm.proxy.proxy_server import llm_router
@ -1378,20 +1387,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
media_source: Final = messages if messages is not None else original_input
is_embedding_call: Final = call_type in EMBEDDING_API_CALL_TYPES
projected, media = (
split_media_for_tpm(media_source)
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
else (original_input, ())
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
fallback_model: Final = request_model if isinstance(request_model, str) else None
resolved_models: Final = (
media_model_resolver(fallback_model or "", team_id)
if media and media_model_resolver is not None
else (fallback_model or "",)
_media_models(media_model_resolver, fallback_model or "", team_id) if media else (fallback_model or "",)
)
media_tokens: Final = sum(
self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
embedding_text, embedding_media_tokens = (
split_embedding_input_tokens(
original_input,
lambda: _media_models(media_model_resolver, fallback_model or "", team_id),
fallback_model,
)
if include_media and is_embedding_call
else (original_input, 0)
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else embedding_text
media_tokens: Final = embedding_media_tokens + sum(
estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
)
total_chars: Final = (
@ -1436,24 +1453,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
@staticmethod
def _estimate_media_block_tokens(
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
) -> int:
if block.get("type") == "input_audio":
audio: Final = _callback_kwargs_mapping(block.get("input_audio"))
data: Final = audio.get("data")
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
image: Final = _callback_kwargs_mapping(block.get("image_url"))
url: Final = image.get("url")
detail: Final = image.get("detail")
return estimate_image_tokens(
models,
url if isinstance(url, str) else "",
detail if isinstance(detail, str) else "auto",
fallback_model=fallback_model,
)
def _is_redis_cluster(self) -> bool:
"""
Check if the dual cache is using Redis cluster.
@ -4098,12 +4097,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
)
is_embedding_request: Final = self._is_embedding_request(data, call_type)
embedding_text: Final = data.get("input") if is_embedding_request else None
request: Final = as_str_mapping(data)
raw_embedding_input: Final = request.get("input") if is_embedding_request and request is not None else None
pretokenized_input_tokens: Final = (
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
self._count_pretokenized_embedding_input(raw_embedding_input) if is_embedding_request else None
)
if pretokenized_input_tokens is not None:
return pretokenized_input_tokens
embedding_text, embedding_media_tokens = (
split_embedding_input_tokens(
raw_embedding_input, lambda: self._media_model_resolver(model or "", team_id), model
)
if is_embedding_request
else (None, 0)
)
prompt: Final = data.get("prompt")
fallback_text: Final = prompt if prompt is not None else data.get("input")
@ -4153,12 +4160,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",))
),
)
return estimate + audio_token_estimate
return estimate + audio_token_estimate + embedding_media_tokens
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data=data, call_type=call_type, include_media=False
data=data,
call_type=call_type,
include_media=is_embedding_request,
team_id=team_id,
media_model_resolver=self._media_model_resolver,
)
return estimated_input_tokens + audio_token_estimate

View file

@ -225,6 +225,32 @@ def _guardrail_info_lifted_from_litellm_metadata(
return {"standard_logging_guardrail_information": guardrail_info}
def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object:
metadata: Final = _str_keyed_fields(request_data.get("metadata"))
lifted: Final = _guardrail_info_lifted_from_litellm_metadata(
_str_keyed_fields(request_data.get("litellm_metadata")),
guardrail_info_already_set="standard_logging_guardrail_information" in metadata,
)
return {**metadata, **lifted}.get("standard_logging_guardrail_information")
def failure_row_response_cost(request_data: Mapping[str, object]) -> float:
stashed_cost: Final = request_data.get("response_cost")
recovered_stream_cost: Final = (
max(float(stashed_cost), 0.0)
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
and isinstance(stashed_cost, (int, float))
else 0.0
)
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
finalized_amount: Final = finalized_failure.get("response_cost")
return (
float(finalized_amount)
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data))
)
def _redaction_opt_in_lifted_from_litellm_metadata(
litellm_metadata: ObjectMapping, *, caller_set_headers: bool
) -> ObjectMapping:
@ -577,19 +603,7 @@ class _ProxyDBLogger(CustomLogger):
# recovered cost onto request_data (the usage rides along in
# ``combined_usage_object`` for the token columns), so attribute the
# real partial spend to this failure row instead of zero.
recovered_stream_cost: Final = (
max(float(request_data.get("response_cost") or 0.0), 0.0)
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
else 0.0
)
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
finalized_amount: Final = finalized_failure.get("response_cost")
recovered_response_cost: Final = (
float(finalized_amount)
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
else recovered_stream_cost
+ guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information"))
)
recovered_response_cost: Final = failure_row_response_cost(request_data)
failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it(
request_data.get("standard_logging_object"),

View file

@ -47,7 +47,7 @@ from litellm.proxy.auth.password_policy import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.openai_error_payload import openai_shaped_proxy_exception, proxy_exception_for
from litellm.proxy.common_utils.user_api_key_cache import (
AUTH_OBJECTS_TARGET,
object_permission_cache_key,
@ -1048,6 +1048,7 @@ async def _check_user_info_v2_access(
1. Proxy admins / proxy admin viewers can access any user
2. User can access their own info
3. Team admins can access info of users in their teams
4. Org admins can access info of users in their organizations
Raises on unexpected DB errors so they surface as 500s, not silent 404s.
"""
@ -1090,9 +1091,33 @@ async def _check_user_info_v2_access(
if team.team_id in (target_user.teams or []):
return target_user
if user_api_key_dict.user_id is not None and await _target_in_caller_admin_org(
prisma_client=prisma_client,
caller_user_id=user_api_key_dict.user_id,
target_user_id=target_user_id,
):
return await _fetch_target_user()
return None
async def _target_in_caller_admin_org(
prisma_client: "PrismaClient",
caller_user_id: str,
target_user_id: str,
) -> bool:
admin_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
where={"user_id": caller_user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value}
)
if not admin_memberships:
return False
admin_org_ids: Final = {membership.organization_id for membership in admin_memberships}
target_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
where={"user_id": target_user_id}
)
return any(membership.organization_id in admin_org_ids for membership in target_memberships)
@router.get(
"/v2/user/info",
tags=["Internal User management"],
@ -1197,6 +1222,8 @@ async def user_info_v2(
),
)
except Exception as e:
if isinstance(e, HTTPException) and e.status_code < 500:
raise openai_shaped_proxy_exception(e, str(e.detail), e.status_code)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)

View file

@ -14,9 +14,7 @@ from litellm.proxy.spend_tracking.savings import (
marks_gateway_injection,
prompt_caching_savings_for_request,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_query_raw_rows, # pyright: ignore[reportPrivateUsage] # existing typed spend-query adapter; rows validated below
)
from litellm.proxy.utils import query_raw_within_statement_timeout
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.prompt_caching_requests import (
@ -158,7 +156,7 @@ async def get_prompt_caching_requests(
else cursor_start_time
)
rows: Final = _REQUEST_ROWS.validate_python(
await _query_raw_rows(
await query_raw_within_statement_timeout(
prisma_client,
prompt_caching_requests_sql(filter),
start.isoformat(),
@ -166,8 +164,8 @@ async def get_prompt_caching_requests(
page_size + 1,
cursor_time.isoformat() if cursor_time is not None else None,
cursor_request_id,
what="Prompt caching requests",
)
or ()
)
def current_router() -> "Router | None":

View file

@ -224,7 +224,12 @@ from litellm.proxy.management_helpers.utils import (
add_new_member,
management_endpoint_wrapper,
)
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
from litellm.proxy.utils import (
PrismaClient,
ProxyLogging,
handle_exception_on_proxy,
query_raw_within_statement_timeout,
)
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.chunked_in import delete_many_in, find_many_in
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
@ -7473,6 +7478,9 @@ class _TeamUserSpendDbRow(TypedDict):
failed_requests: ReadOnly[int]
_TEAM_USER_SPEND_ROWS: Final = TypeAdapter(tuple[_TeamUserSpendDbRow, ...])
@router.get(
"/team/spend/by_user",
response_model=TeamUserSpendResponse,
@ -7523,12 +7531,16 @@ async def get_team_spend_by_user(
own_user_only: Final = scope.api_key_filter is not None
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
start_date,
end_date,
*scoped_team_ids,
*user_param,
rows: Final = _TEAM_USER_SPEND_ROWS.validate_python(
await query_raw_within_statement_timeout(
prisma_client,
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
start_date,
end_date,
*scoped_team_ids,
*user_param,
what="Spend per user",
)
)
results: Final = tuple(
TeamUserSpendRow(

View file

@ -23,7 +23,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import StartupOutputSettings, print_startup_message, print_startup_warning, terminal_text
from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE, DEFAULT_NUM_WORKERS_LITELLM_PROXY
from litellm.proxy.db.pgbouncer import (
PgBouncerError,
PgBouncerSettings,
@ -409,6 +409,7 @@ class ProxyInitializationHelpers:
log_config: str | None = None,
keepalive_timeout: int | None = None,
timeout_worker_healthcheck: int | None = None,
max_request_header_size: int | None = None,
) -> dict:
"""
Get the arguments for `uvicorn` worker
@ -427,6 +428,9 @@ class ProxyInitializationHelpers:
"port": port,
"server_header": False,
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
"h11_max_incomplete_event_size": (
max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE
),
}
if log_config is not None:
print_startup_message(f"Using log_config: {log_config}")
@ -680,6 +684,7 @@ class ProxyInitializationHelpers:
ssl_keyfile_path: str,
max_requests_before_restart: int | None = None,
max_requests_before_restart_jitter: int | None = None,
max_request_header_size: int | None = None,
):
"""
Run litellm with `gunicorn`
@ -754,7 +759,7 @@ class ProxyInitializationHelpers:
gunicorn_options: Final = {
"bind": f"{host}:{port}",
"workers": num_workers, # default is 1
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout, max_request_header_size),
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
"preload": True, # Add the preload flag,
"accesslog": "-", # Log to stdout
@ -1108,6 +1113,18 @@ class ProxyInitializationHelpers:
help="Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)",
envvar="KEEPALIVE_TIMEOUT",
)
@click.option(
"--max_request_header_size",
default=None,
show_default=f"{DEFAULT_MAX_REQUEST_HEADER_SIZE} bytes with h11",
type=click.IntRange(min=1),
help=(
"Maximum bytes buffered for an incomplete HTTP request header block. "
"Applies when uvicorn or --run_gunicorn uses h11. Parser selection is unchanged; httptools ignores this setting. "
"Ignored under --run_hypercorn / --run_granian."
),
envvar="MAX_REQUEST_HEADER_SIZE",
)
@click.option(
"--timeout_worker_healthcheck",
default=None,
@ -1253,6 +1270,7 @@ def run_server(
check_migrations_against_existing_data: bool,
keepalive_timeout,
timeout_worker_healthcheck,
max_request_header_size: int | None,
max_requests_before_restart: int | None,
max_requests_before_restart_jitter: int | None,
limit_concurrency: int | None,
@ -1747,6 +1765,7 @@ def run_server(
log_config=log_config,
keepalive_timeout=keepalive_timeout,
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
max_request_header_size=max_request_header_size,
)
# Optional: recycle uvicorn workers after N requests
if max_requests_before_restart is not None:
@ -1821,6 +1840,7 @@ def run_server(
ssl_keyfile_path=ssl_keyfile_path,
max_requests_before_restart=max_requests_before_restart,
max_requests_before_restart_jitter=max_requests_before_restart_jitter,
max_request_header_size=max_request_header_size,
)
elif run_hypercorn is True:
ProxyInitializationHelpers.init_hypercorn_server(

View file

@ -530,6 +530,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
encrypt_value_helper,
)
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
from litellm.proxy.common_utils.fips import enforce_fips_mode
from litellm.proxy.common_utils.healthy_model_filter import (
get_hidden_unhealthy_model_names,
is_healthy_only_listing_default,
@ -1673,6 +1674,8 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener
if decoded_worker_config is not None:
await initialize_from_worker_config(decoded_worker_config)
enforce_fips_mode(general_settings_view())
async with AsyncExitStack() as boot_database:
enforce_master_key_boot_verdict(
await with_stored_secrets_counted(
@ -18611,8 +18614,9 @@ async def model_deprecations(
Reads `deprecation_date` metadata from `model_prices_and_context_window.json`
(and any per-deployment `model_info.deprecation_date` overrides) for the
models configured on this proxy. An entry also names its `successor_model`
when the deployment sets `model_info.successor_model`.
models configured on this proxy. An entry also names its `successor_model`:
`model_info.successor_model` when the deployment sets one, else the
replacement the cost map records from the provider's deprecation notice.
Parameters:
warn_within_days: Window (in days) used to bucket "imminent" models,

View file

@ -3567,6 +3567,34 @@
],
"default_model_placeholder": "reka/reka-flash"
},
{
"provider": "WALLY",
"provider_display_name": "Wally",
"litellm_provider": "wally",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://inference.runanywhere.ai/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "wally/glm-5.3-flash"
},
{
"provider": "Sail",
"provider_display_name": "Sail",

View file

@ -1,10 +1,21 @@
from uvicorn.workers import UvicornWorker
from litellm.constants import DEFAULT_MAX_REQUEST_HEADER_SIZE
def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]:
def build_graceful_shutdown_uvicorn_worker(
timeout_graceful_shutdown: float,
max_request_header_size: int | None = None,
) -> type[UvicornWorker]:
class GracefulShutdownUvicornWorker(UvicornWorker):
def __init__(self, *args: object) -> None:
self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown}
self.CONFIG_KWARGS = {
**UvicornWorker.CONFIG_KWARGS,
"timeout_graceful_shutdown": timeout_graceful_shutdown,
"h11_max_incomplete_event_size": (
max_request_header_size if max_request_header_size is not None else DEFAULT_MAX_REQUEST_HEADER_SIZE
),
}
super().__init__(*args)
return GracefulShutdownUvicornWorker

View file

@ -3,6 +3,7 @@ import collections
import json
import os
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import date, datetime, timedelta, timezone
from functools import partial
from itertools import groupby
@ -72,7 +73,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
get_spend_by_team,
get_spend_by_team_and_customer,
)
from litellm.proxy.utils import handle_exception_on_proxy
from litellm.proxy.utils import handle_exception_on_proxy, query_raw_within_statement_timeout
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SpendLogsRepository
from litellm.repositories.team_repository import TeamRepository
@ -267,6 +268,30 @@ async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args:
return await _query_raw(prisma_client, sql_query, *args)
async def _query_spend_logs(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]:
"""``_query_raw`` for raw ``LiteLLM_SpendLogs`` reads, which Postgres cancels at the statement timeout."""
return cast( # cast-ok: callers declare the row shape their SELECT list produces, as with _query_raw
"Sequence[_RowT]", await query_raw_within_statement_timeout(prisma_client, sql_query, *args)
)
async def _query_spend_logs_or_none(
prisma_client: PrismaClient, sql_query: str, *args: object
) -> Sequence[_RowT] | None:
"""``_query_spend_logs`` for the call sites that guard the result against ``None``."""
return await _query_spend_logs(prisma_client, sql_query, *args)
@dataclass(frozen=True, slots=True)
class _SpendLogsReader:
"""``read_session_page``'s raw-query handle, each read cancelled by Postgres at the statement timeout."""
prisma_client: PrismaClient
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
return await query_raw_within_statement_timeout(self.prisma_client, query, *args)
class _TeamTable(Protocol):
"""The subset of the Prisma team table API this module uses."""
@ -1566,7 +1591,9 @@ async def get_global_spend_report(
ORDER BY
total_cost DESC;
"""
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj, api_key)
db_response = await _query_spend_logs_or_none(
prisma_client, sql_query, start_date_obj, end_date_obj, api_key
)
if db_response is None:
return []
@ -1609,7 +1636,7 @@ async def get_global_spend_report(
ORDER BY
total_cost DESC;
"""
db_response = await _query_raw_or_none(
db_response = await _query_spend_logs_or_none(
prisma_client, sql_query, start_date_obj, end_date_obj, internal_user_id
)
if db_response is None:
@ -1676,7 +1703,7 @@ async def get_global_spend_report(
group_by_day;
"""
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
db_response = await _query_spend_logs_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
if db_response is None:
return []
@ -1717,12 +1744,14 @@ async def get_global_spend_report(
ORDER BY
total_cost DESC;
"""
db_response = await _query_raw_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
db_response = await _query_spend_logs_or_none(prisma_client, sql_query, start_date_obj, end_date_obj)
if db_response is None:
return []
return db_response
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@ -1966,7 +1995,7 @@ async def get_key_spend_report(
caller_value=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict),
scope_name="api_key",
)
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
prisma_client,
_scoped_spend_report_sql(scope_column="api_key"),
start_date_obj,
@ -2008,7 +2037,7 @@ async def get_user_spend_report(
caller_value=user_api_key_dict.user_id,
scope_name="internal_user_id",
)
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
prisma_client,
_scoped_spend_report_sql(scope_column="user"),
start_date_obj,
@ -2050,7 +2079,7 @@ async def get_team_spend_report(
caller_value=user_api_key_dict.team_id,
scope_name="team_id",
)
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
prisma_client,
_scoped_spend_report_sql(scope_column="team_id"),
start_date_obj,
@ -2092,7 +2121,7 @@ async def get_organization_spend_report(
organization_id=organization_id,
prisma_client=prisma_client,
)
db_response: Sequence[Mapping[str, object]] | None = await _query_raw_or_none(
db_response: Sequence[Mapping[str, object]] | None = await _query_spend_logs_or_none(
prisma_client,
_ORG_SPEND_REPORT_SQL,
start_date_obj,
@ -3017,7 +3046,7 @@ async def ui_view_spend_logs(
LIMIT ${p}
) AS bounded_matches
"""
count_rows: Final[Sequence[_SpendLogsCountRow] | None] = await _query_raw_or_none(
count_rows: Final[Sequence[_SpendLogsCountRow] | None] = await _query_spend_logs_or_none(
prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1
)
raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0
@ -3048,7 +3077,7 @@ async def ui_view_spend_logs(
)
sql_params.extend([page_size, skip])
data: Final[Sequence[Mapping[str, object]]] = await _query_raw(prisma_client, sql_query, *sql_params)
data: Final[Sequence[Mapping[str, object]]] = await _query_spend_logs(prisma_client, sql_query, *sql_params)
if request_id is not None and not is_v2 and not is_admin_view:
await _assert_user_owns_fetched_spend_rows(
@ -3102,7 +3131,7 @@ async def _fetch_session_representatives(
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_representatives
"""
rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place
rep_rows: Final[Sequence[dict[str, object]]] = await _query_spend_logs( # mutable-ok: rows are enriched in place
prisma_client,
rep_query,
*sql_params,
@ -3130,7 +3159,7 @@ async def _ui_session_grouped_spend_logs(
and each is shown by its final agent outcome or newest non-MCP row."""
where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE"
session_page: Final = await read_session_page(
prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__
_SpendLogsReader(prisma_client), # pyright: ignore[reportArgumentType] # rows are the TypedDict the session SQL selects
where_clause=where_clause,
params=sql_params,
status=status_filter,
@ -4605,7 +4634,7 @@ async def _build_ui_spend_logs_response(
if (row.get("api_key") if isinstance(row, dict) else getattr(row, "api_key", None)) is not None
}
)
rows: Final[Sequence[_SessionSpendRow]] = await _query_raw(
rows: Final[Sequence[_SessionSpendRow]] = await _query_spend_logs(
prisma_client,
f"""
SELECT s.*, COALESCE(m.session_models, ARRAY[]::text[]) AS session_models
@ -4676,7 +4705,7 @@ async def _build_ui_spend_logs_response(
if row.get("session_id") and row.get("api_key") is not None
for models in (TypeAdapter(list[str]).validate_python(row.get("session_models") or ()),)
}
except PrismaError:
except (PrismaError, HTTPException):
verbose_proxy_logger.debug(
"Failed to enrich session spend aggregates for spend logs UI",
exc_info=True,

View file

@ -75,7 +75,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.proxy.utils import PrismaClient, hash_token
from litellm.proxy.utils import PrismaClient, hash_token, query_raw_within_statement_timeout
from litellm.responses.litellm_completion_transformation.compaction_items import (
compaction_summary,
summary_only_compaction_content,
@ -649,10 +649,12 @@ def get_logging_payload(
usage = _usage
# A request that failed mid-stream has no usable response_obj usage, and a stream
# continued on a fallback only has the fallback's. combined_usage_object holds the
# usage of every call the client's answer took, so it wins in both cases.
# continued on a fallback or a success that bills a failed follow-up only has its own.
# combined_usage_object holds the usage of every call the row bills, so it wins in each case.
_combined_usage: Final = kwargs.get("combined_usage_object")
if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage):
if (
not usage or "continued_partial_usage" in kwargs or "failed_attempts_billed_on_success" in kwargs
) and isinstance(_combined_usage, litellm.Usage):
usage = _combined_usage.model_dump()
id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata)
@ -1092,7 +1094,8 @@ async def _query_raw_rows(
sql_query: str,
*args: object,
) -> Sequence[Mapping[str, object]] | None:
return await prisma_client.db.query_raw(sql_query, *args)
"""A raw ``LiteLLM_SpendLogs`` read that Postgres cancels at the statement timeout."""
return await query_raw_within_statement_timeout(prisma_client, sql_query, *args)
async def get_spend_by_team(

View file

@ -1360,7 +1360,7 @@ async def update_sso_settings(
if isinstance(environment_variables_param, Mapping)
else MappingProxyType({})
)
stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None}
stored_values: Final = {k: v for k, v in (before_sso_data or {}).items() if v is not None and v != ""}
fallback_values: Final = {**legacy_env_values, **stored_values}
submitted: Final = sso_config.model_dump(include=sso_config.model_fields_set)
sso_data: Final = {

View file

@ -66,6 +66,7 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT,
SPEND_LOG_QUEUE_MAX_BYTES,
SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS,
SPEND_LOG_WRITE_BATCH_MAX_BYTES,
SPEND_LOG_WRITE_BATCH_MAX_ROWS,
VLLM_CHOICE_FIELDS,
@ -5309,6 +5310,39 @@ _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(ma
_DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60
_PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5)
_PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2)
_SPEND_LOG_READ_STATEMENT_TIMEOUT: Final = timedelta(seconds=SPEND_LOG_READ_STATEMENT_TIMEOUT_SECONDS)
_STATEMENT_TIMEOUT_TRANSACTION_GRACE: Final = timedelta(seconds=5)
class _ReadTransactionClient(Protocol):
@property
def read_db(self) -> PrismaWrapper: ...
async def query_raw_within_statement_timeout(
prisma_client: _ReadTransactionClient,
sql: str,
*params: object,
what: str = "Spend logs",
timeout: timedelta = _SPEND_LOG_READ_STATEMENT_TIMEOUT,
) -> Sequence[Mapping[str, object]]:
"""One raw read on the read target that Postgres cancels at ``timeout``.
Prisma's HTTP client gives up on a statement after 30s anyway, so without
this the Admin UI's error is the same but Postgres keeps running the scan.
A cancelled read surfaces as a 503 asking for a narrower date range.
"""
try:
async with prisma_client.read_db.tx(timeout=timeout + _STATEMENT_TIMEOUT_TRANSACTION_GRACE) as transaction:
await transaction.execute_raw(f"SET LOCAL statement_timeout = {int(timeout / timedelta(milliseconds=1))}")
return await transaction.query_raw(sql, *params)
except Exception as e:
if not PrismaDBExceptionHandler.is_statement_timeout_error(e):
raise
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail={"error": f"{what} took longer than {int(timeout.total_seconds())}s. Pick a shorter date range"},
) from e
async def _lookup_deprecated_key(

View file

@ -124,9 +124,7 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"user_id": user_id})
return self._to_model_list(records)
async def find_newest_reusable_llm_api_key(
self, user_id: str, team_id: str | None
) -> LiteLLM_VerificationToken | None:
async def find_newest_reusable_key(self, user_id: str, team_id: str | None) -> LiteLLM_VerificationToken | None:
records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(
where={
"user_id": user_id,
@ -140,12 +138,6 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
{"team_id": {"not": UI_SESSION_TOKEN_TEAM_ID}},
]
},
{
"OR": [
{"allowed_routes": {"is_empty": True}},
{"allowed_routes": {"has": "llm_api_routes"}},
]
},
],
},
order={"created_at": "desc"},

View file

@ -110,6 +110,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
replace_complexity_router_headers,
response_total_token_count,
)
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.batch_utils import (
batch_create_routing,
@ -135,7 +136,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models, deployment_pricing_models
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
@ -483,11 +484,13 @@ from litellm.types.services import ServiceTypes
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
COST_MAP_LOOKUP_KEY,
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS,
PROMPT_QUOTING_ROUTING_DECISION_FIELDS,
CustomPricingLiteLLMParams,
GenericBudgetConfigType,
LiteLLMBatch,
LlmProviders,
LlmProvidersSet,
ModelInfo,
ModelResponseStream,
StandardLoggingPayload,
@ -510,9 +513,11 @@ from litellm.utils import (
TextCompletionResponse,
_restore_correlation_context_if_supported, # pyright: ignore[reportPrivateUsage] # same task-local cleanup as the SDK wrapper
aresolve_api_key_secret_reference,
catalog_with_runtime_registrations,
function_setup,
get_llm_provider,
get_non_default_completion_params,
get_potential_model_names,
get_secret,
get_utc_datetime,
is_generalized_model_info,
@ -4044,6 +4049,8 @@ class PythonCore(DeprecatedRouterMembers):
if continuation_output is not None
else None
)
if cut_partial is not None:
model_response.logging_obj.mark_stash_billed_by_continuation()
if continue_after_content:
emitted_tokens: Final = int(
getattr(complete_response_object_usage, "completion_tokens", 0) or 0
@ -7373,6 +7380,8 @@ class PythonCore(DeprecatedRouterMembers):
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
trigger = _mid_stream_retry_trigger(last_error)
if is_agentic_followup_failure(trigger):
return _StreamRetriesExhausted(last_error)
if not policy_applies and not self._stream_should_retry(
trigger, healthy_deployments, all_deployments, initial_kwargs
):
@ -9620,7 +9629,7 @@ class PythonCore(DeprecatedRouterMembers):
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
return response
except Exception as e:
if is_guardrail_intervention(e):
if is_guardrail_intervention(e) or is_agentic_followup_failure(e):
raise
if is_create_that_may_already_exist(_OPAQUE_VALUE_ADAPTER.validate_python(original_function), e):
log_create_not_resent(model_group)
@ -9718,6 +9727,7 @@ class PythonCore(DeprecatedRouterMembers):
# Update retry tracking metadata before each retry attempt
_metadata["attempted_retries"] = current_attempt + 1
_metadata["max_retries"] = num_retries
carry_superseded_attempt_usage(kwargs)
# if the function call is successful, no exception will be raised and we'll break out of the loop
response = await self.make_call(original_function, *args, **kwargs)
if coroutine_checker.is_async_callable(response): # async errors are often returned as coroutines
@ -9731,6 +9741,8 @@ class PythonCore(DeprecatedRouterMembers):
return response
except Exception as e:
if is_agentic_followup_failure(e):
raise
if is_create_that_may_already_exist(_OPAQUE_VALUE_ADAPTER.validate_python(original_function), e):
log_create_not_resent(model_group)
raise
@ -12186,6 +12198,25 @@ class PythonCore(DeprecatedRouterMembers):
return (backend_key, backend_key.replace("responses/", ""))
return (backend_key,)
@staticmethod
def _catalog_entry_has_token_limit(catalog_entry: Mapping[str, object], field: str) -> bool:
return catalog_entry.get(field) is not None or (
field == "max_tokens" and catalog_entry.get("max_output_tokens") is not None
)
@staticmethod
def _shared_backend_model_info_for_keys(
catalog: Mapping[str, Mapping[str, object]],
backend_keys: tuple[str, ...],
shared_model_info: Mapping[str, object],
) -> Mapping[str, object]:
catalog_token_limit_fields: Final = frozenset(
field
for field in DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS
if any(PythonCore._catalog_entry_has_token_limit(catalog.get(key) or {}, field) for key in backend_keys)
)
return {field: value for field, value in shared_model_info.items() if field not in catalog_token_limit_fields}
@staticmethod
def _deployment_model_cost_payload(deployment: Deployment) -> dict[str, object]: # mutable-ok: cost-map entry
"""The ``model_info`` a deployment contributes to ``litellm.model_cost``.
@ -12298,6 +12329,31 @@ class PythonCore(DeprecatedRouterMembers):
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
backend_keys: Final = PythonCore._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider)
model_provider_prefix: Final = model.partition("/")[0]
catalog_provider: Final = (
custom_llm_provider
if custom_llm_provider is not None
else model_provider_prefix
if model_provider_prefix in LlmProvidersSet
else None
)
potential_names: Final = (
get_potential_model_names(model, catalog_provider) if catalog_provider is not None else None
)
catalog_backend_keys: Final = (
(
*backend_keys,
model,
potential_names["split_model"],
potential_names["stripped_model_name"],
potential_names["combined_model_name"],
potential_names["combined_stripped_model_name"],
potential_names["provider_prefixed_model_name"],
potential_names["region_free_combined_model_name"],
)
if potential_names is not None
else (*backend_keys, model)
)
# For the shared backend key, keep only cost-map schema fields
# (minus custom pricing) so that one deployment's pricing overrides
@ -12305,7 +12361,12 @@ class PythonCore(DeprecatedRouterMembers):
# don't pollute another deployment sharing the same backend model
# name. Each deployment's full model_info is already stored under
# its unique model_id above.
shared_model_info: Final = shared_backend_model_info(model_info)
catalog: Final = catalog_with_runtime_registrations()
shared_model_info: Final = PythonCore._shared_backend_model_info_for_keys(
catalog=catalog,
backend_keys=catalog_backend_keys,
shared_model_info=shared_backend_model_info(model_info),
)
python_host.invoke("register_model")(
model_cost={_key: keep_registered_mode(_key, shared_model_info) for _key in backend_keys},
persist_across_reloads=False,
@ -14894,6 +14955,7 @@ class PythonCore(DeprecatedRouterMembers):
input: str | list[object] | None,
request_kwargs: Mapping[str, object] | None = None,
tokenizer_model: str = "",
deployments: Sequence[Mapping[str, object]] = (),
) -> int:
"""
Count input tokens for context-window pre-call checks.
@ -14928,11 +14990,15 @@ class PythonCore(DeprecatedRouterMembers):
if input is not None:
from openai.types.responses.response_create_params import ResponseInputParam
from litellm.litellm_core_utils.token_counter import split_embedding_input_tokens
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
typed_input: Final = cast(str | ResponseInputParam, input) # cast-ok: str | list matches transform input
text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments))
typed_input: Final = cast( # cast-ok: str | list matches transform input
str | ResponseInputParam, text_input
)
input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=typed_input,
responses_api_request={"instructions": instructions} if instructions is not None else {},
@ -15000,6 +15066,7 @@ class PythonCore(DeprecatedRouterMembers):
self.served_openai_models(model) or served_openai_models(healthy_deployments)
)
or "",
deployments=healthy_deployments,
)
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
verbose_router_logger.error(
@ -15030,6 +15097,7 @@ class PythonCore(DeprecatedRouterMembers):
self.served_openai_models(model) or served_openai_models(deployments)
)
or "",
deployments=deployments,
)
except Exception as e:
verbose_router_logger.error(

View file

@ -198,7 +198,7 @@ def _detect_loop(history: list[str], new_calls: Sequence[Mapping[str, object]])
return False
_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504}
_EXHAUSTION_STATUSES: Final = {408, 413, 429, 503, 504, 529}
_EXHAUSTION_KEYWORDS: Final = (
"context length",

View file

@ -2,16 +2,22 @@
# picks based on response time (for streaming, this is time to first token)
from collections.abc import Mapping
from datetime import datetime
from functools import partial
from typing import Final
import litellm
from litellm import ModelResponse, token_counter, verbose_logger
from litellm import ModelResponse, verbose_logger
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.litellm_core_utils.deployment_pricing import (
deployment_backend_models,
deployment_pricing_models,
str_keyed,
)
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
@ -235,7 +241,9 @@ class LowestCostLoggingHandler(CustomLogger):
}
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0

View file

@ -3,18 +3,21 @@
import random
from collections.abc import Sequence
from datetime import datetime, timedelta
from functools import partial
from math import ceil, isfinite
from typing import TYPE_CHECKING, Any, Final
from pydantic import Field
import litellm
from litellm import ModelResponse, token_counter, verbose_logger
from litellm import ModelResponse, verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import LiteLLMPydanticObjectBase
@ -439,7 +442,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
}
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0

View file

@ -2,14 +2,16 @@
# identifies lowest tpm deployment
import traceback
from datetime import datetime
from functools import partial
from typing import Final
from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.types.utils import LiteLLMPydanticObjectBase
from litellm.utils import print_verbose
@ -174,7 +176,9 @@ class LowestTPMLoggingHandler(CustomLogger):
verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict)
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0
verbose_router_logger.debug("input_tokens=%s", input_tokens)

View file

@ -5,18 +5,20 @@ from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, Final
import httpx
import litellm
from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger, verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.types.router import RouterErrors
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
from litellm.utils import get_utc_datetime, print_verbose
@ -422,7 +424,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
rpm_dict[rpm_keys[idx].split(":")[0]] = rpm_values[idx]
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0
verbose_router_logger.debug("input_tokens=%s", input_tokens)

View file

@ -20,6 +20,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm import token_counter
@ -28,6 +29,7 @@ from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError
from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION
from litellm.litellm_core_utils.core_helpers import coerce_token_limit
from litellm.litellm_core_utils.internal_call_metadata import (
IO_TOKEN_RESERVATION_KEYS,
ITPM_CACHE_KEY,
@ -41,6 +43,8 @@ from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
from litellm.types.router import RouterCacheEnum, RouterErrors
from litellm.utils import get_utc_datetime
_MODEL_INFO_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -227,8 +231,20 @@ def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployme
)
def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int:
model_max: Final = _model_max_output_tokens(model_name) if model_name else None
def _default_otpm_reservation(deployment: Mapping[str, object], model_name: str, otpm_limit: int) -> int:
model_info: Final[object] = deployment.get("model_info")
configured_model_max: Final = (
coerce_token_limit(_MODEL_INFO_ADAPTER.validate_python(model_info).get("max_output_tokens"))
if isinstance(model_info, Mapping)
else None
)
model_max: Final = (
max(0, configured_model_max)
if configured_model_max is not None
else _model_max_output_tokens(model_name)
if model_name
else None
)
share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))
return min(4096 if model_max is None else model_max, share_of_limit)
@ -242,7 +258,7 @@ def _otpm_reservation(
from_request: Final = _explicit_max_tokens(request_kwargs)
if from_request is not None:
return from_request
default: Final = _default_otpm_reservation(model_name, otpm_limit)
default: Final = _default_otpm_reservation(deployment, model_name, otpm_limit)
from_deployment: Final = _deployment_output_cap(request_kwargs, deployment)
return default if from_deployment is None else min(from_deployment, default)

View file

@ -74,14 +74,13 @@ class AgenticLoopSafetyError(ValueError):
Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the
repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so
callers that already catch the broader type keep working.
Only the anthropic messages loop raises this today. The chat completions
loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises
a plain ``ValueError`` from its own copy of the same rails, so catching
this type alone will not cover that surface until it is moved over.
"""
class AgenticLoopLimitError(AgenticLoopSafetyError):
pass
class StandardCustomLoggerInitParams(LiteLLMBaseModel):
"""
Params for initializing a CustomLogger.

View file

@ -34,7 +34,10 @@ class ModelDeprecationInfo(LiteLLMBaseModel):
litellm_provider: str | None = Field(default=None, description="The provider this model belongs to.")
successor_model: str | None = Field(
default=None,
description=("The model to migrate to, as set in model_info.successor_model on the deployment."),
description=(
"The model to migrate to: model_info.successor_model on the deployment when set, "
"else the successor_model the cost map names for the deprecated model."
),
)

View file

@ -383,6 +383,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_character: float | None # only for vertex ai models
input_cost_per_audio_token: float | None
input_cost_per_audio_token_flex: ReadOnly[float | None]
input_cost_per_audio_token_above_200k_tokens: ReadOnly[float | None]
input_cost_per_audio_token_above_200k_tokens_flex: ReadOnly[float | None]
input_cost_per_audio_token_above_200k_tokens_priority: ReadOnly[float | None]
input_cost_per_audio_token_above_200k_tokens_batches: ReadOnly[float | None]
input_cost_per_audio_token_priority: ReadOnly[float | None]
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
@ -408,10 +412,12 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token_batches: float | None
input_cost_per_video_token_batches: ReadOnly[float | None]
input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
input_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None]
input_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None]
input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
output_cost_per_token_batches: float | None
output_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
output_cost_per_token_above_200k_tokens_flex: ReadOnly[float | None]
output_cost_per_token_above_100k_tokens_batches: ReadOnly[float | None]
output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
output_cost_per_token: Required[float | None]
@ -444,6 +450,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_pixel: ReadOnly[float | None]
output_cost_per_image_token: float | None
output_cost_per_image_token_batches: ReadOnly[float | None]
output_cost_per_image_token_priority: ReadOnly[float | None]
output_cost_per_video_token: float | None # for gemini omni models with video output
output_vector_size: int | None
output_cost_per_reasoning_token: float | None
@ -4173,6 +4180,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_character_above_128k_tokens: float | None = None
input_cost_per_audio_token: float | None = None
input_cost_per_audio_token_flex: float | None = None
input_cost_per_audio_token_above_200k_tokens: float | None = None
input_cost_per_audio_token_above_200k_tokens_flex: float | None = None
input_cost_per_audio_token_above_200k_tokens_priority: float | None = None
input_cost_per_audio_token_above_200k_tokens_batches: float | None = None
input_cost_per_audio_token_priority: float | None = None
input_cost_per_token_cache_hit: float | None = None
input_cost_per_token_above_128k_tokens: float | None = None
@ -4183,6 +4194,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_token_above_272k_tokens_flex: float | None = None
input_cost_per_token_above_272k_tokens_ultrafast: float | None = None
input_cost_per_token_above_200k_tokens_batches: float | None = None
input_cost_per_token_above_200k_tokens_flex: float | None = None
input_cost_per_token_above_100k_tokens_batches: float | None = None
input_cost_per_token_above_272k_tokens_batches: float | None = None
input_cost_per_query: float | None = None
@ -4212,12 +4224,14 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_above_272k_tokens_flex: float | None = None
output_cost_per_token_above_272k_tokens_ultrafast: float | None = None
output_cost_per_token_above_200k_tokens_batches: float | None = None
output_cost_per_token_above_200k_tokens_flex: float | None = None
output_cost_per_token_above_100k_tokens_batches: float | None = None
output_cost_per_token_above_272k_tokens_batches: float | None = None
output_cost_per_character_above_128k_tokens: float | None = None
output_cost_per_image: float | None = None
output_cost_per_image_token: float | None = None
output_cost_per_image_token_batches: float | None = None
output_cost_per_image_token_priority: float | None = None
output_cost_per_video_token: float | None = None
output_cost_per_reasoning_token: float | None = None
output_cost_per_reasoning_token_flex: float | None = None
@ -4284,6 +4298,10 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
}
)
DEPLOYMENT_SCOPED_TOKEN_LIMIT_FIELDS: Final[frozenset[str]] = frozenset(
{"max_input_tokens", "max_output_tokens", "max_tokens"}
)
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset(
{
"token_count_framing",
@ -4756,6 +4774,7 @@ class LlmProviders(str, Enum):
SAIL = "sail"
REKA = "reka"
STREAMLAKE = "streamlake"
WALLY = "wally"
LITELLM_AGENT = "litellm_agent"
CURSOR = "cursor"
BEDROCK_MANTLE = "bedrock_mantle"

View file

@ -83,6 +83,7 @@ from litellm.constants import (
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
CHAT_ALLOWED_TOOLS_LAYOUT,
parse_chat_allowed_tools_tool_choice,
@ -2312,7 +2313,7 @@ def client(original_function):
is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_completion_litellm_router_call
num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2346,7 +2347,7 @@ def client(original_function):
is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_responses_litellm_router_call
num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2645,7 +2646,10 @@ def client(original_function):
is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
if (
num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError)
num_retries
and not is_acompletion_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops
@ -2676,7 +2680,10 @@ def client(original_function):
is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
if (
num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError)
num_retries
and not is_aresponses_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops
@ -7445,6 +7452,18 @@ def get_model_info_helper(
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
input_cost_per_audio_token_above_200k_tokens=_model_info.get(
"input_cost_per_audio_token_above_200k_tokens", None
),
input_cost_per_audio_token_above_200k_tokens_flex=_model_info.get(
"input_cost_per_audio_token_above_200k_tokens_flex", None
),
input_cost_per_audio_token_above_200k_tokens_priority=_model_info.get(
"input_cost_per_audio_token_above_200k_tokens_priority", None
),
input_cost_per_audio_token_above_200k_tokens_batches=_model_info.get(
"input_cost_per_audio_token_above_200k_tokens_batches", None
),
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
@ -7458,6 +7477,9 @@ def get_model_info_helper(
input_cost_per_token_above_200k_tokens_batches=_model_info.get(
"input_cost_per_token_above_200k_tokens_batches"
),
input_cost_per_token_above_200k_tokens_flex=_model_info.get(
"input_cost_per_token_above_200k_tokens_flex", None
),
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
"input_cost_per_token_above_272k_tokens_batches"
),
@ -7465,6 +7487,9 @@ def get_model_info_helper(
output_cost_per_token_above_200k_tokens_batches=_model_info.get(
"output_cost_per_token_above_200k_tokens_batches"
),
output_cost_per_token_above_200k_tokens_flex=_model_info.get(
"output_cost_per_token_above_200k_tokens_flex", None
),
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
"output_cost_per_token_above_272k_tokens_batches"
),
@ -7526,6 +7551,7 @@ def get_model_info_helper(
output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
output_cost_per_image_token_priority=_model_info.get("output_cost_per_image_token_priority", None),
output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None),
output_vector_size=_model_info.get("output_vector_size", None),
citation_cost_per_token=_model_info.get("citation_cost_per_token", None),

File diff suppressed because it is too large Load diff

View file

@ -5250,6 +5250,32 @@
"generateContent": false
}
},
"wally": {
"display_name": "Wally (`wally`)",
"url": "https://docs.litellm.ai/docs/providers/wally",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"empiriolabs": {
"display_name": "EmpirioLabs (`empiriolabs`)",
"url": "https://docs.litellm.ai/docs/providers/empiriolabs",

View file

@ -266,6 +266,7 @@ proxy-dev = [
"azure-identity==1.25.2",
"a2a-sdk==1.1.0",
"backports-zstd==1.8.0; python_version < '3.14'",
"httptools>=0.7.1,<0.8",
]
ci = [
"psutil==7.2.2",

View file

@ -0,0 +1,54 @@
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { dismissFeedbackPopup, navigateToPage } from "../../helpers/navigation";
import { createVirtualKey, deleteVirtualKey, uniqueSuffix } from "../../helpers/traffic";
const paletteSearch = (page: PlaywrightPage) => page.getByRole("combobox", { name: "Search" });
const headerTrigger = (page: PlaywrightPage) => page.getByRole("button", { name: "Search keys…", exact: true });
test.describe("Command palette", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test.beforeEach(async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
await expect(headerTrigger(page)).toBeVisible();
});
test("opens a key found by its alias", async ({ page }) => {
const alias = `e2e-palette-key-${uniqueSuffix()}`;
const created = await createVirtualKey(page.request, { key_alias: alias });
try {
await page.keyboard.press("ControlOrMeta+K");
await paletteSearch(page).fill(alias);
await expect(page.getByRole("option", { name: new RegExp(`^${alias}`) })).toBeVisible();
await page.keyboard.press("Enter");
await expect(page).toHaveURL(new RegExp(`[?&]key=${created.token}`));
await expect(paletteSearch(page)).toBeHidden();
} finally {
await deleteVirtualKey(page.request, created.token);
}
});
test("does not open over the Create New Key dialog", async ({ page }) => {
await page.getByRole("button", { name: /Create New Key/i }).click();
const keyName = page.getByLabel(/Key Name/);
await keyName.click();
await page.keyboard.press("ControlOrMeta+K");
await page.keyboard.type("still-in-the-dialog");
await expect(keyName).toHaveValue("still-in-the-dialog");
await expect(paletteSearch(page)).toBeHidden();
});
test("opens from the header trigger and closes with Escape", async ({ page }) => {
await headerTrigger(page).click();
await expect(paletteSearch(page)).toBeFocused();
await page.keyboard.press("Escape");
await expect(paletteSearch(page)).toBeHidden();
});
});

View file

@ -7,13 +7,15 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from hashlib import sha256
from pathlib import Path
from typing import Final
from typing import Final, Literal
import httpx
import jwt
import pytest
import yaml
from cryptography.hazmat.primitives.asymmetric import rsa
from pydantic import JsonValue
from redis import Redis
from tests.integration._support.client import Gateway, eventually, string_value
from tests.integration._support.database import read_rows
@ -31,7 +33,15 @@ def _hash(key: str) -> str:
return sha256(key.encode()).hexdigest()
def _config(directory: Path, claim_field: str) -> Path:
def _config(
directory: Path,
claim_field: str,
*,
behavior: Literal["auto_register", "link_only", "reject"] = "auto_register",
map_existing_key: bool = True,
user_id_upsert: bool = False,
issuers: tuple[dict[str, JsonValue], ...] = (),
) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["general_settings"] = {
**config["general_settings"],
@ -40,8 +50,10 @@ def _config(directory: Path, claim_field: str) -> Path:
"user_id_jwt_field": "sub",
"user_email_jwt_field": "email",
"virtual_key_claim_field": claim_field,
"unregistered_jwt_client_behavior": "auto_register",
"auto_register_map_existing_key": True,
"unregistered_jwt_client_behavior": behavior,
"auto_register_map_existing_key": map_existing_key,
"user_id_upsert": user_id_upsert,
"issuers": list(issuers),
},
}
path: Final = directory / f"jwt_map_existing_key_{claim_field}.yaml"
@ -107,7 +119,7 @@ def _billed_key(response: httpx.Response) -> str:
return string_value(rows[0]["api_key"])
def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_ineligible_newer_key(
def test_first_jwt_call_reuses_the_newest_durable_key_and_skips_every_ineligible_newer_key(
gateway: Gateway, tmp_path: Path
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
@ -118,7 +130,6 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig
skipped: Final = {
"older_durable": older_durable,
"expiring": scenario.key(user_id=user, duration="1h"),
"management_only": scenario.key(user_id=user, allowed_routes=["management_routes"]),
"auto_registered_look_alike": scenario.key(user_id=user, metadata={"auto_registered": True}),
"other_team": scenario.key(user_id=user, team_id=scenario.team()),
"blocked": scenario.key(user_id=user),
@ -140,6 +151,84 @@ def test_first_jwt_call_reuses_the_newest_durable_llm_key_and_skips_every_inelig
assert _billed_key(response) == _hash(durable)
@pytest.mark.parametrize(
("behavior", "map_existing_key"), [("auto_register", True), ("link_only", False), ("link_only", True)]
)
def test_route_restricted_keys_are_linked_without_bypassing_other_key_checks(
gateway: Gateway,
tmp_path: Path,
behavior: Literal["auto_register", "link_only"],
map_existing_key: bool,
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
scenario.key(user_id=user)
restricted: Final = scenario.key(user_id=user, allowed_routes=["/v1/chat/completions"])
scenario.key(user_id=user, duration="1h")
scenario.key(user_id=user, metadata={"auto_registered": True})
scenario.key(user_id=user, team_id=scenario.team())
blocked: Final = scenario.key(user_id=user)
gateway.post("/key/block", {"key": blocked})
keys_before: Final = _user_key_hashes(user)
token: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(
tmp_path,
"sub",
behavior=behavior,
map_existing_key=map_existing_key,
),
) as candidate:
response: Final = _chat(candidate, model, token)
repeated: Final = _chat(candidate, model, token)
restricted_route: Final = candidate.request("GET", "/v1/models", key=token)
assert response.status_code == 200, response.text
assert repeated.status_code == 200, repeated.text
assert restricted_route.status_code == 403, restricted_route.text
assert _mapped_token("sub", user) == _hash(restricted)
assert _user_key_hashes(user) == keys_before, "a new key was minted instead of linking an existing key"
assert _billed_key(response) == _hash(restricted)
assert _billed_key(repeated) == _hash(restricted)
@pytest.mark.parametrize("behavior", ["auto_register", "link_only"])
def test_management_route_keys_are_linked_by_default_and_keep_their_route_permissions(
gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"]
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
target: Final = f"integration-{uuid.uuid4().hex}"
scenario.key(user_id=user)
management_key: Final = scenario.key(user_id=user, allowed_routes=["/user/new"])
keys_before: Final = _user_key_hashes(user)
token: Final = _token(private_key, user)
with owned_proxy(
gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=_config(tmp_path, "sub", behavior=behavior)
) as candidate:
denied: Final = _chat(candidate, model, token)
allowed: Final = candidate.request(
"POST",
"/user/new",
{"user_id": target, "auto_create_key": False, "user_role": "internal_user"},
key=token,
)
assert allowed.status_code == 200, allowed.text
scenario.cleanups.callback(scenario.delete_user, target)
repeated: Final = _chat(candidate, model, token)
assert denied.status_code == 403, denied.text
assert repeated.status_code == 403, repeated.text
assert allowed.json()["user_id"] == target
assert _mapped_token("sub", user) == _hash(management_key)
assert _user_key_hashes(user) == keys_before
def test_user_matched_by_email_instead_of_sub_still_reuses_their_existing_key(gateway: Gateway, tmp_path: Path) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
@ -183,8 +272,9 @@ def test_shared_client_claim_never_maps_a_second_user_onto_the_first_users_perso
assert read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (_hash(personal),)) == []
def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviving_mapped_key(
gateway: Gateway, tmp_path: Path
@pytest.mark.parametrize("behavior", ["auto_register", "link_only"])
def test_concurrent_first_jwt_calls_both_succeed_on_one_surviving_mapped_key(
gateway: Gateway, tmp_path: Path, behavior: Literal["auto_register", "link_only"]
) -> None:
writer_url: Final = os.environ.get("INTEGRATION_PROXY_DATABASE_URL") or os.environ["DATABASE_URL"]
with (
@ -194,6 +284,7 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
):
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
existing: Final = scenario.key(user_id=user) if behavior == "link_only" else None
token: Final = _token(private_key, user)
overrides: Final = {
"JWT_PUBLIC_KEY_URL": jwks_url,
@ -202,7 +293,12 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
}
with (
owned_proxy(gateway, tmp_path, overrides, config=_config(tmp_path, "sub")) as candidate,
owned_proxy(
gateway,
tmp_path,
overrides,
config=_config(tmp_path, "sub", behavior=behavior, map_existing_key=False),
) as candidate,
ThreadPoolExecutor(max_workers=1) as pool,
):
held_call: Final = pool.submit(_chat, candidate, model, token)
@ -215,5 +311,240 @@ def test_concurrent_first_jwt_calls_of_a_keyless_user_both_succeed_on_one_surviv
assert held.status_code == 200, held.text
keys: Final = _user_key_hashes(user)
assert len(keys) == 1, keys
if existing is not None:
assert keys == frozenset({_hash(existing)})
assert _mapped_token("sub", user) in keys
assert (_billed_key(held), _billed_key(racing)) == (_mapped_token("sub", user),) * 2
@pytest.mark.parametrize("ineligible_keys", [False, True])
def test_link_only_rejects_without_writes_then_links_a_newly_available_key(
gateway: Gateway, tmp_path: Path, ineligible_keys: bool
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
if ineligible_keys:
scenario.key(user_id=user, duration="1h")
scenario.key(user_id=user, metadata={"auto_registered": True})
scenario.key(user_id=user, team_id=scenario.team())
blocked: Final = scenario.key(user_id=user)
gateway.post("/key/block", {"key": blocked})
keys_before: Final = _user_key_hashes(user)
token: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
) as candidate:
first: Final = _chat(candidate, model, token)
repeated: Final = _chat(candidate, model, token)
assert first.status_code == 403, first.text
assert repeated.status_code == 403, repeated.text
assert _user_key_hashes(user) == keys_before
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
available: Final = scenario.key(user_id=user)
accepted: Final = _chat(candidate, model, token)
assert accepted.status_code == 200, accepted.text
assert _mapped_token("sub", user) == _hash(available)
assert _user_key_hashes(user) == keys_before | {_hash(available)}
assert _billed_key(accepted) == _hash(available)
@pytest.mark.parametrize("claim_field", ["client_id", "email"])
def test_link_only_rejects_shared_or_missing_claims_without_linking_a_personal_key(
gateway: Gateway, tmp_path: Path, claim_field: str
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
key: Final = scenario.key(user_id=user)
shared_claim: Final = f"integration-client-{uuid.uuid4().hex}"
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, claim_field, behavior="link_only", map_existing_key=False),
) as candidate:
response: Final = _chat(candidate, model, _token(private_key, user, client_id=shared_claim))
assert response.status_code == 403, response.text
assert _user_key_hashes(user) == frozenset({_hash(key)})
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE token=%s', (_hash(key),)) == []
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (shared_claim,)) == []
@pytest.mark.parametrize("restriction", ["models", "allowed_routes"])
def test_link_only_enforces_key_permissions_on_the_first_and_cached_requests(
gateway: Gateway, tmp_path: Path, restriction: str
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
allowed: Final = scenario.model()
denied: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
key: Final = scenario.key(
user_id=user,
**({"models": [allowed]} if restriction == "models" else {"allowed_routes": ["/v1/models"]}),
)
token: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
) as candidate:
first: Final = _chat(candidate, denied, token)
repeated: Final = _chat(candidate, denied, token)
control: Final = (
_chat(candidate, allowed, token)
if restriction == "models"
else candidate.request("GET", "/v1/models", key=token)
)
assert first.status_code == 403, first.text
assert repeated.status_code == 403, repeated.text
assert control.status_code == 200, control.text
assert _mapped_token("sub", user) == _hash(key)
assert _user_key_hashes(user) == frozenset({_hash(key)})
def test_link_only_does_not_replace_an_existing_mapping_to_a_blocked_key(gateway: Gateway, tmp_path: Path) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
blocked: Final = scenario.key(user_id=user)
available: Final = scenario.key(user_id=user)
gateway.post("/jwt/key/mapping/new", {"jwt_claim_name": "sub", "jwt_claim_value": user, "key": blocked})
gateway.post("/key/block", {"key": blocked})
token: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
) as candidate:
first: Final = _chat(candidate, model, token)
repeated: Final = _chat(candidate, model, token)
assert first.status_code == 401, first.text
assert repeated.status_code == 401, repeated.text
assert _mapped_token("sub", user) == _hash(blocked)
assert _user_key_hashes(user) == frozenset({_hash(blocked), _hash(available)})
def test_link_only_exempts_proxy_admins_but_rechecks_the_role_after_a_cached_exemption(
gateway: Gateway, tmp_path: Path
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
admin: Final = _token(private_key, user, scope="litellm_proxy_admin")
ordinary: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
) as candidate:
first: Final = candidate.request("GET", "/v1/models", key=admin)
repeated: Final = candidate.request("GET", "/v1/models", key=admin)
denied: Final = candidate.request("GET", "/v1/models", key=ordinary)
forbidden_route: Final = _chat(candidate, model, admin)
assert first.status_code == 200, first.text
assert repeated.status_code == 200, repeated.text
assert denied.status_code == 403, denied.text
assert forbidden_route.status_code == 401, forbidden_route.text
assert _user_key_hashes(user) == frozenset()
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
@pytest.mark.parametrize("user_id_upsert", [False, True])
def test_link_only_keeps_user_provisioning_independent_of_key_creation(
gateway: Gateway, tmp_path: Path, user_id_upsert: bool
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
subject: Final = f"integration-link-only-{uuid.uuid4().hex}"
try:
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(
tmp_path, "sub", behavior="link_only", map_existing_key=False, user_id_upsert=user_id_upsert
),
) as candidate:
response: Final = _chat(candidate, model, _token(private_key, subject))
assert response.status_code == (403 if user_id_upsert else 401), response.text
users: Final = read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,))
assert users == ([{"user_id": subject}] if user_id_upsert else [])
assert _user_key_hashes(subject) == frozenset()
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (subject,)) == []
finally:
if read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id=%s', (subject,)):
scenario.delete_user(subject)
@pytest.mark.parametrize("issuer_behavior", ["auto_register", "link_only"])
def test_issuer_specific_registration_mode_overrides_the_global_mode(
gateway: Gateway, tmp_path: Path, issuer_behavior: Literal["auto_register", "link_only"]
) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
issuer: Final = f"https://integration.example.com/{uuid.uuid4().hex}"
token: Final = _token(private_key, user, iss=issuer, aud="integration")
path: Final = _config(
tmp_path,
"sub",
behavior="auto_register" if issuer_behavior == "link_only" else "link_only",
map_existing_key=False,
issuers=(
{
"issuer": issuer,
"jwks_url": jwks_url,
"audience": "integration",
"user_id_jwt_field": "sub",
"virtual_key_claim_field": "sub",
"unregistered_jwt_client_behavior": issuer_behavior,
},
),
)
with owned_proxy(gateway, tmp_path, {"JWT_PUBLIC_KEY_URL": jwks_url}, config=path) as candidate:
response: Final = _chat(candidate, model, token)
assert response.status_code == (403 if issuer_behavior == "link_only" else 200), response.text
if issuer_behavior == "link_only":
assert _user_key_hashes(user) == frozenset()
assert read_rows('SELECT token FROM "LiteLLM_JWTKeyMapping" WHERE jwt_claim_value=%s', (user,)) == []
else:
assert len(_user_key_hashes(user)) == 1
assert _mapped_token("sub", user) in _user_key_hashes(user)
assert _billed_key(response) == _mapped_token("sub", user)
def test_link_only_links_after_a_cached_rejection_from_the_previous_mode(gateway: Gateway, tmp_path: Path) -> None:
with _issuer() as (private_key, jwks_url), gateway.scenario() as scenario:
model: Final = scenario.model()
user: Final = scenario.user(user_role="internal_user")
key: Final = scenario.key(user_id=user)
token: Final = _token(private_key, user)
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="reject", map_existing_key=False),
) as rejecting:
denied: Final = _chat(rejecting, model, token)
assert denied.status_code == 403, denied.text
with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache:
cached: Final = cache.get(f"jwt_key_mapping:sub:{user}")
assert cached is not None and b"__NO_MAPPING__" in cached, cached
with owned_proxy(
gateway,
tmp_path,
{"JWT_PUBLIC_KEY_URL": jwks_url},
config=_config(tmp_path, "sub", behavior="link_only", map_existing_key=False),
) as linking:
accepted: Final = _chat(linking, model, token)
assert accepted.status_code == 200, accepted.text
assert _mapped_token("sub", user) == _hash(key)
assert _user_key_hashes(user) == frozenset({_hash(key)})
assert _billed_key(accepted) == _hash(key)

View file

@ -0,0 +1,39 @@
import _hashlib
import json
from pathlib import Path
from typing import Final
import pytest
from tests.integration._support.client import Gateway
from tests.integration._support.process import refused_boot_log
def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path:
config: Final = directory / "fips_boot.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "fips-model",
"litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"},
}
],
"general_settings": {"fips_mode": fips_mode},
}
)
)
return config
def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None:
if _hashlib.get_fips_mode() == 1:
pytest.skip("this runner's OpenSSL is in FIPS mode")
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True))
assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log
def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None:
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced"))
assert "general_settings.fips_mode='enforced' is not a boolean" in log, log

View file

@ -122,6 +122,9 @@ def test_patch_rename_is_rejected_and_model_keeps_working(gateway: Gateway) -> N
"PATCH", f"/credentials/{name}", {"credential_name": name, "credential_info": {"description": "y"}}
)
assert same_name.status_code == 200, same_name.text
assert read_rows(
'SELECT credential_info FROM "LiteLLM_CredentialsTable" WHERE credential_name = %s', (name,)
) == [{"credential_info": {"custom_llm_provider": "openai", "description": "y"}}]
assert _served_credential(gateway, name)["credential_info"] == {
"custom_llm_provider": "openai",
"description": "y",

View file

@ -669,7 +669,7 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
import httpx2
import yaml
from integration._support.asgi import asgi_server
from integration._support.mcp import _drain_sse_streams
from integration._support.mcp import _acknowledge_notifications_after_dispatch, _drain_sse_streams
from integration._support.process import owned_proxy
from mcp import ClientSession, MCPError
from mcp.client.streamable_http import streamable_http_client
@ -769,7 +769,9 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
watchers = tuple(task for task in asyncio.all_tasks() if "_shutdown_watcher" in repr(task.get_coro()))
await asyncio.wait_for(asyncio.gather(*watchers), 2)
hosted: Final = Starlette(routes=[Mount("/", app=app)], lifespan=lifespan)
hosted: Final = Starlette(
routes=[Mount("/", app=_acknowledge_notifications_after_dispatch(app))], lifespan=lifespan
)
with asgi_server(hosted) as upstream_url, httpx.Client() as client:
config: Final = tmp_path / "proxy.yaml"
config.write_text(

View file

@ -0,0 +1,724 @@
import json
import os
import threading
import time
import uuid
from collections.abc import Callable, Generator, Mapping
from contextlib import contextmanager
from typing import Final, Literal
from urllib.parse import unquote
import anthropic
import httpx
import openai
import pytest
from integration._support.anthropic_sse import (
LIFECYCLE,
error_frame,
event_type,
event_types,
message_id,
message_json,
message_stream,
parse_sse,
status_reply,
stream_reply,
user_prompt,
)
from integration._support.client import (
JSON_OBJECT,
Gateway,
JsonValue,
Scenario,
eventually,
object_value,
string_value,
)
from integration._support.database import read_rows
from integration._support.openai_wire import openai_error
from integration._support.upstream import aws_event_stream_frame
from integration._support.wire import Reply, Request, Wire, wire_server
from redis import Redis
_MODEL: Final = "claude-sonnet-4-5-20250929"
_API_KEY: Final = "synthetic-anthropic-key"
_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
_BEDROCK_TOKEN: Final = "synthetic-bedrock-bearer"
_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}'
_CONVERSE_MODEL: Final = f"bedrock/converse/{_BEDROCK_MODEL}"
_CONVERSE_STREAM_TARGET: Final = f"/model/{_BEDROCK_MODEL}/converse-stream"
_OPENAI_BACKEND: Final = "gpt-4o-mini"
_CHAT_ONLY_BACKEND: Final = "audit-chat-only"
_CODEX_BACKEND: Final = "gpt-5.3-codex"
_TEXT: Final = "Hello"
_KEEPALIVE_SECONDS: Final = 0.2
_LATE_ANSWER_SECONDS: Final = 1.0
_OVERLOAD_CLASS: Final = "503_PROVIDER_OVERLOADED"
Shape = Literal["stream", "non-stream"]
def _prompt() -> str:
return "overloaded-" + uuid.uuid4().hex
def _always(reply: Reply) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert request.method == "POST", request
return reply
return respond
def _late(reply: Reply, seconds: float) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert request.method == "POST", request
time.sleep(seconds)
return reply
return respond
def _anthropic_sdk(gateway: Gateway) -> anthropic.Anthropic:
return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
def _async_anthropic_sdk(gateway: Gateway) -> anthropic.AsyncAnthropic:
return anthropic.AsyncAnthropic(
base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60
)
def _openai_sdk(gateway: Gateway) -> openai.OpenAI:
return openai.OpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
def _async_openai_sdk(gateway: Gateway) -> openai.AsyncOpenAI:
return openai.AsyncOpenAI(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0, timeout=60)
def _messages_body(model: str, prompt: str, *, stream: bool) -> dict[str, JsonValue]:
return {
"model": model,
"max_tokens": 16,
"stream": stream,
"messages": [{"role": "user", "content": prompt}],
"num_retries": 0,
}
def _anthropic_error(response: httpx.Response) -> dict[str, JsonValue]:
assert response.headers["content-type"].startswith("application/json"), dict(response.headers)
payload: Final = object_value(json.loads(response.content))
assert payload["type"] == "error", response.text
return object_value(payload["error"])
def _openai_error(response: httpx.Response) -> dict[str, JsonValue]:
return object_value(object_value(json.loads(response.content))["error"])
def _assert_overloaded(response: httpx.Response, *, message: str) -> None:
assert response.status_code == 529, response.text
error: Final = _anthropic_error(response)
assert error["type"] == "overloaded_error", response.text
assert message in string_value(error["message"]), response.text
assert "content_block_delta" not in response.text, response.text
def _assert_api_error(response: httpx.Response, *, status: int, message: str) -> None:
assert response.status_code == status, response.text
error: Final = _anthropic_error(response)
assert error["type"] == "api_error", response.text
assert message in string_value(error["message"]), response.text
def _assert_internal_server_error(response: httpx.Response) -> None:
assert response.status_code == 500, response.text
error: Final = _openai_error(response)
assert error["code"] == "500", response.text
assert "InternalServerError" in string_value(error["message"]), response.text
def _one_post(wire: Wire, target: str) -> None:
assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)]
def _spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
"SELECT status, metadata->'error_information'->>'error_code' AS code, "
"metadata->'error_information'->>'error_class' AS error_class, "
"metadata->'error_information'->>'normalized_error' AS normalized_error "
'FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
(model,),
),
lambda rows: len(rows) >= expected,
seconds=70,
)
def _assert_logged_failure(model: str, *, rows: int, error_class: str, code: str = "529") -> None:
assert (
_spend_rows(model, rows)
== [{"status": "failure", "code": code, "error_class": error_class, "normalized_error": _OVERLOAD_CLASS}] * rows
)
def test_messages_529_answers_overloaded_error_to_the_sync_sdk_and_logs_each_call_once(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
sdk: Final = _anthropic_sdk(gateway)
prompt: Final = _prompt()
for _ in range(2):
with pytest.raises(anthropic.APIStatusError) as caught:
sdk.messages.create(
model=model,
max_tokens=16,
messages=[{"role": "user", "content": prompt}],
extra_body={"num_retries": 0},
)
assert caught.value.status_code == 529, caught.value.response.text
_assert_overloaded(caught.value.response, message="scripted 529")
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2
_assert_logged_failure(model, rows=2, error_class="InternalServerError")
async def test_messages_529_answers_overloaded_error_to_the_async_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(anthropic.APIStatusError) as caught:
await _async_anthropic_sdk(gateway).messages.create(
model=model,
max_tokens=16,
messages=[{"role": "user", "content": _prompt()}],
extra_body={"num_retries": 0},
)
assert caught.value.status_code == 529, caught.value.response.text
_assert_overloaded(caught.value.response, message="scripted 529")
_one_post(wire, "/v1/messages")
@pytest.mark.parametrize(
("upstream_status", "error_class", "error_type"),
((529, "InternalServerError", "overloaded_error"), (503, "ServiceUnavailableError", "api_error")),
)
def test_messages_stream_rejected_before_its_first_byte_answers_the_provider_status_as_json(
gateway: Gateway, upstream_status: int, error_class: str, error_type: str
) -> None:
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
assert response.status_code == upstream_status, response.text
error: Final = _anthropic_error(response)
assert error["type"] == error_type, response.text
assert f"scripted {upstream_status}" in string_value(error["message"]), response.text
_one_post(wire, "/v1/messages")
_assert_logged_failure(model, rows=1, error_class=error_class, code=str(upstream_status))
def test_messages_stream_529_raises_529_in_the_sync_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(anthropic.APIStatusError) as caught:
with _anthropic_sdk(gateway).messages.stream(
model=model,
max_tokens=16,
messages=[{"role": "user", "content": _prompt()}],
extra_body={"num_retries": 0},
) as stream:
stream.get_final_message()
assert caught.value.status_code == 529, caught.value.response.text
_assert_overloaded(caught.value.response, message="scripted 529")
_one_post(wire, "/v1/messages")
def _error_frames(text: str) -> tuple[dict[str, JsonValue], ...]:
return tuple(
object_value(json.loads(line.removeprefix("data:").strip()))
for line in text.splitlines()
if line.startswith("data:") and '"error"' in line
)
def test_messages_529_after_the_keepalive_opened_the_response_ends_with_an_overloaded_error_event(
gateway: Gateway,
) -> None:
with (
wire_server(_late(status_reply(529), _LATE_ANSWER_SECONDS)) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, keepalive_seconds=_KEEPALIVE_SECONDS
)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
assert response.status_code == 200, response.text
assert response.headers["content-type"].startswith("text/event-stream"), dict(response.headers)
assert response.text.startswith(":"), response.text
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
assert failure.event == "error", response.text
assert object_value(failure.data["error"])["type"] == "overloaded_error", response.text
assert "[DONE]" not in response.text, response.text
_one_post(wire, "/v1/messages")
def _late_bedrock_deployment(scenario: Scenario, wire: Wire) -> str:
return scenario.model(
model=f"bedrock/{_BEDROCK_MODEL}",
api_key=_BEDROCK_TOKEN,
aws_region_name="us-east-1",
api_base=wire.url,
keepalive_seconds=_KEEPALIVE_SECONDS,
)
def _one_bedrock_post(wire: Wire) -> None:
(request,) = wire.drain()
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
def test_messages_bedrock_503_after_the_keepalive_opened_the_response_ends_with_an_api_error_event(
gateway: Gateway,
) -> None:
with (
wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire,
gateway.scenario() as scenario,
):
model: Final = _late_bedrock_deployment(scenario, wire)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
assert response.status_code == 200, response.text
assert response.text.startswith(":"), response.text
assert "[DONE]" not in response.text, response.text
(failure,) = (event for event in parse_sse(response.text) if event.data and event.event != "ping")
assert failure.event == "error", response.text
error: Final = object_value(failure.data["error"])
assert error["type"] == "api_error", response.text
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
_one_bedrock_post(wire)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def test_chat_completions_bedrock_503_after_the_keepalive_opened_the_response_keeps_the_openai_error_frame(
gateway: Gateway,
) -> None:
with (
wire_server(_late(Reply(status=503, body=_BEDROCK_503), _LATE_ANSWER_SECONDS)) as wire,
gateway.scenario() as scenario,
):
model: Final = _late_bedrock_deployment(scenario, wire)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0},
)
assert response.status_code == 200, response.text
assert response.text.startswith(":"), response.text
assert "event: error" not in response.text, response.text
(frame,) = _error_frames(response.text)
assert object_value(frame["error"])["code"] == "503", response.text
assert response.text.rstrip().endswith("data: [DONE]"), response.text
_one_bedrock_post(wire)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def test_chat_completions_overloaded_frame_after_content_ends_with_a_500_error_frame(gateway: Gateway) -> None:
prompt: Final = _prompt()
upstream: Final = (*message_stream(prompt, _MODEL, _TEXT)[:3], error_frame(529, "Overloaded"))
with wire_server(_always(stream_reply(upstream))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
)
assert response.status_code == 200, response.text
assert _TEXT in response.text, response.text
(frame,) = _error_frames(response.text)
error: Final = object_value(frame["error"])
assert error["code"] == "500", response.text
_one_post(wire, "/v1/messages")
def _overloaded_before_content(prompt: str) -> Reply:
return stream_reply((*message_stream(prompt, _MODEL, _TEXT)[:1], error_frame(529, "Overloaded")))
def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_relayed_as_an_overloaded_event(
gateway: Gateway,
) -> None:
prompt: Final = _prompt()
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=True))
assert response.status_code == 200, response.text
events: Final = parse_sse(response.text)
assert event_types(events) == ("message_start", "error"), response.text
assert message_id(events) == prompt, response.text
assert events[1].data == {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
_one_post(wire, "/v1/messages")
assert _spend_rows(model, 1) == [
{"status": "success", "code": None, "error_class": None, "normalized_error": None}
]
def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None:
prompt: Final = _prompt()
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
)
_assert_internal_server_error(response)
_one_post(wire, "/v1/messages")
def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(openai.APIStatusError) as caught:
_openai_sdk(gateway).chat.completions.create(
model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0}
)
assert caught.value.status_code == 500, caught.value.response.text
_assert_internal_server_error(caught.value.response)
_one_post(wire, "/v1/messages")
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500")
async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(openai.APIStatusError) as caught:
await _async_openai_sdk(gateway).chat.completions.create(
model=model,
messages=[{"role": "user", "content": _prompt()}],
stream=True,
extra_body={"num_retries": 0},
)
assert caught.value.status_code == 500, caught.value.response.text
_assert_internal_server_error(caught.value.response)
_one_post(wire, "/v1/messages")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0}
)
_assert_internal_server_error(response)
_one_post(wire, "/v1/messages")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_messages_bedrock_503_answers_503_api_error(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(Reply(status=503, body=_BEDROCK_503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{_BEDROCK_MODEL}", api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url
)
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_api_error(response, status=503, message="Bedrock is unable to process your request.")
(request,) = wire.drain()
assert unquote(request.target).startswith(f"/model/{_BEDROCK_MODEL}/"), request
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def _converse_event(event: str, payload: Mapping[str, JsonValue]) -> bytes:
return aws_event_stream_frame(
{":event-type": event, ":content-type": "application/json", ":message-type": "event"},
json.dumps(payload).encode(),
)
def _converse_503_after_text() -> Reply:
unavailable: Final = aws_event_stream_frame(
{
":exception-type": "serviceUnavailableException",
":content-type": "application/json",
":message-type": "exception",
},
_BEDROCK_503,
)
frames: Final = (
_converse_event("messageStart", {"role": "assistant"}),
_converse_event("contentBlockDelta", {"delta": {"text": _TEXT}, "contentBlockIndex": 0}),
unavailable,
)
return Reply(body=b"".join(frames), content_type="application/vnd.amazon.eventstream")
def _converse_deployment(scenario: Scenario, wire: Wire) -> str:
return scenario.model(model=_CONVERSE_MODEL, api_key=_BEDROCK_TOKEN, aws_region_name="us-east-1", api_base=wire.url)
def test_messages_bedrock_stream_503_after_text_ends_with_an_api_error_event(gateway: Gateway) -> None:
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
model: Final = _converse_deployment(scenario, wire)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
assert response.status_code == 200, response.text
events: Final = parse_sse(response.text)
types: Final = event_types(events)
assert "message_stop" not in types, response.text
assert types.index("content_block_delta") < types.index("error"), response.text
assert _TEXT in response.text, response.text
(failure,) = (event for event in events if event_type(event) == "error")
error: Final = object_value(failure.data["error"])
assert error["type"] == "api_error", response.text
assert "Bedrock is unable to process your request." in string_value(error["message"]), response.text
_one_post(wire, _CONVERSE_STREAM_TARGET)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
def test_chat_completions_bedrock_stream_503_after_text_ends_with_a_503_error_frame(gateway: Gateway) -> None:
with wire_server(_always(_converse_503_after_text())) as wire, gateway.scenario() as scenario:
model: Final = _converse_deployment(scenario, wire)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _prompt()}], "stream": True, "num_retries": 0},
)
assert response.status_code == 200, response.text
assert _TEXT in response.text, response.text
(frame,) = _error_frames(response.text)
error: Final = object_value(frame["error"])
assert error["code"] == "503", response.text
_one_post(wire, _CONVERSE_STREAM_TARGET)
_assert_logged_failure(model, rows=1, error_class="ServiceUnavailableError", code="503")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_chat_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"hosted_vllm/{_CHAT_ONLY_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-vllm-key"
)
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_api_error(response, status=503, message="scripted 503")
_one_post(wire, "/v1/chat/completions")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_responses_bridge_503_answers_503_api_error_on_messages(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(openai_error(503))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"openai/{_CODEX_BACKEND}", api_base=wire.url + "/v1", api_key="synthetic-openai-key"
)
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
_assert_api_error(response, status=503, message="scripted 503")
_one_post(wire, "/v1/responses")
_SETTLED_ANSWERS: Final = 4
def _messages_to(gateway: Gateway, model: str) -> tuple[tuple[str, httpx.Response], ...]:
prompts: Final = tuple(_prompt() for _ in range(_SETTLED_ANSWERS))
return tuple(
(prompt, gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False)))
for prompt in prompts
)
def _failure_rows_of(models: tuple[str, str], expected: int) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
"SELECT metadata->'error_information'->>'error_code' AS code, "
"metadata->'error_information'->>'error_class' AS error_class, "
"metadata->'error_information'->>'normalized_error' AS normalized_error "
'FROM "LiteLLM_SpendLogs" WHERE status = %s AND model_group IN (%s, %s)',
("failure", *models),
),
lambda rows: len(rows) >= expected,
seconds=70,
)
@contextmanager
def _paused_deployment(gateway: Gateway) -> Generator[tuple[Wire, str, str]]:
unpaused: Final = Reply(body=message_json("msg_unpaused", _MODEL, _TEXT))
with wire_server(_always(unpaused)) as wire, gateway.scenario() as scenario:
identity: Final = f"audit-paused-{uuid.uuid4().hex}"
name: Final = scenario.model(
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, model_info={"id": identity}
)
paused: Final = gateway.request("PATCH", f"/model/{identity}/update", {"blocked": True})
assert paused.status_code == 200, paused.text
yield wire, identity, name
def _assert_refused_without_an_upstream_call(
gateway: Gateway, wire: Wire, target: str, *, status: int, error_type: str, message: str
) -> None:
answers: Final = eventually(
lambda: _messages_to(gateway, target),
lambda settled: all(response.status_code != 200 for _, response in settled),
seconds=90,
)
texts: Final = [response.text for _, response in answers]
assert [response.status_code for _, response in answers] == [status] * _SETTLED_ANSWERS, texts
errors: Final = [_anthropic_error(response) for _, response in answers]
assert [error["type"] for error in errors] == [error_type] * _SETTLED_ANSWERS, texts
assert all(message in string_value(error["message"]) for error in errors), texts
reached: Final = {user_prompt(JSON_OBJECT.validate_json(request.body)) for request in wire.drain()}
assert reached.isdisjoint(prompt for prompt, _ in answers), reached
def test_messages_to_a_paused_deployment_named_by_its_id_answer_503_api_error(gateway: Gateway) -> None:
with _paused_deployment(gateway) as (wire, identity, name):
_assert_refused_without_an_upstream_call(
gateway, wire, identity, status=503, error_type="api_error", message="is currently paused"
)
rows: Final = _failure_rows_of((identity, name), _SETTLED_ANSWERS)
assert {(row["code"], row["error_class"], row["normalized_error"]) for row in rows} == {
("503", "ServiceUnavailableError", _OVERLOAD_CLASS)
}, rows
def test_messages_to_a_paused_model_group_named_by_its_name_answer_403_permission_error(gateway: Gateway) -> None:
with _paused_deployment(gateway) as (wire, _, name):
_assert_refused_without_an_upstream_call(
gateway, wire, name, status=403, error_type="permission_error", message="Model is blocked"
)
_MALFORMED_529_BODIES: Final = {
"empty": b"",
"text-5k": b"x" * 5000,
"html": b"<html><body><h1>529</h1></body></html>",
"api_error-type": b'{"type": "error", "error": {"type": "api_error", "message": "capacity"}}',
}
def _overloaded_for_sad_prompts(body: bytes) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
prompt: Final = user_prompt(object_value(json.loads(request.body)))
if prompt.startswith("sad:"):
return Reply(status=529, body=body, content_type="text/plain" if body[:1] != b"{" else "application/json")
return Reply(body=message_json(f"msg_{prompt}", _MODEL, _TEXT))
return respond
@pytest.mark.parametrize("body", _MALFORMED_529_BODIES.keys())
def test_messages_529_with_a_malformed_body_still_answers_overloaded_error_and_unrelated_calls_keep_working(
gateway: Gateway, body: str
) -> None:
with wire_server(_overloaded_for_sad_prompts(_MALFORMED_529_BODIES[body])) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
sad: Final = gateway.request("POST", "/v1/messages", _messages_body(model, "sad:" + _prompt(), stream=False))
assert sad.status_code == 529, sad.text
assert _anthropic_error(sad)["type"] == "overloaded_error", sad.text
prompt: Final = _prompt()
healthy: Final = gateway.request("POST", "/v1/messages", _messages_body(model, prompt, stream=False))
assert healthy.status_code == 200, healthy.text
assert object_value(json.loads(healthy.content))["id"] == f"msg_{prompt}", healthy.text
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2
assert sorted(_spend_rows(model, 2), key=lambda row: str(row["status"])) == [
{
"status": "failure",
"code": "529",
"error_class": "InternalServerError",
"normalized_error": _OVERLOAD_CLASS,
},
{"status": "success", "code": None, "error_class": None, "normalized_error": None},
]
@pytest.mark.parametrize(("upstream_status", "error_type"), ((500, "api_error"), (429, "rate_limit_error")))
def test_messages_other_upstream_statuses_keep_their_status_and_type(
gateway: Gateway, upstream_status: int, error_type: str
) -> None:
with wire_server(_always(status_reply(upstream_status))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
assert response.status_code == upstream_status, response.text
assert _anthropic_error(response)["type"] == error_type, response.text
_one_post(wire, "/v1/messages")
def _anthropic_rejection(status: int, error_type: str, message: str) -> Reply:
return Reply(
status=status, body=json.dumps({"type": "error", "error": {"type": error_type, "message": message}}).encode()
)
_REJECTIONS_SAYING_OVERLOADED: Final = {
"400-tool-name": _anthropic_rejection(
400, "invalid_request_error", "tools.0.name: 'Overloaded' is not a valid tool name"
),
"404-model": _anthropic_rejection(404, "not_found_error", "model: claude-Overloaded"),
}
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
@pytest.mark.parametrize("rejection", _REJECTIONS_SAYING_OVERLOADED.keys())
def test_messages_4xx_whose_text_says_overloaded_is_not_answered_as_an_overload(
gateway: Gateway, rejection: str, shape: Shape
) -> None:
with wire_server(_always(_REJECTIONS_SAYING_OVERLOADED[rejection])) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/messages", _messages_body(model, _prompt(), stream=shape == "stream")
)
assert response.status_code == 500, response.text
error: Final = _anthropic_error(response)
assert error["type"] == "api_error", response.text
assert "Overloaded" in string_value(error["message"]), response.text
_one_post(wire, "/v1/messages")
assert _spend_rows(model, 1) == [
{
"status": "failure",
"code": "500",
"error_class": "InternalServerError",
"normalized_error": _OVERLOAD_CLASS,
}
]
def _string_values(cache: Redis) -> tuple[bytes, ...]:
keys: Final = tuple(key for key in cache.scan_iter(count=1000) if cache.type(key) == b"string")
return tuple(value for value in cache.mget(keys) if value is not None) if keys else ()
def _cached_somewhere(served_id: str) -> bool:
with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache:
return any(served_id.encode() in value for value in _string_values(cache))
def _healthy_until(overloaded: threading.Event) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
if overloaded.is_set():
return status_reply(529)
prompt: Final = user_prompt(object_value(json.loads(request.body)))
return stream_reply(message_stream(f"msg_{prompt}", _MODEL, _TEXT))
return respond
def test_messages_cache_hit_is_served_while_the_upstream_is_overloaded(gateway: Gateway) -> None:
overloaded: Final = threading.Event()
with wire_server(_healthy_until(overloaded)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
prompt: Final = _prompt()
body: Final = _messages_body(model, prompt, stream=True)
first: Final = gateway.request("POST", "/v1/messages", body)
assert first.status_code == 200, first.text
assert event_types(parse_sse(first.text)) == LIFECYCLE, first.text
assert message_id(parse_sse(first.text)) == f"msg_{prompt}", first.text
eventually(lambda: _cached_somewhere(f"msg_{prompt}"), bool, seconds=30)
overloaded.set()
replay: Final = gateway.request("POST", "/v1/messages", body)
assert replay.status_code == 200, replay.text
assert message_id(parse_sse(replay.text)) == f"msg_{prompt}", replay.text
fresh: Final = gateway.request("POST", "/v1/messages", _messages_body(model, _prompt(), stream=True))
_assert_overloaded(fresh, message="scripted 529")
assert [unquote(request.target) for request in wire.drain()] == ["/v1/messages"] * 2

View file

@ -135,6 +135,13 @@ def _drop_or_500(request: Request, marker: str) -> Reply:
return openai_error(500)
def _overloaded_while(outage: threading.Event) -> Callable[[Request, str], Reply]:
def first_attempt(request: Request, marker: str) -> Reply:
return status_reply(529) if outage.is_set() else _served(request, marker, 1)
return first_attempt
def _outage(statuses: Mapping[str, int]) -> Callable[[Request, str], Reply]:
def first_attempt(request: Request, marker: str) -> Reply:
if request.target == "/v1/messages":
@ -165,7 +172,7 @@ class _Upstream:
return self.first_attempt(request, marker)
def _config(wire: Wire, directory: Path, models: _Models) -> Path:
def _config(wire: Wire, directory: Path, models: _Models, *, retries: int = 1) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["model_list"] = [
{
@ -174,7 +181,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path:
"model": f"anthropic/{_ANTHROPIC_BACKEND}",
"api_base": wire.url,
"api_key": _ANTHROPIC_KEY,
"num_retries": 1,
"num_retries": retries,
},
},
{
@ -183,7 +190,7 @@ def _config(wire: Wire, directory: Path, models: _Models) -> Path:
"model": f"openai/{_OPENAI_BACKEND}",
"api_base": wire.url + "/v1",
"api_key": _OPENAI_KEY,
"num_retries": 1,
"num_retries": retries,
},
},
]
@ -408,3 +415,68 @@ async def test_outage_on_every_first_attempt_is_absorbed_by_the_deployment_budge
_assert_completed_on_the_second_attempt(item)
_assert_each_served_id_landed_exactly_once(models, served)
assert all(upstream.attempts.count(call.marker) == 2 for call in calls)
def _messages_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
"SELECT request_id, status, metadata->'error_information'->>'error_code' AS code "
'FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
(model,),
),
lambda found: len(found) >= expected,
seconds=90,
)
@pytest.mark.timeout(300)
async def test_overloaded_outage_during_a_burst_with_a_worker_sigkill_answers_529_and_lands_every_call_once(
gateway: Gateway, tmp_path: Path
) -> None:
models: Final = _models()
calls: Final = _calls((("messages", True, 12), ("messages", False, 6)))
release: Final = threading.Event()
held: Final[SimpleQueue[str]] = SimpleQueue()
outage: Final = threading.Event()
outage.set()
upstream: Final = _Upstream(Attempts(), _overloaded_while(outage), held, release)
with wire_server(upstream) as wire:
config: Final = _config(wire, tmp_path, models, retries=0)
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned:
candidate: Final = owned.gateway
base_url: Final = str(candidate.client.base_url)
workers: Final = eventually(
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
lambda pids: len(pids) == 2,
seconds=30,
)
burst: Final = asyncio.create_task(
_burst(base_url, candidate.key, models, calls, tolerate_transport_errors=True)
)
await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(calls), 90)
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
assert sum(held_by.values()) == len(calls), held_by
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
with owned.replacing_worker(victim_pid) as kill:
kill.freeze()
await asyncio.to_thread(kill)
release.set()
served: Final = await burst
assert held_by[survivor_pid] >= 5, held_by
assert len(served) == held_by[survivor_pid], (held_by, len(served))
for item in served:
assert item.status == 529, (item.call, item.text)
assert "overloaded_error" in item.text, (item.call, item.text)
assert "content_block_delta" not in item.text, (item.call, item.text)
outage.clear()
follow_up: Final = _Call(endpoint="messages", stream=True, marker=uuid.uuid4().hex)
(answered,) = await _burst(base_url, candidate.key, models, (follow_up,))
assert answered.status == 200, answered.text
assert message_id(parse_sse(answered.text)) == f"msg_{follow_up.marker}_a1", answered.text
rows: Final = _messages_rows(models.messages, len(served) + 1)
assert sorted(string_value(row["status"]) for row in rows) == ["failure"] * len(served) + ["success"], rows
assert {row["code"] for row in rows if row["status"] == "failure"} == {"529"}, rows
assert [row["request_id"] for row in rows if row["status"] == "success"] == [
f"msg_{follow_up.marker}_a1"
], rows
assert all(upstream.attempts.count(item.call.marker) == 1 for item in (*served, answered))

View file

@ -397,14 +397,14 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway:
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1)
response: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "deployment"))
_assert_rejected_before_content(response, wire, status=500, attempts=2, error="error")
_assert_rejected_before_content(response, wire, status=529, attempts=2, error="overloaded_error")
rows: Final = eventually(
lambda: _request_rows(gateway, response.headers["x-litellm-call-id"]),
lambda values: len(values) >= 1,
seconds=70,
)
rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0))
_assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error")
_assert_rejected_before_content(rejected, wire, status=529, attempts=1, error="overloaded_error")
rejection_rows: Final = eventually(
lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]),
lambda values: len(values) >= 1,

View file

@ -0,0 +1,404 @@
import gzip
import json
import time
import uuid
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from urllib.parse import unquote
import httpx
import pytest
import yaml
from integration._support.anthropic_sse import error_body
from integration._support.client import (
JSON_OBJECT,
Gateway,
JsonValue,
eventually,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.otlp_sink import Span, SpanSinks, recorded_spans, spans_for_trace
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.prometheus_series import Sample, scrape
from integration._support.wire import Reply, Request, Wire, wire_server
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_MODEL: Final = "claude-sonnet-4-5-20250929"
_BEDROCK_MODEL: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"
_BEDROCK_503: Final = b'{"message": "Bedrock is unable to process your request."}'
_MESSAGES_ANTHROPIC: Final = "audit-signals-messages-anthropic"
_MESSAGES_BEDROCK: Final = "audit-signals-messages-bedrock"
_CHAT_ANTHROPIC: Final = "audit-signals-chat-anthropic"
_TRACED_MESSAGES: Final = "audit-signals-traced-messages"
_TRACED_CHAT: Final = "audit-signals-traced-chat"
_DATADOG_MESSAGES_ANTHROPIC: Final = "audit-signals-datadog-messages-anthropic"
_DATADOG_MESSAGES_BEDROCK: Final = "audit-signals-datadog-messages-bedrock"
_DATADOG_CHAT_ANTHROPIC: Final = "audit-signals-datadog-chat-anthropic"
_GUARDED_MESSAGES: Final = "audit-signals-guarded-messages"
_GUARDED_CHAT: Final = "audit-signals-guarded-chat"
_AKTO_DOWN: Final = "audit-signals-akto-down"
_AKTO_UNREACHABLE: Final = "Akto guardrail service unreachable"
_LATE_PASS_THROUGH_BACKEND: Final = "claude-late-pass-through"
_KEEPALIVE_SECONDS: Final = 0.2
_LATE_ANSWER_SECONDS: Final = 1.0
_OVERLOADED: Final = Reply(status=529, body=error_body(529, "Overloaded"))
_SERVER: Final = 2
AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
@dataclass(frozen=True, slots=True)
class _Rig:
proxy: Gateway
wire: Wire
spans: str
datadog: Wire
def _respond(request: Request) -> Reply:
assert request.method == "POST", request
if unquote(request.target).startswith("/model/"):
return Reply(status=503, body=_BEDROCK_503)
if json.loads(request.body).get("model") == _LATE_PASS_THROUGH_BACKEND:
time.sleep(_LATE_ANSWER_SECONDS)
return _OVERLOADED
def _datadog_intake(request: Request) -> Reply:
assert request.target == "/api/v2/logs", request.target
return Reply(status=202, body=b"{}")
def _config(directory: Path, tracing: Path, wire: Wire) -> Path:
configuration: Final = JSON_OBJECT.validate_python(yaml.safe_load(tracing.read_text()))
anthropic: Final = {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": "synthetic-anthropic-key"}
bedrock: Final = {
"model": f"bedrock/{_BEDROCK_MODEL}",
"api_base": wire.url,
"api_key": "synthetic-bedrock-bearer",
"aws_region_name": "us-east-1",
}
deployments: Final = (
(_MESSAGES_ANTHROPIC, anthropic),
(_MESSAGES_BEDROCK, bedrock),
(_CHAT_ANTHROPIC, anthropic),
(_TRACED_MESSAGES, anthropic),
(_TRACED_CHAT, anthropic),
(_DATADOG_MESSAGES_ANTHROPIC, anthropic),
(_DATADOG_MESSAGES_BEDROCK, bedrock),
(_DATADOG_CHAT_ANTHROPIC, anthropic),
(_GUARDED_MESSAGES, anthropic),
(_GUARDED_CHAT, anthropic),
)
akto_down: Final = {
"guardrail_name": _AKTO_DOWN,
"litellm_params": {
"guardrail": "akto",
"mode": "pre_call",
"akto_base_url": "http://127.0.0.1:9",
"akto_api_key": "synthetic-akto-key",
"unreachable_fallback": "fail_closed",
},
}
path: Final = directory / "anthropic-overloaded-failure-signals.yaml"
path.write_text(
yaml.safe_dump(
{
**configuration,
"model_list": [{"model_name": name, "litellm_params": params} for name, params in deployments],
"router_settings": {**object_value(configuration.get("router_settings", {})), "num_retries": 0},
"litellm_settings": {
**object_value(configuration["litellm_settings"]),
"sse_keepalive_ping_interval_seconds": _KEEPALIVE_SECONDS,
},
"guardrails": [akto_down],
}
)
)
return path
@pytest.fixture(scope="module")
def rig(
tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter
) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals")
multiprocess: Final = directory / "prometheus"
multiprocess.mkdir()
tracing: Final = otel_audit_config(directory, {"callbacks": ["otel", "prometheus", "datadog"]})
with (
wire_server(_respond) as wire,
wire_server(_datadog_intake) as datadog,
gateway_from_environment() as shared,
owned_proxy(
shared,
directory,
{
"OTEL_EXPORTER": "http/json",
"OTEL_ENDPOINT": audit_sinks.operator,
"PROMETHEUS_MULTIPROC_DIR": str(multiprocess),
"DD_API_KEY": "synthetic-datadog-key",
"DD_SITE": "datadog.invalid",
"DD_BASE_URL": datadog.url,
"DD_BATCH_SIZE": "1",
"ANTHROPIC_API_BASE": wire.url,
"ANTHROPIC_API_KEY": "synthetic-anthropic-key",
},
config=_config(directory, tracing, wire),
workers=2,
) as owned,
):
yield _Rig(owned, wire, audit_sinks.operator, datadog)
@pytest.fixture(scope="module")
def v2_rig(
tmp_path_factory: pytest.TempPathFactory, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter
) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-failure-signals-v2")
tracing: Final = otel_audit_config(directory, {"callbacks": ["otel"]})
with (
wire_server(_respond) as wire,
wire_server(_datadog_intake) as datadog,
gateway_from_environment() as shared,
owned_proxy(
shared,
directory,
{"OTEL_EXPORTER": "http/json", "OTEL_ENDPOINT": audit_sinks.operator, "LITELLM_OTEL_V2": "1"},
config=_config(directory, tracing, wire),
workers=2,
) as owned,
):
yield _Rig(owned, wire, audit_sinks.operator, datadog)
def _send(
rig: _Rig,
route: str,
model: str,
trace_id: str | None = None,
*,
key: str | None = None,
extra: Mapping[str, JsonValue] | None = None,
) -> httpx.Response:
traceparent: Final = {} if trace_id is None else {"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}
return rig.proxy.request(
"POST",
route,
{
"model": model,
"max_tokens": 16,
"messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}],
"num_retries": 0,
**(extra or {}),
},
key=key,
headers={"anthropic-version": "2023-06-01", **traceparent},
)
def _series(samples: Sequence[Sample], name: str, model: str) -> tuple[Sample, ...]:
return tuple(sample for sample in samples if sample.name == name and sample.labels.get("requested_model") == model)
def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
return eventually(lambda: _series(scrape(rig.proxy), name, model), bool, seconds=30)
@pytest.mark.parametrize(
("route", "model", "answered", "deployment_status", "deployment_class"),
(
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
),
)
def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status(
rig: _Rig, route: str, model: str, answered: int, deployment_status: str, deployment_class: str
) -> None:
rig.wire.drain()
response: Final = _send(rig, route, model)
assert response.status_code == answered, response.text
assert len(rig.wire.drain()) == 1
failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model)
assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [
(str(answered), route, 1.0)
], failed
deployment: Final = _until(rig, "litellm_deployment_failure_responses_total", model)
assert [(sample.labels["exception_status"], sample.value) for sample in deployment] == [(deployment_status, 1.0)], (
deployment
)
assert all(deployment_class in sample.labels["exception_class"] for sample in deployment), deployment
def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]:
traced: Final = spans_for_trace(recorded_spans(rig.spans, since)[1], trace_id)
return tuple(span for span in traced if span["kind"] == _SERVER)
@pytest.mark.parametrize(
("route", "model", "answered"),
(
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"),
),
)
def test_the_server_span_carries_the_status_the_route_answered(
rig: _Rig, route: str, model: str, answered: int
) -> None:
rig.wire.drain()
since, _ = recorded_spans(rig.spans)
trace_id: Final = uuid.uuid4().hex
response: Final = _send(rig, route, model, trace_id)
assert response.status_code == answered, response.text
assert len(rig.wire.drain()) == 1
server: Final = eventually(
lambda: _server_spans(rig, since, trace_id),
lambda spans: any("error.code" in span["attributes"] for span in spans),
seconds=40,
return_last_on_timeout=True,
)
assert [
(str(span["attributes"].get("http.response.status_code")), str(span["attributes"].get("error.code")))
for span in server
] == [(str(answered), str(answered))], server
def _inflated(request: Request) -> bytes:
return gzip.decompress(request.body) if request.headers.get("content-encoding") == "gzip" else request.body
def _datadog_events(batches: Sequence[Request]) -> Iterator[JsonValue]:
for batch in batches:
yield from json.loads(_inflated(batch))
def _datadog_messages(batches: Sequence[Request]) -> tuple[dict[str, JsonValue], ...]:
return tuple(
JSON_OBJECT.validate_json(string_value(object_value(event)["message"])) for event in _datadog_events(batches)
)
def _keyed_by(message: Mapping[str, JsonValue], alias: str) -> bool:
proxy_side: Final = message.get("user_api_key_dict")
llm_side: Final = message.get("metadata")
return any(isinstance(side, dict) and side.get("user_api_key_alias") == alias for side in (proxy_side, llm_side))
def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[str, str], ...]:
batches: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls must keep earlier batches
def delivered() -> tuple[tuple[str, str], ...]:
batches.extend(rig.datadog.drain())
return tuple(
sorted(
("proxy", str(message["status_code"]))
if "user_api_key_dict" in message
else ("llm", str(object_value(message["error_information"])["error_code"]))
for message in _datadog_messages(batches)
if _keyed_by(message, alias)
)
)
return eventually(delivered, lambda codes: len(codes) >= expected, seconds=40, return_last_on_timeout=True)
@pytest.mark.parametrize(
("route", "model", "answered", "deployment_status"),
(
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
),
)
def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status(
rig: _Rig, route: str, model: str, answered: int, deployment_status: str
) -> None:
rig.wire.drain()
rig.datadog.drain()
alias: Final = f"audit-signals-datadog-{uuid.uuid4().hex}"
with rig.proxy.scenario() as scenario:
response: Final = _send(rig, route, model, key=scenario.key(key_alias=alias, models=[model]))
assert response.status_code == answered, response.text
assert len(rig.wire.drain()) == 1
assert _datadog_status_codes(rig, alias, 2) == (("llm", deployment_status), ("proxy", str(answered)))
@pytest.mark.parametrize(
("route", "model", "answered"),
(
pytest.param("/v1/messages", _GUARDED_MESSAGES, 503, id="messages"),
pytest.param("/v1/chat/completions", _GUARDED_CHAT, 503, id="chat"),
),
)
def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes_status(
rig: _Rig, route: str, model: str, answered: int
) -> None:
rig.wire.drain()
rig.datadog.drain()
alias: Final = f"audit-signals-guarded-{uuid.uuid4().hex}"
with rig.proxy.scenario() as scenario:
response: Final = _send(
rig, route, model, key=scenario.key(key_alias=alias, models=[model]), extra={"guardrails": [_AKTO_DOWN]}
)
assert response.status_code == answered, response.text
assert _AKTO_UNREACHABLE in response.text, response.text
assert rig.wire.drain() == ()
failed: Final = _until(rig, "litellm_proxy_failed_requests_metric_total", model)
assert [(sample.labels["exception_status"], sample.labels["route"], sample.value) for sample in failed] == [
(str(answered), route, 1.0)
], failed
assert _datadog_status_codes(rig, alias, 2) == (("llm", str(answered)), ("proxy", str(answered)))
@pytest.mark.parametrize(
("route", "model", "answered"),
(
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
),
)
def test_the_otel_v2_server_span_carries_the_status_the_route_answered(
v2_rig: _Rig, route: str, model: str, answered: int
) -> None:
v2_rig.wire.drain()
since, _ = recorded_spans(v2_rig.spans)
trace_id: Final = uuid.uuid4().hex
response: Final = _send(v2_rig, route, model, trace_id)
assert response.status_code == answered, response.text
assert len(v2_rig.wire.drain()) == 1
server: Final = eventually(
lambda: _server_spans(v2_rig, since, trace_id),
lambda spans: any("litellm.provider.error.code" in span["attributes"] for span in spans),
seconds=40,
return_last_on_timeout=True,
)
assert [str(span["attributes"].get("litellm.provider.error.code")) for span in server] == [str(answered)], server
def test_an_anthropic_pass_through_stream_that_fails_after_the_keepalive_opened_relays_the_upstream_body(
rig: _Rig,
) -> None:
rig.wire.drain()
response: Final = rig.proxy.request(
"POST",
"/anthropic/v1/messages",
{
"model": _LATE_PASS_THROUGH_BACKEND,
"max_tokens": 16,
"stream": True,
"messages": [{"role": "user", "content": f"signals-{uuid.uuid4().hex}"}],
},
headers={"anthropic-version": "2023-06-01"},
)
assert response.status_code == 200, response.text
pings, _, relayed = response.text.rpartition("\n\n")
assert pings and set(pings.split("\n")) <= {"", ": ping"}, response.text
assert relayed == _OVERLOADED.body.decode(), response.text
assert [unquote(request.target) for request in rig.wire.drain()] == ["/v1/messages"]

View file

@ -5,6 +5,7 @@ from pathlib import Path
from typing import Final
import pytest
from litellm import get_model_info
from integration._support.client import Gateway, object_value
from integration._support.process import owned_proxy
from pydantic import JsonValue
@ -104,6 +105,21 @@ def test_v1_models_carries_cost_map_context_window_for_a_known_model(gateway: Ga
assert listed["max_output_tokens"] == 16384, listed
def test_v1_models_keeps_a_sibling_override_off_a_plain_deployment_of_the_same_model(gateway: Gateway) -> None:
catalog_info: Final = get_model_info("openai/gpt-4o-mini")
with gateway.scenario() as scenario:
overridden: Final = scenario.model(
model="openai/gpt-4o-mini",
model_info={"max_input_tokens": 4321, "max_output_tokens": 987},
)
plain: Final = scenario.model(model="openai/gpt-4o-mini")
listed_override: Final = _listed_model(gateway, overridden)
listed_plain: Final = _listed_model(gateway, plain)
assert _limits(listed_plain) == (catalog_info["max_input_tokens"], catalog_info["max_output_tokens"])
assert _limits(listed_override) == (4321, 987)
def test_v1_models_carries_deployment_model_info_limits_for_an_unknown_model(gateway: Gateway) -> None:
unknown: Final = f"openai/custom-{uuid.uuid4().hex}"
with gateway.scenario() as scenario:

View file

@ -419,7 +419,7 @@ class _LoginDeployment:
_LOGIN_DEPLOYMENTS: Final = (
_LoginDeployment("github_copilot", f"github_copilot/{_CODEX}", None),
_LoginDeployment("chatgpt", f"chatgpt/{_CODEX}", None),
_LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3", "synthetic-azure-key"),
_LoginDeployment("azure_ai-foundry-host", "azure_ai/deepseek-v3.2", "synthetic-azure-key"),
)

View file

@ -0,0 +1,121 @@
import json
import os
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, gateway_from_environment
from integration._support.forward_proxy import ForwardProxy, tunnelling_forward_proxy
from integration._support.process import owned_proxy
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.wire import Reply, Request, Wire, wire_server
_LOCATION: Final = "us-central1"
_HOST: Final = f"{_LOCATION}-aiplatform.googleapis.com"
_PROJECT: Final = "audit-vertex-pass-through"
_STREAMING_MODEL: Final = "gemini-3.8-flash"
_UNAVAILABLE_MODEL: Final = "gemini-3.8-flash-unavailable"
_VENDOR_TOKEN: Final = "Bearer synthetic-vertex-token"
_CHUNKS: Final = (
b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "Hel"}]}}]}\r\n\r\n',
b'data: {"candidates": [{"content": {"role": "model", "parts": [{"text": "lo"}]}, "finishReason": "STOP"}],'
b' "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}}\r\n\r\n',
)
_UNAVAILABLE_MESSAGE: Final = "The service is currently unavailable."
_UNAVAILABLE_BODY: Final = json.dumps(
{"error": {"code": 503, "message": _UNAVAILABLE_MESSAGE, "status": "UNAVAILABLE"}}
).encode()
_SCRUBBED: Final = frozenset(
{
"HTTPS_PROXY",
"HTTP_PROXY",
"ALL_PROXY",
"NO_PROXY",
"SSL_CERT_FILE",
"DEFAULT_VERTEXAI_PROJECT",
"DEFAULT_VERTEXAI_LOCATION",
"DEFAULT_GOOGLE_APPLICATION_CREDENTIALS",
"GOOGLE_APPLICATION_CREDENTIALS",
}
)
@dataclass(frozen=True, slots=True)
class _Vertex:
wire: Wire
tunnel: ForwardProxy
proxy: Gateway
def stream(self, model: str, marker: str) -> httpx.Response:
return self.proxy.request(
"POST",
f"/vertex_ai/v1/{_model_path(model)}:streamGenerateContent",
{"contents": [{"role": "user", "parts": [{"text": marker}]}]},
headers={"Authorization": _VENDOR_TOKEN, "x-litellm-api-key": f"Bearer {self.proxy.key}"},
)
def _model_path(model: str) -> str:
return f"projects/{_PROJECT}/locations/{_LOCATION}/publishers/google/models/{model}"
def _respond(request: Request) -> Reply:
if request.target.startswith(f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:"):
return Reply(status=503, body=_UNAVAILABLE_BODY)
if request.target.startswith(f"/v1/{_model_path(_STREAMING_MODEL)}:"):
return Reply(content_type="text/event-stream", chunks=_CHUNKS)
return Reply(status=404, body=b'{"error": "no scripted Vertex model"}')
@pytest.fixture(scope="module")
def vertex(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Vertex]:
directory: Final = tmp_path_factory.mktemp("vertex-pass-through-stream")
cert, key = write_self_signed_cert(directory, (_HOST,))
with (
wire_server(_respond, tls=server_context(cert, key)) as wire,
tunnelling_forward_proxy(int(wire.url.rsplit(":", 1)[1])) as tunnel,
gateway_from_environment() as rig,
owned_proxy(
rig,
directory,
{"HTTPS_PROXY": tunnel.url, "NO_PROXY": "127.0.0.1,localhost", "SSL_CERT_FILE": str(cert)},
remove_environment=tuple(name for name in os.environ if name.upper() in _SCRUBBED),
workers=2,
) as proxy,
):
yield _Vertex(wire, tunnel, proxy)
@pytest.fixture(autouse=True)
def fresh_edges(vertex: _Vertex) -> None:
vertex.wire.drain()
vertex.tunnel.drain()
def test_vertex_pass_through_stream_relays_every_sse_chunk_byte_for_byte(vertex: _Vertex) -> None:
marker: Final = uuid.uuid4().hex
response: Final = vertex.stream(_STREAMING_MODEL, marker)
sent: Final = vertex.wire.drain()
assert response.status_code == 200, response.text
assert response.headers["content-type"].startswith("text/event-stream"), response.headers
assert response.content == b"".join(_CHUNKS), response.text
assert [request.target for request in sent] == [
f"/v1/{_model_path(_STREAMING_MODEL)}:streamGenerateContent?alt=sse"
], sent
assert sent[0].headers["authorization"] == _VENDOR_TOKEN
assert json.loads(sent[0].body) == {"contents": [{"role": "user", "parts": [{"text": marker}]}]}
assert set(vertex.tunnel.targets()) == {f"{_HOST}:443"}
def test_vertex_pass_through_stream_relays_an_upstream_503_as_503(vertex: _Vertex) -> None:
response: Final = vertex.stream(_UNAVAILABLE_MODEL, uuid.uuid4().hex)
assert response.status_code == 503, response.text
assert response.content == _UNAVAILABLE_BODY, response.text
assert [request.target for request in vertex.wire.drain()] == [
f"/v1/{_model_path(_UNAVAILABLE_MODEL)}:streamGenerateContent?alt=sse"
]

View file

@ -0,0 +1,247 @@
import json
import os
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from urllib.parse import unquote
import httpx
import pytest
import yaml
from integration._support.anthropic_sse import (
LIFECYCLE,
error_body,
event_types,
message_id,
message_stream,
parse_sse,
status_reply,
stream_reply,
user_prompt,
)
from integration._support.client import (
Gateway,
JsonValue,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, Wire, wire_server
_OVERLOADED: Final = "claude-overloaded"
_HEALTHY: Final = "claude-healthy"
_COUNTED: Final = "claude-counted"
_COUNTED_OVERLOADED: Final = "claude-counted-overloaded"
_COUNTED_RATE_LIMITED: Final = "claude-counted-rate-limited"
_CHAT_ONLY_BACKEND: Final = "audit-chat-only"
_API_KEY: Final = "synthetic-anthropic-key"
_TEXT: Final = "Hello"
_COOLDOWN_GROUP: Final = "audit-overloaded-cooldown"
_COUNT_DISABLED_GROUP: Final = "audit-count-tokens-disabled"
_COUNT_UPSTREAM_GROUP: Final = "audit-count-tokens-upstream-503"
_COUNT_OVERLOADED_GROUP: Final = "audit-count-tokens-upstream-529"
_COUNT_RATE_LIMITED_GROUP: Final = "audit-count-tokens-upstream-429"
_GATEWAY: Final = "/claude_code_gateway"
_GATEWAY_GROUPS: Final = {
"non-stream": "audit-gateway-overloaded-non-stream",
"stream": "audit-gateway-overloaded-stream",
}
_COUNT_STATUSES: Final = {_COUNTED: 503, _COUNTED_OVERLOADED: 529, _COUNTED_RATE_LIMITED: 429}
_COUNT_TOKENS_TARGET: Final = "/v1/messages/count_tokens"
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def _served_id(marker: str) -> str:
return f"msg_{_HEALTHY}_{marker}"
def _respond(request: Request) -> Reply:
body: Final = object_value(json.loads(request.body))
backend: Final = str(body["model"])
if unquote(request.target) == _COUNT_TOKENS_TARGET:
status: Final = _COUNT_STATUSES[backend]
return Reply(status=status, body=error_body(status, f"scripted {status}"))
if backend == _OVERLOADED:
return status_reply(529)
if backend == _HEALTHY:
return stream_reply(message_stream(_served_id(user_prompt(body)), backend, _TEXT))
return Reply(status=404, body=b'{"error": "no scripted backend"}')
def _deployment(name: str, model: str, api_base: str, api_key: str, **params: JsonValue) -> dict[str, JsonValue]:
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": api_key, **params}}
def _config(wire: Wire, directory: Path) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["model_list"] = [
_deployment(_COOLDOWN_GROUP, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=1),
_deployment(_COOLDOWN_GROUP, f"anthropic/{_HEALTHY}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY, order=2),
_deployment(
_COUNT_DISABLED_GROUP,
f"hosted_vllm/{_CHAT_ONLY_BACKEND}",
_UPSTREAM_URL_PLACEHOLDER + "/v1",
"synthetic-vllm-key",
),
_deployment(_COUNT_UPSTREAM_GROUP, f"anthropic/{_COUNTED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY),
_deployment(_COUNT_OVERLOADED_GROUP, f"anthropic/{_COUNTED_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY),
_deployment(
_COUNT_RATE_LIMITED_GROUP, f"anthropic/{_COUNTED_RATE_LIMITED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY
),
*(
_deployment(group, f"anthropic/{_OVERLOADED}", _UPSTREAM_URL_PLACEHOLDER, _API_KEY)
for group in _GATEWAY_GROUPS.values()
),
]
config["router_settings"] = {
"num_retries": 0,
"allowed_fails_policy": {"InternalServerErrorAllowedFails": 0},
"cooldown_time": 120,
"redis_host": os.environ["REDIS_HOST"],
"redis_port": int(os.environ["REDIS_PORT"]),
}
config["litellm_settings"]["disable_token_counter"] = True
config["general_settings"]["enable_claude_code_gateway"] = True
path: Final = directory / "anthropic-overloaded-cooldown.yaml"
path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url))
return path
@dataclass(frozen=True, slots=True)
class _Rig:
proxy: Gateway
wire: Wire
def messages(self, path: str, model: str, marker: str, **extra: JsonValue) -> httpx.Response:
return self.proxy.request(
"POST",
path,
{
"model": model,
"max_tokens": 16,
"stream": True,
"messages": [{"role": "user", "content": marker}],
**extra,
},
headers={"anthropic-version": "2023-06-01"},
)
def count_tokens(self, model: str, prefix: str = "") -> httpx.Response:
return self.proxy.request(
"POST",
prefix + _COUNT_TOKENS_TARGET,
{"model": model, "messages": [{"role": "user", "content": "count me"}]},
headers={"anthropic-version": "2023-06-01"},
)
def targets_called(self) -> tuple[str, ...]:
return tuple(unquote(request.target) for request in self.wire.drain())
def backends_called(self) -> tuple[str, ...]:
return tuple(str(object_value(json.loads(request.body))["model"]) for request in self.wire.drain())
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("anthropic-overloaded-cooldown")
with gateway_from_environment() as gateway, wire_server(_respond) as wire:
overrides: Final = {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _API_KEY}
with owned_proxy(gateway, directory, overrides, config=_config(wire, directory), workers=2) as proxy:
yield _Rig(proxy, wire)
@pytest.fixture(autouse=True)
def _fresh_wire(rig: _Rig) -> None:
rig.wire.drain()
def _assert_overloaded_json(response: httpx.Response) -> None:
assert response.status_code == 529, response.text
payload: Final = object_value(json.loads(response.content))
assert object_value(payload["error"])["type"] == "overloaded_error", response.text
def _count_tokens_error(response: httpx.Response, status: int) -> dict[str, JsonValue]:
assert response.status_code == status, response.text
envelope: Final = object_value(json.loads(response.content))
assert envelope["type"] == "error", response.text
return object_value(envelope["error"])
def test_an_overloaded_deployment_is_cooled_down_by_the_internal_server_error_allowed_fails_policy(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
first: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True)
_assert_overloaded_json(first)
assert rig.backends_called() == (_OVERLOADED,)
second: Final = rig.messages("/v1/messages", _COOLDOWN_GROUP, marker, num_retries=0, disable_fallbacks=True)
assert second.status_code == 200, second.text
events: Final = parse_sse(second.text)
assert event_types(events) == LIFECYCLE, events
assert message_id(events) == _served_id(marker), events
assert rig.backends_called() == (_HEALTHY,)
def test_count_tokens_disabled_without_a_provider_counter_answers_503_api_error(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_DISABLED_GROUP)
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "Token counting is disabled" in string_value(error["message"]), response.text
assert rig.targets_called() == ()
def test_count_tokens_whose_upstream_answers_503_stays_503_api_error(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP)
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "scripted 503" in string_value(error["message"]), response.text
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
@pytest.mark.parametrize(
("group", "status", "error_type"),
(
pytest.param(_COUNT_OVERLOADED_GROUP, 529, "overloaded_error", id="529"),
pytest.param(_COUNT_RATE_LIMITED_GROUP, 429, "rate_limit_error", id="429"),
),
)
def test_count_tokens_keeps_other_upstream_statuses_and_their_error_body(
rig: _Rig, group: str, status: int, error_type: str
) -> None:
response: Final = rig.count_tokens(group)
error: Final = _count_tokens_error(response, status)
assert error["type"] == error_type, response.text
assert f"scripted {status}" in string_value(error["message"]), response.text
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)
def test_pass_through_messages_stream_forwards_an_upstream_529_verbatim(rig: _Rig) -> None:
response: Final = rig.messages("/anthropic/v1/messages", _OVERLOADED, uuid.uuid4().hex)
assert response.status_code == 529, response.text
assert response.content == error_body(529, "scripted 529"), response.text
assert rig.backends_called() == (_OVERLOADED,)
@pytest.mark.parametrize("shape", _GATEWAY_GROUPS.keys())
def test_the_claude_code_gateway_messages_route_answers_an_upstream_529_as_overloaded_error(
rig: _Rig, shape: str
) -> None:
response: Final = rig.messages(
f"{_GATEWAY}/v1/messages",
_GATEWAY_GROUPS[shape],
uuid.uuid4().hex,
stream=shape == "stream",
num_retries=0,
)
_assert_overloaded_json(response)
assert rig.backends_called() == (_OVERLOADED,)
def test_the_claude_code_gateway_count_tokens_route_answers_an_upstream_503_as_503(rig: _Rig) -> None:
response: Final = rig.count_tokens(_COUNT_UPSTREAM_GROUP, prefix=_GATEWAY)
error: Final = _count_tokens_error(response, 503)
assert error["type"] == "api_error", response.text
assert "scripted 503" in string_value(error["message"]), response.text
assert rig.targets_called() == (_COUNT_TOKENS_TARGET,)

View file

@ -45,10 +45,19 @@ _FALLBACK_GROUP: Final = "audit-fallback"
_CW_FALLBACK_GROUP: Final = "audit-cw-fallback"
_LONELY: Final = "audit-lonely"
_POLICY: Final = "audit-policy"
_SERVICE_UNAVAILABLE_RETRY_POLICY: Final = "audit-service-unavailable-retry-policy"
_INTERNAL_RETRY_POLICY: Final = "audit-internal-retry-policy"
_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url"
Behavior = Literal[
"drop-once", "drop-always", "hold-ping", "drop-then-too-long", "drop-then-401", "overloaded-frames-fallback-503"
"drop-once",
"drop-always",
"hold-ping",
"drop-then-too-long",
"drop-then-401",
"overloaded-frames-fallback-529",
"overloaded-frames",
"overloaded-once",
]
pytestmark = pytest.mark.timeout(240)
@ -81,13 +90,15 @@ def _primary_reply(
return Reply(status=400, body=error_body(400, "prompt is too long: 250000 tokens > 200000 maximum"))
case "drop-then-401":
return dropped if attempt == 1 else status_reply(401)
case "overloaded-frames-fallback-503":
case "overloaded-frames-fallback-529" | "overloaded-frames":
return stream_reply((error_frame(529, "scripted overloaded"),))
case "overloaded-once":
return status_reply(529) if attempt == 1 else stream_reply(full)
raise AssertionError(behavior)
def _fallback_reply(behavior: str, full: tuple[bytes, bytes, bytes, bytes]) -> Reply:
if behavior == "overloaded-frames-fallback-503":
if behavior == "overloaded-frames-fallback-529":
return status_reply(503)
return stream_reply(full)
@ -141,13 +152,19 @@ def _config(wire: Wire, directory: Path) -> Path:
_deployment(_CW_FALLBACK_GROUP, _CONTEXT_WINDOW),
_deployment(_LONELY, _PRIMARY, num_retries=1),
_deployment(_POLICY, _PRIMARY),
_deployment(_SERVICE_UNAVAILABLE_RETRY_POLICY, _PRIMARY),
_deployment(_INTERNAL_RETRY_POLICY, _PRIMARY),
]
config["router_settings"] = {
"num_retries": 1,
"disable_cooldowns": True,
"fallbacks": [{_WITH_FALLBACKS: [_FALLBACK_GROUP]}],
"context_window_fallbacks": [{_WITH_FALLBACKS: [_CW_FALLBACK_GROUP]}],
"model_group_retry_policy": {_POLICY: {"DefaultRetries": 2}},
"model_group_retry_policy": {
_POLICY: {"DefaultRetries": 2},
_SERVICE_UNAVAILABLE_RETRY_POLICY: {"ServiceUnavailableErrorRetries": 1, "InternalServerErrorRetries": 0},
_INTERNAL_RETRY_POLICY: {"InternalServerErrorRetries": 1, "ServiceUnavailableErrorRetries": 0},
},
}
path: Final = directory / "messages-retry-budget-sources.yaml"
path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url))
@ -290,10 +307,34 @@ def test_a_retry_rejected_with_401_ends_the_retries_and_reaches_the_client_uncha
assert rig.attempts_on(_PRIMARY, marker) == 2
def test_overloaded_frames_whose_fallback_fails_answer_the_mapped_internal_server_error(rig: _Rig) -> None:
def test_overloaded_frames_whose_fallback_fails_answer_529_overloaded_error(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-503", marker)
assert response.status_code == 500, response.text
response: Final = rig.stream(_WITH_FALLBACKS, "overloaded-frames-fallback-529", marker)
assert response.status_code == 529, response.text
assert "overloaded_error" in response.text, response.text
assert "content_block_delta" not in response.text, response.text
assert rig.attempts_on(_PRIMARY, marker) == 2
assert rig.attempts_on(_FALLBACK, marker) == 2
def test_overloaded_frames_that_spend_the_budget_without_a_fallback_relay_the_last_attempts_frame(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
response: Final = rig.stream(_LONELY, "overloaded-frames", marker)
assert response.status_code == 200, response.text
assert response.text == error_frame(529, "scripted overloaded").decode(), response.text
assert rig.attempts_on(_PRIMARY, marker) == 2
def test_a_529_is_retried_under_the_internal_server_error_retry_policy(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
_assert_completed(rig.stream(_INTERNAL_RETRY_POLICY, "overloaded-once", marker), _served_id(_PRIMARY, marker, 2))
assert rig.attempts_on(_PRIMARY, marker) == 2
def test_a_529_is_not_retried_under_the_service_unavailable_error_retry_policy(rig: _Rig) -> None:
marker: Final = uuid.uuid4().hex
response: Final = rig.stream(_SERVICE_UNAVAILABLE_RETRY_POLICY, "overloaded-once", marker)
assert response.status_code == 529, response.text
assert "overloaded_error" in response.text, response.text
assert "content_block_delta" not in response.text, response.text
assert rig.attempts_on(_PRIMARY, marker) == 1

View file

@ -848,60 +848,36 @@ def test_wrong_azure_key_surfaces_azures_401(rig: _Rig, scenario: Scenario) -> N
assert len(_upstream(rig, deployment).model_calls) == 1
def _tool_call(call: Mapping[str, JsonValue]) -> tuple[str, Mapping[str, JsonValue]]:
function: Final = object_value(call["function"])
return string_value(function["name"]), iv.JSON_OBJECT.validate_json(string_value(function["arguments"]))
def _assert_failed_once(call_id: str, *, spend: float) -> None:
rows: Final = eventually(
lambda: read_rows('SELECT status, spend FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)),
lambda found: len(found) >= 1,
seconds=70,
)
assert [row["status"] for row in rows] == ["failure"], rows
assert float(rows[0]["spend"]) == pytest.approx(spend), rows
def _stream_choices(stream: _Stream) -> Iterator[Mapping[str, JsonValue]]:
for frame in stream.frames:
yield from rv.ITEMS.validate_python(frame.get("choices") or [])
def _streamed_tool_calls(stream: _Stream) -> Iterator[tuple[str, Mapping[str, JsonValue]]]:
for choice in _stream_choices(stream):
yield from map(
_tool_call, rv.ITEMS.validate_python(object_value(choice.get("delta") or {}).get("tool_calls") or [])
)
def _handed_back(deployment: _Deployment) -> list[tuple[str, Mapping[str, JsonValue]]]:
return [(iv.SEARCH_TOOL, {"query": iv.query(deployment.name)})]
def _assert_succeeded_once(model: str, identity: str) -> None:
rows: Final = _spend_rows(model, 1)
assert [string_value(row["request_id"]) for row in rows if row["status"] == "success"] == [identity], rows
def test_failed_follow_up_call_hands_the_search_tool_call_back_with_its_usage(rig: _Rig, scenario: Scenario) -> None:
deployment: Final = _deploy(rig, scenario, "followup-fails")
stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)}))
assert stream.status == 200 and stream.content == "", stream.text
assert list(_streamed_tool_calls(stream)) == _handed_back(deployment), stream.text
finished: Final = [choice["finish_reason"] for choice in _stream_choices(stream) if choice.get("finish_reason")]
assert finished == ["tool_calls"], stream.text
assert [_totals(usage) for usage in stream.usages] == [iv.FIRST_USAGE["total_tokens"]], stream.text
(identity,) = stream.ids
@pytest.mark.parametrize("stream", [True, False], ids=["stream", "json"])
def test_failed_follow_up_call_answers_with_the_providers_error_instead_of_the_search_tool_call(
rig: _Rig, scenario: Scenario, stream: bool
) -> None:
deployment: Final = _deploy(
rig, scenario, "followup-fails", priced=True
)
extra: Final = {**_NO_CACHE, "stream_options": dict(_USAGE)} if stream else _NO_CACHE
body: Final = _chat_body(deployment, stream=stream, extra=extra)
response: Final = rig.gateway.request("POST", "/v1/chat/completions", body)
assert response.status_code == 500 and iv.FOLLOWUP_FAILURE in response.text, response.text
assert iv.SEARCH_TOOL not in response.text, response.text
_assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1)
_assert_succeeded_once(deployment.model, identity)
_assert_failed_once(
response.headers["x-litellm-call-id"],
spend=iv.FIRST_USAGE["prompt_tokens"] * 0.001 + iv.FIRST_USAGE["completion_tokens"] * 0.002,
)
assert rig.gateway.chat(_deploy(rig, scenario, "plain").model)["object"] == "chat.completion"
def test_failed_follow_up_call_on_a_json_request_hands_the_search_tool_call_back(rig: _Rig, scenario: Scenario) -> None:
deployment: Final = _deploy(rig, scenario, "followup-fails")
response: Final = rig.gateway.request("POST", "/v1/chat/completions", _chat_body(deployment, stream=False))
assert response.status_code == 200, response.text
body: Final = iv.JSON_OBJECT.validate_json(response.content)
(choice,) = rv.ITEMS.validate_python(body["choices"])
calls: Final = rv.ITEMS.validate_python(object_value(choice["message"])["tool_calls"])
assert [_tool_call(call) for call in calls] == _handed_back(deployment), body
assert choice["finish_reason"] == "tool_calls", body
assert _totals(object_value(body["usage"])) == iv.FIRST_USAGE["total_tokens"], body
_assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1)
_assert_succeeded_once(deployment.model, string_value(body["id"]))
def test_search_outage_still_restreams_the_follow_up_answer(rig: _Rig, scenario: Scenario) -> None:
deployment: Final = _deploy(rig, scenario, "search-down")
stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)}))

View file

@ -859,7 +859,7 @@ async def test_batch_logging_azure_credentials_regression():
"timeout": 600,
}
batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n'
batch_output = b'{"id": "batch_req_1", "custom_id": "request-1", "response": {"status_code": 200, "body": {"id": "chatcmpl-azure", "object": "chat.completion", "model": "gpt-4.1", "usage": {"prompt_tokens": 15, "completion_tokens": 25, "total_tokens": 40}}}}\n'
print("\n1. Testing credential extraction...")

View file

@ -5,16 +5,24 @@ All sandbox dependencies are injected: a FakeSandbox stands in for the real e2b
config and records how it is called.
"""
import asyncio
import time
from collections.abc import Mapping
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.integrations.code_interpreter_interception.handler import (
_INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY,
)
from litellm.integrations.code_interpreter_interception.handler import (
_SANDBOX_KEY,
_SANDBOX_LOST_RESULT_OUTPUT,
_SANDBOX_UNAVAILABLE_OUTPUT,
_SESSION_SCOPED_KEY,
LITELLM_CODE_EXECUTION_TOOL_NAME,
CodeInterpreterInterceptionLogger,
@ -23,6 +31,7 @@ from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
CODE_INTERPRETER_STREAM_OPTIONS_KEY,
AgenticLoopPlan,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
is_interception_internal_key,
)
@ -831,11 +840,7 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels():
@pytest.mark.asyncio
async def test_build_plan_deletes_sandbox_when_execution_raises():
"""If sandbox execution raises before a plan is built (e.g. E2B aborts
output over its cap), the cached sandbox must be deleted before re-raising,
otherwise a caller can leak paid containers until the prune TTL."""
async def test_a_failed_code_run_still_deletes_its_sandbox_once_the_loop_ends():
class RaisingSandbox(FakeSandbox):
async def arun_code(self, *, container, code, **kwargs):
raise ValueError("output exceeded cap")
@ -843,13 +848,11 @@ async def test_build_plan_deletes_sandbox_when_execution_raises():
sandbox = RaisingSandbox()
logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
with pytest.raises(ValueError, match="exceeded cap"):
await _build_plan(logger, sandbox, call_id="k1")
plan = await _build_plan(logger, sandbox, call_id="k1")
await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={})
assert len(sandbox.create_calls) == 1, "the sandbox must have been created"
assert len(sandbox.delete_calls) == 1, (
"a build failure must delete the cached sandbox so it does not keep running and billing"
)
assert len(sandbox.create_calls) == 1
assert len(sandbox.delete_calls) == 1
assert "sbxkey1" not in logger._container_cache
@ -1448,3 +1451,215 @@ async def test_per_identity_cap_evicts_lru_session():
"the entry with the oldest last_accessed timestamp must be evicted first (LRU)"
)
assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache"
class _SandboxThatCannotStart(FakeSandbox):
async def acreate_sandbox(self, **kwargs):
raise ConnectionError("sandbox provider unreachable")
class _SandboxThatCannotRunCode(FakeSandbox):
async def arun_code(self, *, container, code, **kwargs):
raise TimeoutError("sandbox run timed out")
_SANDBOX_FAILURES: Final = (
pytest.param(None, _SANDBOX_UNAVAILABLE_OUTPUT, id="no_sandbox"),
pytest.param(_SandboxThatCannotStart(), _SANDBOX_UNAVAILABLE_OUTPUT, id="create_fails"),
pytest.param(_SandboxThatCannotRunCode(), _SANDBOX_LOST_RESULT_OUTPUT, id="run_fails"),
)
@pytest.mark.asyncio
@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES)
async def test_a_sandbox_failure_answers_the_responses_tool_call_with_a_fixed_failure(sandbox, expected_output):
plan: Final = await _build_plan(CodeInterpreterInterceptionLogger(sandbox_config=sandbox), sandbox)
outputs: Final = [message for message in _iter_messages(plan) if message.get("type") == "function_call_output"]
assert plan.run_agentic_loop is True
assert [(output["call_id"], output["output"]) for output in outputs] == [("c1", expected_output)]
assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"]
@pytest.mark.asyncio
@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES)
async def test_a_sandbox_failure_answers_the_chat_tool_call_with_a_fixed_failure(sandbox, expected_output):
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
plan: Final = await logger.async_build_agentic_loop_plan(
tools={
"tool_calls": [
{"id": "call_1", "name": LITELLM_CODE_EXECUTION_TOOL_NAME, "arguments": '{"code":"print(40 + 2)"}'}
]
},
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]},
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={"tools": []},
logging_obj=FakeLogging(litellm_call_id="k1"),
stream=False,
kwargs={
"litellm_call_id": "k1",
_ACTIVE_KEY: True,
_SANDBOX_KEY: "sbxkey1",
"_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE,
},
)
tool_messages: Final = [message for message in _iter_messages(plan) if message.get("role") == "tool"]
assert plan.run_agentic_loop is True
assert [(message["tool_call_id"], message["content"]) for message in tool_messages] == [
("call_1", expected_output)
]
assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"]
_SESSION_PLAN_SURFACES: Final = (
pytest.param({}, FakeResponse(output=[_function_call_item()]), id="responses"),
pytest.param(
{_ACTIVE_KEY: True, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE},
{"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]},
id="chat",
),
)
async def _build_session_plan(
logger: CodeInterpreterInterceptionLogger,
request_id: str,
surface_kwargs: Mapping[str, object] | None = None,
response: object | None = None,
) -> AgenticLoopPlan:
return await logger.async_build_agentic_loop_plan(
tools={
"tool_calls": [
{
"id": "call_1",
"call_id": "c1",
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
"arguments": '{"code":"print(1)"}',
}
]
},
model="gpt-5",
messages=[{"role": "user", "content": "x"}],
response=response or FakeResponse(output=[_function_call_item()]),
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={"tools": []},
logging_obj=FakeLogging(litellm_call_id=request_id),
stream=False,
kwargs={
"litellm_call_id": request_id,
_SANDBOX_KEY: "conv-1",
_SESSION_SCOPED_KEY: True,
**(surface_kwargs or {}),
},
)
@pytest.mark.asyncio
@pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES)
async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request(
surface_kwargs: Mapping[str, object], response: object
) -> None:
sandbox: Final = _SandboxThatCannotRunCode()
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
await _build_session_plan(logger, "req1", surface_kwargs, response)
await _build_session_plan(logger, "req2", surface_kwargs, response)
assert len(sandbox.create_calls) == 2
assert len(sandbox.delete_calls) == 2
class _SandboxWhoseFirstContainerDies(FakeSandbox):
def __init__(self) -> None:
super().__init__()
self.handles: list[FakeHandle] = []
self.first_run_started = asyncio.Event()
self.first_run_may_finish = asyncio.Event()
async def acreate_sandbox(self, **kwargs: object) -> FakeHandle:
self.create_calls.append(kwargs)
handle: Final = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}")
self.handles.append(handle)
return handle
async def arun_code(self, *, container: FakeHandle, code: str, **kwargs: object) -> CodeExecutionResult:
self.run_calls.append({"container": container, "code": code})
if len(self.run_calls) == 1:
self.first_run_started.set()
await self.first_run_may_finish.wait()
if container is self.handles[0]:
raise TimeoutError("sandbox expired")
return CodeExecutionResult(stdout=self.stdout)
@pytest.mark.asyncio
async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement() -> None:
sandbox: Final = _SandboxWhoseFirstContainerDies()
logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox)
slow_request: Final = asyncio.create_task(_build_session_plan(logger, "req-slow"))
await sandbox.first_run_started.wait()
await _build_session_plan(logger, "req-fails-fast")
await _build_session_plan(logger, "req-gets-replacement")
sandbox.first_run_may_finish.set()
await slow_request
await _build_session_plan(logger, "req-after")
assert [call["container"] for call in sandbox.delete_calls] == [sandbox.handles[0]]
assert [call["container"] for call in sandbox.run_calls[-2:]] == [sandbox.handles[1], sandbox.handles[1]]
def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]:
return {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": content,
"stop_reason": stop_reason,
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 5},
}
@pytest.mark.asyncio
async def test_a_responses_request_bridged_to_claude_runs_the_code_and_ends_without_the_internal_call(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
sandbox: Final = FakeSandbox(stdout="42")
monkeypatch.setattr(litellm, "callbacks", [CodeInterpreterInterceptionLogger(sandbox_config=sandbox)])
provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(
side_effect=[
httpx.Response(
200,
json=_anthropic_message(
[
{
"type": "tool_use",
"id": "toolu_1",
"name": LITELLM_CODE_EXECUTION_TOOL_NAME,
"input": {"code": "print(40 + 2)"},
}
],
"tool_use",
),
),
httpx.Response(200, json=_anthropic_message([{"type": "text", "text": "The answer is 42."}], "end_turn")),
]
)
response: Final = await litellm.aresponses(
model="anthropic/claude-sonnet-5-5",
input="compute 40 + 2",
api_key="sk-test",
tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
)
assert provider.call_count == 2
assert [run["code"] for run in sandbox.run_calls] == ["print(40 + 2)"]
assert not [item for item in response.output if getattr(item, "type", None) == "function_call"]

View file

@ -12,6 +12,7 @@ config.yaml through to the settings the loop actually reads.
"""
import json
import time
from unittest.mock import MagicMock, patch
import pytest
@ -24,6 +25,8 @@ from litellm.integrations.websearch_interception.handler import (
WebSearchInterceptionLogger,
)
from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
@ -31,6 +34,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.secret_managers.main import get_secret
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
@ -38,6 +42,7 @@ from litellm.types.integrations.custom_logger import (
)
INTERNAL_TOOL_NAME = "litellm_web_search"
CODE_EXECUTION_TOOL_NAME = "litellm_code_execution"
@pytest.fixture(autouse=True)
@ -99,6 +104,29 @@ def _tool_use_names(response: dict) -> list[str]:
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse:
return ResponsesAPIResponse(
id="resp_1",
created_at=0,
status="completed",
model="claude-sonnet-4-5",
object="response",
output=[
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}],
},
*(
{"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"}
for call_id, name in function_calls
),
],
)
class _InterceptingCallback(CustomLogger):
"""
Stands in for the websearch interceptor: asks for another loop whenever the
@ -152,14 +180,46 @@ class _InterceptingCallback(CustomLogger):
return response
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
class _ResponsesInterceptingCallback(_InterceptingCallback):
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
calls = [
{"id": item.call_id, "call_id": item.call_id, "name": item.name}
for item in response.output
if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME
]
return bool(calls), {"tool_calls": calls}
def _logging_obj(*callbacks: CustomLogger, converted_stream: bool = False) -> MagicMock:
logging_obj = MagicMock()
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
logging_obj.dynamic_success_callbacks = [callback]
logging_obj.dynamic_success_callbacks = list(callbacks)
logging_obj.litellm_call_id = "call-abc"
return logging_obj
class _CodeExecutionGate(CustomLogger):
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
calls = (
[
{"id": block["id"], "name": block["name"]}
for block in response["content"]
if block.get("type") == "tool_use" and block.get("name") == CODE_EXECUTION_TOOL_NAME
]
if isinstance(response, dict)
else [
{"call_id": item.call_id, "name": item.name}
for item in response.output
if item.type == "function_call" and item.name == CODE_EXECUTION_TOOL_NAME
]
)
return bool(calls), {"tool_calls": calls}
async def _run_hooks(
handler: BaseLLMHTTPHandler,
callback: CustomLogger,
@ -314,6 +374,48 @@ class TestCappedLoopReturnsTerminalResponse:
assert _tool_use_names(result) == ["get_weather"]
assert result["stop_reason"] == "tool_use"
@pytest.mark.asyncio
async def test_every_hooks_internal_call_is_dropped_and_the_clients_survives(self):
response = _response_asking_for_another_search()
response["content"] += [
{"id": "toolu_code_1", "type": "tool_use", "name": CODE_EXECUTION_TOOL_NAME, "input": {}},
{"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}},
]
result = await self.handler._call_agentic_completion_hooks(
response=response,
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={},
logging_obj=_logging_obj(self.callback, _CodeExecutionGate()),
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
)
assert _tool_use_names(result) == ["get_weather"]
assert self.callback.plan_calls == 0
@pytest.mark.asyncio
async def test_responses_surface_drops_every_hooks_internal_call_and_keeps_the_clients(self):
result = await self.handler._call_agentic_completion_hooks(
response=_responses_turn(
("call_search", INTERNAL_TOOL_NAME), ("call_code", CODE_EXECUTION_TOOL_NAME), ("call_client", "lookup")
),
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={},
logging_obj=_logging_obj(_ResponsesInterceptingCallback(), _CodeExecutionGate()),
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
api_surface="responses",
)
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_client"]
def test_only_the_refused_tool_calls_are_dropped(self):
"""
A block is matched on the id the rail refused, not on the tool name, so a
@ -347,6 +449,18 @@ class TestCappedLoopReturnsTerminalResponse:
assert _tool_use_names(result) == []
assert result["stop_reason"] == "end_turn"
def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self):
response = _response_asking_for_another_search()
response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}}
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=response,
tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]},
)
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
"""
@ -365,19 +479,34 @@ class TestCappedLoopReturnsTerminalResponse:
assert self.callback.plan_calls == 0
@pytest.mark.asyncio
async def test_responses_surface_is_left_to_its_existing_behavior(self):
"""
The responses surface carries a pydantic model rather than the anthropic
dict this finalizer rewrites, so it keeps raising instead of being handed
a response that was never actually finalized.
"""
with pytest.raises(AgenticLoopSafetyError):
await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
api_surface="responses",
)
async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self):
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup"))
callback = _ResponsesInterceptingCallback()
result = await _run_hooks(
self.handler,
callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=response,
api_surface="responses",
)
assert isinstance(result, ResponsesAPIResponse)
assert [(item.type, getattr(item, "name", None)) for item in result.output] == [
("message", None),
("function_call", "lookup"),
]
assert callback.plan_calls == 0
def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self):
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME))
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=response,
tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]},
)
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"]
@pytest.mark.asyncio
async def test_non_dict_response_is_returned_untouched(self):
@ -509,6 +638,156 @@ def _response_asking_for_searches(*queries: str) -> dict:
}
def _request_logging_obj(call_type: str, *callbacks: CustomLogger) -> Logging:
logging_obj = Logging(
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
stream=False,
call_type=call_type,
start_time=time.time(),
litellm_call_id="call-abc",
function_id="f",
dynamic_success_callbacks=list(callbacks),
)
logging_obj.update_environment_variables(
model="claude-sonnet-4-5", user="", optional_params={}, litellm_params={}, custom_llm_provider="anthropic"
)
return logging_obj
def _assert_first_call_billed(logging_obj: Logging, *, input_tokens: int, output_tokens: int) -> None:
price = litellm.model_cost["claude-sonnet-4-5"]
assert logging_obj.model_call_details["combined_usage_object"].total_tokens == input_tokens + output_tokens
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
input_tokens * price["input_cost_per_token"] + output_tokens * price["output_cost_per_token"]
)
class TestFailedFollowUpCallReachesTheClient:
def setup_method(self):
self.handler = BaseLLMHTTPHandler()
self.logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"])
@staticmethod
async def _rate_limited_follow_up(**call_kwargs):
raise RateLimitError(
"UserByModelByMinuteUncachedInputTokens", llm_provider="anthropic", model="claude-sonnet-4-5"
)
@staticmethod
async def _search(query, kwargs=None, rich=None):
found = SearchResult(title="Result", url="https://example.com", snippet="A result.", date=None)
return ("Title: Result\nURL: https://example.com", SearchResponse(results=[found]))
@pytest.mark.asyncio
@pytest.mark.parametrize("converted_stream", (False, True), ids=("non_streaming", "streaming_client"))
async def test_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch, converted_stream):
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up)
with (
patch.object(self.logger, "_execute_search", side_effect=self._search),
pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"),
):
await self.handler._call_agentic_completion_hooks(
response=_response_asking_for_searches("who won the world cup"),
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
logging_obj=_logging_obj(self.logger, converted_stream=converted_stream),
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
)
@pytest.mark.asyncio
async def test_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch):
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up)
logging_obj = _request_logging_obj("anthropic_messages", self.logger)
with (
patch.object(self.logger, "_execute_search", side_effect=self._search),
pytest.raises(RateLimitError),
):
await self.handler._call_agentic_completion_hooks(
response=_response_asking_for_searches("who won the world cup"),
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
logging_obj=logging_obj,
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
)
_assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5)
@pytest.mark.asyncio
async def test_a_crashed_plan_build_is_not_retried_and_bills_the_first_call(self):
logging_obj = _request_logging_obj("anthropic_messages", self.logger)
with (
patch.object(self.logger, "async_build_agentic_loop_plan", side_effect=ValueError("the plan crashed")),
pytest.raises(ValueError, match="the plan crashed") as raised,
):
await self.handler._call_agentic_completion_hooks(
response=_response_asking_for_searches("who won the world cup"),
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]},
logging_obj=logging_obj,
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True},
)
assert is_agentic_followup_failure(raised.value)
_assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5)
@pytest.mark.asyncio
async def test_responses_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch):
monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up)
callback = _ResponsesInterceptingCallback()
logging_obj = _request_logging_obj("aresponses", callback)
first_call = _responses_turn(("call_search", INTERNAL_TOOL_NAME)).model_copy(
update={"usage": {"input_tokens": 13, "output_tokens": 6, "total_tokens": 19}}
)
with pytest.raises(RateLimitError):
await self.handler._call_agentic_completion_hooks(
response=first_call,
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={},
logging_obj=logging_obj,
stream=False,
custom_llm_provider="anthropic",
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3},
api_surface="responses",
)
_assert_first_call_billed(logging_obj, input_tokens=13, output_tokens=6)
@pytest.mark.asyncio
async def test_responses_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch):
monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up)
callback = _ResponsesInterceptingCallback()
with pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"):
await _run_hooks(
self.handler,
callback,
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3},
response=_responses_turn(("call_search", INTERNAL_TOOL_NAME)),
api_surface="responses",
)
assert callback.plan_calls == 1
class TestFailedSearchEndsTheTurn:
"""
A search that failed used to come back to the client as an empty successful

View file

@ -371,8 +371,7 @@ class TestLoggingObjExcludedFromFollowUp:
class TestFollowUpErrorScenarios:
"""Regression tests: the agentic loop must surface errors properly and
not silently swallow them (except at the _call_agentic_completion_hooks
level which intentionally catches to return the initial response)."""
not silently swallow them."""
@pytest.mark.asyncio
async def test_followup_400_raises(self):

View file

@ -12,7 +12,7 @@ import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, PromptTokensDetailsWrapper, Usage
TIER_MODEL: Final = "tier-priced-test-model"
TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
@ -48,6 +48,190 @@ AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
)
@pytest.mark.parametrize("image_tokens", (60, 100))
@pytest.mark.parametrize(
("service_tier", "priority_rate", "image_rate"),
((None, 11e-06, 7e-06), ("priority", 11e-06, 11e-06), ("flex", 11e-06, 7e-06), ("priority", 0.0, 0.0)),
)
def test_completion_cost_uses_priority_image_output_rate(
local_model_cost_map: None, service_tier: str | None, priority_rate: float, image_rate: float, image_tokens: int
) -> None:
model: Final = "gemini-synthetic-image-priority"
litellm.register_model(
{
model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 5e-06,
"output_cost_per_image_token": 7e-06,
"output_cost_per_image_token_priority": priority_rate,
}
}
)
response: Final = ModelResponse(
model=model,
usage=Usage(
prompt_tokens=20,
completion_tokens=100,
total_tokens=120,
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=image_tokens),
),
)
cost: Final = litellm.completion_cost(
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier
)
assert cost == pytest.approx(20 * 3e-06 + (100 - image_tokens) * 5e-06 + image_tokens * image_rate)
@pytest.mark.parametrize("prompt_tokens", (199_999, 200_000, 200_001))
@pytest.mark.parametrize(
("service_tier", "flat_rate", "context_rate"),
((None, 7e-06, 11e-06), ("priority", 13e-06, 17e-06), ("flex", 19e-06, 23e-06)),
)
def test_completion_cost_uses_total_prompt_for_audio_context_tiers(
local_model_cost_map: None, prompt_tokens: int, service_tier: str | None, flat_rate: float, context_rate: float
) -> None:
model: Final = "gemini-synthetic-audio-context"
litellm.register_model(
{
model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 5e-06,
"input_cost_per_audio_token": 7e-06,
"input_cost_per_audio_token_above_200k_tokens": 11e-06,
"input_cost_per_audio_token_priority": 13e-06,
"input_cost_per_audio_token_above_200k_tokens_priority": 17e-06,
"input_cost_per_audio_token_flex": 19e-06,
"input_cost_per_audio_token_above_200k_tokens_flex": 23e-06,
}
}
)
response: Final = ModelResponse(
model=model,
usage=Usage(
prompt_tokens=prompt_tokens,
completion_tokens=10,
total_tokens=prompt_tokens + 10,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
),
)
cost: Final = litellm.completion_cost(
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier=service_tier
)
audio_rate: Final = context_rate if prompt_tokens > 200_000 else flat_rate
assert cost == pytest.approx((prompt_tokens - 100) * 3e-06 + 100 * audio_rate + 10 * 5e-06)
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001))
def test_batch_cost_uses_audio_context_and_image_output_tiers(local_model_cost_map: None, prompt_tokens: int) -> None:
from litellm.cost_calculator import batch_cost_calculator
model: Final = "gemini-synthetic-batch-audio-context"
litellm.register_model(
{
model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 6e-06,
"output_cost_per_token": 10e-06,
"input_cost_per_token_batches": 3e-06,
"output_cost_per_token_batches": 5e-06,
"input_cost_per_token_above_200k_tokens_batches": 13e-06,
"output_cost_per_token_above_200k_tokens_batches": 17e-06,
"output_cost_per_image_token_batches": 19e-06,
"input_cost_per_audio_token_batches": 7e-06,
"input_cost_per_audio_token_above_200k_tokens_batches": 11e-06,
}
}
)
usage: Final = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=20,
total_tokens=prompt_tokens + 20,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=10),
)
cost: Final = batch_cost_calculator(model=model, custom_llm_provider="vertex_ai", usage=usage)
audio_rate: Final = 11e-06 if prompt_tokens > 200_000 else 7e-06
input_rate: Final = 13e-06 if prompt_tokens > 200_000 else 3e-06
output_rate: Final = 17e-06 if prompt_tokens > 200_000 else 5e-06
assert sum(cost) == pytest.approx(
(prompt_tokens - 100) * input_rate + 100 * audio_rate + 10 * output_rate + 10 * 19e-06
)
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001))
def test_completion_cost_uses_flex_context_input_and_output_rates(
local_model_cost_map: None, prompt_tokens: int
) -> None:
model: Final = "gemini-synthetic-flex-context"
litellm.register_model(
{
model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 6e-06,
"output_cost_per_token": 10e-06,
"input_cost_per_token_flex": 3e-06,
"output_cost_per_token_flex": 5e-06,
"input_cost_per_token_above_200k_tokens_flex": 7e-06,
"output_cost_per_token_above_200k_tokens_flex": 11e-06,
}
}
)
response: Final = ModelResponse(
model=model, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=10, total_tokens=prompt_tokens + 10)
)
cost: Final = litellm.completion_cost(
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="flex"
)
input_rate: Final = 7e-06 if prompt_tokens > 200_000 else 3e-06
output_rate: Final = 11e-06 if prompt_tokens > 200_000 else 5e-06
assert cost == pytest.approx(prompt_tokens * input_rate + 10 * output_rate)
@pytest.mark.parametrize(
("audio_fields", "audio_rate"),
(
({}, 3e-06),
({"input_cost_per_audio_token": 7e-06}, 7e-06),
({"input_cost_per_audio_token_above_200k_tokens": 11e-06}, 11e-06),
({"input_cost_per_audio_token_above_200k_tokens_priority": 0.0}, 0.0),
),
)
def test_audio_context_rates_preserve_fallbacks_and_explicit_zero(
local_model_cost_map: None, audio_fields: Mapping[str, float], audio_rate: float
) -> None:
model: Final = "gemini-synthetic-audio-fallback"
litellm.register_model(
{
model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 5e-06,
**audio_fields,
}
}
)
response: Final = ModelResponse(
model=model,
usage=Usage(
prompt_tokens=200_001,
completion_tokens=10,
total_tokens=200_011,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=100),
),
)
cost: Final = litellm.completion_cost(
completion_response=response, model=model, custom_llm_provider="vertex_ai", service_tier="priority"
)
assert cost == pytest.approx(199_901 * 3e-06 + 100 * audio_rate + 10 * 5e-06)
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
return (
(PROMPT_TOKENS - CACHED_TOKENS) * prices[f"input_cost_per_token{column_suffix}"]

View file

@ -39,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
schema_has_unresolvable_ref,
schema_property_count,
split_concatenated_json_objects,
split_embedding_media_for_tpm,
split_media_for_tpm,
strip_encrypted_reasoning_from_messages,
strip_litellm_response_only_fields,
@ -2709,3 +2710,22 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
assert messages == messages_before
@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"])
def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None:
import gc
import tracemalloc
uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii")
split_embedding_media_for_tpm(f"data:{mime};base64,AAAA")
gc.collect()
tracemalloc.start()
try:
_, media = split_embedding_media_for_tpm(uri)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert media
assert peak < 1.5 * len(uri)

View file

@ -21,6 +21,8 @@ they stay out of the body even without it.
"""
import datetime
import json
from collections.abc import Mapping, Sequence
from typing import Any, Dict, Final, List, Optional, Tuple
from unittest.mock import AsyncMock, MagicMock, patch
@ -28,11 +30,13 @@ import pytest
from openai._legacy_response import LegacyAPIResponse
import litellm
from litellm.caching.caching import Cache
from litellm.integrations.code_interpreter_interception.handler import (
CodeInterpreterInterceptionLogger,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.chat_completion_agentic_loop import (
_execute_chat_completion_agentic_plan,
maybe_run_chat_completion_agentic_loop,
)
from litellm.litellm_core_utils.litellm_logging import Logging
@ -43,6 +47,7 @@ from litellm.types.integrations.custom_logger import (
WEBSEARCH_INTERCEPTION_PREFIX,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -50,6 +55,7 @@ from litellm.types.utils import (
Function,
Message,
ModelResponse,
Usage,
)
# The internal control fields that must never reach a provider request body.
@ -260,8 +266,6 @@ class _GateOnlyLogger(CustomLogger):
class _RecordingGateLogger(_GateOnlyLogger):
"""Also keeps every follow-up response the loop hands to the post hook."""
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
super().__init__(plan=plan, tool_calls=tool_calls)
self.followups: tuple[ModelResponse, ...] = ()
@ -441,72 +445,544 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ
assert call_kwargs["api_base"] == "https://a"
_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
_REFUSING_RAILS: Final = (
{"_agentic_loop_depth": 3, "max_agentic_loops": 3},
{
"_agentic_loop_depth": 0,
"max_agentic_loops": 3,
"_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)],
},
)
def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall:
return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}"))
async def _run_refused_rerun(
response: ModelResponse,
rail_kwargs: Dict[str, object],
stream: bool = False,
gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS,
other_hooks_tool_calls: tuple[Dict[str, object], ...] = (),
) -> Any:
litellm.callbacks = [
_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=tool_calls)
for tool_calls in (gate_tool_calls, *other_hooks_tool_calls)
]
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
try:
return await maybe_run_chat_completion_agentic_loop(
response=response,
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs=dict(rail_kwargs),
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=stream,
)
finally:
acompletion_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops(
restore_callbacks,
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call(
restore_callbacks, rail_kwargs: Dict[str, object]
):
"""depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops,
before any follow-up call is attempted."""
logger = _GateOnlyLogger(
plan=AgenticLoopPlan(run_agentic_loop=True),
result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs)
assert isinstance(result, ModelResponse)
assert not result.choices[0].message.tool_calls
assert result.choices[0].finish_reason == "stop"
@pytest.mark.asyncio
async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks):
response = ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content="Running it.",
tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")],
),
)
]
)
result = await _run_refused_rerun(response, _REFUSING_RAILS[0])
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
assert result.choices[0].finish_reason == "tool_calls"
assert result.choices[0].message.content == "Running it."
def _indexed_tool_call(call_id: str, name: str, index: int) -> ChatCompletionMessageToolCall:
return ChatCompletionMessageToolCall(
id=call_id, type="function", function=Function(name=name, arguments="{}"), index=index
)
def _turn_with_the_internal_call_between_the_clients() -> ModelResponse:
return ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
_indexed_tool_call("call_weather", "get_weather", 0),
_indexed_tool_call("call_abc", "litellm_code_execution", 1),
_indexed_tool_call("call_time", "get_time", 2),
],
),
)
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
async def test_refused_rerun_renumbers_the_kept_tool_calls_from_zero(restore_callbacks, rail_kwargs: Dict[str, object]):
result = await _run_refused_rerun(_turn_with_the_internal_call_between_the_clients(), rail_kwargs)
assert [(call.id, call.index) for call in result.choices[0].message.tool_calls] == [
("call_weather", 0),
("call_time", 1),
]
@pytest.mark.asyncio
async def test_refused_rerun_on_a_converted_stream_replays_the_kept_tool_calls_from_index_zero(
restore_callbacks, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setattr(
litellm,
"callbacks",
[_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=_REFUSED_GATE_TOOL_CALLS)],
)
logging_obj: Final = Logging(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="acompletion",
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
litellm_call_id="call-refused-stream",
function_id="fn-refused-stream",
)
result: Final = await maybe_run_chat_completion_agentic_loop(
response=_turn_with_the_internal_call_between_the_clients(),
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs={**_REFUSING_RAILS[1], f"{CODE_INTERPRETER_INTERCEPTION_PREFIX}_converted_stream": True},
logging_obj=logging_obj,
custom_llm_provider="openai",
stream=False,
)
assert isinstance(result, CustomStreamWrapper)
deltas: Final = [chunk.choices[0].delta async for chunk in result if chunk.choices]
streamed_calls: Final = [call for delta in deltas for call in delta.tool_calls or []]
assert [(call.id, call.index) for call in streamed_calls] == [("call_weather", 0), ("call_time", 1)]
@pytest.mark.asyncio
async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clients(restore_callbacks):
response = ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
_tool_call("call_abc", "litellm_code_execution"),
_tool_call("call_search", "litellm_web_search"),
_tool_call("call_client", "lookup"),
],
),
)
]
)
result = await _run_refused_rerun(
response,
_REFUSING_RAILS[0],
other_hooks_tool_calls=({"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]},),
)
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
@pytest.mark.asyncio
async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks):
response: Final = ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
_tool_call("call_abc", "litellm_code_execution"),
_tool_call("call_search", "litellm_web_search"),
_tool_call("call_client", "lookup"),
],
),
)
]
)
result: Final = await _run_refused_rerun(
response,
_REFUSING_RAILS[0],
gate_tool_calls={"tool_calls": [{"id": "call_abc"}]},
other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},),
)
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response, gate_tool_calls",
[(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)],
ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"],
)
async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged(
restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]]
):
result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls)
assert result.choices[0].model_dump() == response.choices[0].model_dump()
@pytest.mark.asyncio
async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks):
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True)
@pytest.mark.asyncio
async def test_dispatcher_surfaces_a_failed_follow_up_instead_of_the_tool_call_turn():
logger: Final = _GateOnlyLogger(
plan=AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"}
),
),
tool_calls={"tool_calls": [{"id": "call_abc"}]},
)
with pytest.raises(litellm.RateLimitError):
await _dispatch_tool_call_turn(_tool_call_model_response(), logger)
assert logger.cleanup_calls == 1
class _PlanRecordingLogger(_GateOnlyLogger):
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
super().__init__(plan=plan, tool_calls=tool_calls)
self.logging_objs: List[Any] = []
async def async_build_agentic_loop_plan(
self,
tools: Dict[str, Any],
model: str,
messages: List[Dict[str, Any]],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: Dict[str, Any],
logging_obj: Any,
stream: bool,
kwargs: Dict[str, Any],
) -> AgenticLoopPlan:
self.logging_objs.append(logging_obj)
return self._plan
@pytest.mark.asyncio
async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(restore_callbacks):
logger: Final = _PlanRecordingLogger(
plan=AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"}
),
),
tool_calls={"tool_calls": [{"id": "call_abc"}]},
)
litellm.callbacks = [logger]
first_call: Final = ModelResponse(
choices=_tool_call_model_response().choices, usage=Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
)
mock_client: Final = MagicMock()
mock_client.post = AsyncMock(return_value=_raw_response_for(first_call))
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
with pytest.raises(ValueError, match="max_agentic_loops"):
await maybe_run_chat_completion_agentic_loop(
response=_tool_call_model_response(),
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=False,
)
with pytest.raises(litellm.RateLimitError):
await litellm.acompletion(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "what is 6*7?"}],
api_key="sk-test",
client=mock_client,
num_retries=0,
)
acompletion_mock.assert_not_awaited()
(logging_obj,) = logger.logging_objs
price: Final = litellm.model_cost["gpt-4o-mini"]
assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8
assert logging_obj.model_call_details["response_cost"] == pytest.approx(
5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"]
)
async def _agentic_turn_with_follow_up(
logger: _PlanRecordingLogger, monkeypatch: pytest.MonkeyPatch
) -> tuple[ModelResponse, Logging]:
monkeypatch.setattr(litellm, "callbacks", [logger])
mock_client: Final = MagicMock()
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_model_response()))
response: Final = await litellm.acompletion(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "what is 6*7?"}],
api_key="sk-test",
client=mock_client,
num_retries=0,
)
assert isinstance(response, ModelResponse)
(logging_obj,) = logger.logging_objs
return response, logging_obj
def _follow_up_logger() -> _PlanRecordingLogger:
return _PlanRecordingLogger(
plan=AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": "42"}),
),
tool_calls={"tool_calls": [{"id": "call_abc"}]},
)
@pytest.mark.asyncio
async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks):
"""A tool_calls fingerprint already present in _agentic_loop_fingerprints ->
ValueError about the repeated fingerprint (cycle guard), with no follow-up
call."""
import json
async def test_acompletion_records_the_cost_breakdown_of_a_successful_follow_up(monkeypatch):
response, logging_obj = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch)
# The dispatcher fingerprints the whole value the gate returns as its second
# tuple element, so the seeded fingerprint must mirror that dict exactly.
gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str)
response_cost: Final = response._hidden_params["response_cost"]
assert response_cost > 0
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["total_cost"] == pytest.approx(response_cost)
logger = _GateOnlyLogger(
plan=AgenticLoopPlan(run_agentic_loop=True),
tool_calls=gate_tool_calls,
@pytest.mark.asyncio
async def test_acompletion_keeps_a_cached_follow_up_free(monkeypatch):
monkeypatch.setattr(litellm, "cache", Cache(type="local"))
await litellm.acompletion(model="openai/gpt-4o-mini", messages=_patched_messages(), mock_response="42")
response, _ = await _agentic_turn_with_follow_up(_follow_up_logger(), monkeypatch)
assert response._hidden_params["cache_hit"] is True
assert response._hidden_params["response_cost"] == 0
def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse:
return ModelResponse(
model="gpt-4o-mini",
choices=[
Choices(
finish_reason="tool_calls",
message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]),
)
],
usage=usage,
)
litellm.callbacks = [logger]
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
with pytest.raises(ValueError, match="fingerprint"):
await maybe_run_chat_completion_agentic_loop(
response=_tool_call_model_response(),
class _TwoRoundLogger(CustomLogger):
def __init__(self, middle_round: ModelResponse) -> None:
super().__init__()
self._follow_ups: Final = (middle_round, "litellm.RateLimitError")
self.logging_objs: tuple[object, ...] = ()
async def async_should_run_agentic_loop(
self,
response: object,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
assert isinstance(response, ModelResponse)
calls: Final = response.choices[0].message.tool_calls or []
return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]}
async def async_build_agentic_loop_plan(
self,
tools: Mapping[str, object],
model: str,
messages: Sequence[Mapping[str, object]],
response: object,
anthropic_messages_provider_config: object,
anthropic_messages_optional_request_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
follow_up: Final = self._follow_ups[len(self.logging_objs)]
self.logging_objs = (*self.logging_objs, logging_obj)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}),
)
@pytest.mark.asyncio
async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails(
restore_callbacks, monkeypatch: pytest.MonkeyPatch
):
first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15)
logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage))
monkeypatch.setattr(litellm, "callbacks", [logger])
mock_client: Final = MagicMock()
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage)))
with pytest.raises(litellm.RateLimitError):
await litellm.acompletion(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "what is 6*7?"}],
api_key="sk-test",
client=mock_client,
num_retries=0,
)
first_logging_obj, _ = logger.logging_objs
assert isinstance(first_logging_obj, Logging)
price: Final = litellm.model_cost["gpt-4o-mini"]
billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"]
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4)
assert first_logging_obj.model_call_details["response_cost"] == pytest.approx(
(5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool):
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
sandbox_error: Final = RuntimeError("sandbox crashed")
async def failing_follow_up() -> ModelResponse:
raise sandbox_error
with pytest.raises(RuntimeError) as raised:
await bill_first_round_on_failure(
failing_follow_up(),
logging_obj=Logging(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs={
"_agentic_loop_depth": 0,
"max_agentic_loops": 3,
"_agentic_loop_fingerprints": [fingerprint],
},
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=False,
)
stream=stream,
call_type="acompletion",
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
litellm_call_id="call-builtin-error",
function_id="fn-builtin-error",
),
response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)),
stream=stream,
)
acompletion_mock.assert_not_awaited()
assert raised.value is sandbox_error
class _RunOnlyLoopLogger(CustomLogger):
def __init__(self, result: ModelResponse) -> None:
super().__init__()
self._result = result
async def async_should_run_agentic_loop(
self,
response: Any,
model: str,
messages: List[Dict[str, Any]],
tools: Optional[List[Dict[str, Any]]],
stream: bool,
custom_llm_provider: str,
kwargs: Dict[str, Any],
) -> Tuple[bool, Dict[str, Any]]:
return True, {"tool_calls": [{"id": "call_abc"}]}
async def async_run_agentic_loop(
self,
tools: Dict[str, Any],
model: str,
messages: List[Dict[str, Any]],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: Dict[str, Any],
logging_obj: Any,
stream: bool,
kwargs: Dict[str, Any],
) -> Any:
return self._result
class _LoggingWithCallbacks(_LoggingStub):
def __init__(self, callbacks: Tuple[CustomLogger, ...]) -> None:
self.dynamic_success_callbacks = list(callbacks)
async def _dispatch_tool_call_turn(
response: ModelResponse, *callbacks: CustomLogger
) -> ModelResponse | CustomStreamWrapper | None:
return await maybe_run_chat_completion_agentic_loop(
response=response,
model="gpt-4o-mini",
messages=[{"role": "user", "content": "what is 6*7?"}],
optional_params={},
kwargs={},
logging_obj=_LoggingWithCallbacks(callbacks),
custom_llm_provider="openai",
stream=False,
)
@pytest.mark.asyncio
async def test_dispatcher_returns_the_tool_call_turn_when_the_plan_terminates():
response: Final = _tool_call_model_response()
terminating: Final = _GateOnlyLogger(
plan=AgenticLoopPlan(terminate=True), tool_calls={"tool_calls": [{"id": "call_abc"}]}
)
assert await _dispatch_tool_call_turn(response, terminating) is response
@pytest.mark.asyncio
async def test_dispatcher_moves_past_a_plan_that_declines_the_loop():
override: Final = _plain_model_response("from the next callback")
tool_calls: Final = {"tool_calls": [{"id": "call_abc"}]}
declined: Final = _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=False), tool_calls=tool_calls)
overriding: Final = _GateOnlyLogger(plan=AgenticLoopPlan(response_override=override), tool_calls=tool_calls)
assert await _dispatch_tool_call_turn(_tool_call_model_response(), declined, overriding) is override
@pytest.mark.asyncio
async def test_dispatcher_returns_a_run_only_loops_result():
loop_result: Final = _plain_model_response()
assert await _dispatch_tool_call_turn(_tool_call_model_response(), _RunOnlyLoopLogger(loop_result)) is loop_result
@pytest.mark.asyncio
@ -562,3 +1038,55 @@ async def test_converted_stream_replays_the_final_answer_with_the_requested_usag
if include_usage
else []
)
class TestAgenticFollowUpKeepsTheProviderPrefix:
@staticmethod
async def _run_followup(
model: str, custom_llm_provider: str, patched_model: str | None = None
) -> ModelResponse:
response: Final = await _execute_chat_completion_agentic_plan(
plan=AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
messages=[{"role": "user", "content": "hi"}], model=patched_model
),
),
callback=CustomLogger(),
model=model,
optional_params={"mock_response": "ok from followup"},
kwargs={},
logging_obj=None,
custom_llm_provider=custom_llm_provider,
depth=0,
max_loops=2,
fingerprints=[],
fingerprint="fp",
)
assert isinstance(response, ModelResponse)
return response
@pytest.mark.asyncio
async def test_a_sub_path_model_still_resolves_a_provider(self) -> None:
response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock")
assert response.choices[0].message.content == "ok from followup"
@pytest.mark.asyncio
async def test_an_ordinary_model_still_resolves_a_provider(self) -> None:
response: Final = await self._run_followup("gpt-4o-mini", "openai")
assert response.choices[0].message.content == "ok from followup"
@pytest.mark.asyncio
async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None:
response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5")
assert response.model == "claude-sonnet-4-5"
assert response._hidden_params["custom_llm_provider"] == "anthropic"
@pytest.mark.asyncio
async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None:
response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o")
assert response._hidden_params["custom_llm_provider"] == "openrouter"

View file

@ -217,6 +217,22 @@ def test_unknown_exception_falls_back_to_status_then_unclassified() -> None:
assert normalize_error(Exception("x"), "", "x") == "UNCLASSIFIED"
def test_a_529_the_anthropic_routes_answered_is_an_overload() -> None:
paused = _proxy_exc("Model is paused", "service_unavailable_error", 529)
assert StandardLoggingPayloadSetup.get_error_information(paused)["normalized_error"] == "503_PROVIDER_OVERLOADED"
@pytest.mark.parametrize("body", ["", "<html>bad gateway</html>", '{"type":"error","error":{"type":"api_error"}}'])
def test_an_internal_server_error_answered_as_529_is_an_overload_whatever_its_body_says(body: str) -> None:
overloaded = litellm.InternalServerError(body, llm_provider="anthropic", model="claude", provider_status_code=529)
overloaded.status_code = 529
internal = litellm.InternalServerError(body, llm_provider="anthropic", model="claude")
assert (
StandardLoggingPayloadSetup.get_error_information(overloaded)["normalized_error"],
StandardLoggingPayloadSetup.get_error_information(internal)["normalized_error"],
) == ("503_PROVIDER_OVERLOADED", "500_PROVIDER_INTERNAL_ERROR")
def test_budget_exceeded_error_with_custom_wording_is_still_a_budget_error() -> None:
exc = litellm.BudgetExceededError(current_cost=2.0, max_budget=1.0, message="Spending cap reached for key")
assert StandardLoggingPayloadSetup.get_error_information(exc)["normalized_error"] == "429_BUDGET_EXCEEDED"

Some files were not shown because too many files have changed in this diff Show more