scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

Merge main into litellm_bill_video_cost_once

Browse source
This commit is contained in:
mateo-berri 2026-11-03 16:46:55 -08:00
commit e475fcc7a5
167 changed files with 12430 additions and 2706 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm-rust/crates/cost/src/price.rs
View file

@ -151,6 +151,8 @@ pub enum Uplift {
Residency(Residency),
/// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
NonGlobalEndpoint,
/// OpenAI FedRAMP endpoint (gov.api.openai.com), from `fedramp_uplift_multiplier`
FedrampEndpoint,
/// Anthropic `speed: fast`, from `provider_specific_entry.fast`
Speed(Speed),
/// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main

10
litellm-rust/crates/model-catalog/src/lower.rs
View file

@ -104,6 +104,7 @@ impl From<&EntryWire> for Adjustments {
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
(Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
(Uplift::FedrampEndpoint, wire.fedramp_uplift),
]
.into_iter()
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
@ -746,7 +747,8 @@ mod tests {
fn uplifts_are_adjustments_not_prices() {
let wire = wire(json!({
"regional_endpoint_uplift_multiplier": 1.1,
"regional_processing_uplift_multiplier_eu": "1.2"
"regional_processing_uplift_multiplier_eu": "1.2",
"fedramp_uplift_multiplier": 1.1
}));
let adjustments = Adjustments::from(&wire);
@ -764,6 +766,12 @@ mod tests {
Some(1.2)
);
assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
assert_eq!(
adjustments
.get(&Uplift::FedrampEndpoint)
.map(Multiplier::get),
Some(1.1)
);
}
#[rstest]

1
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -96,6 +96,7 @@ pub struct ModelInfoFields {
pub output_cost_per_token_ultrafast: Option<f64>,
pub regional_processing_uplift_multiplier_eu: Option<f64>,
pub regional_processing_uplift_multiplier_us: Option<f64>,
pub fedramp_uplift_multiplier: Option<f64>,
pub regional_endpoint_uplift_multiplier: Option<f64>,
pub output_cost_per_audio_token: Option<f64>,
pub output_cost_per_character: Option<f64>,

2
litellm-rust/crates/model-catalog/src/schema.rs
View file

@ -29,6 +29,8 @@ struct PriceBlocks {
regional_processing_uplift_multiplier_eu: Option<Multiplier>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
regional_processing_uplift_multiplier_us: Option<Multiplier>,
/// Multiplier applied to all token costs when served from the OpenAI FedRAMP endpoint (e.g. 1.10 = +10%).
fedramp_uplift_multiplier: Option<Multiplier>,
/// USD cost per web search query, keyed by search context size.
search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.

8
litellm-rust/crates/model-catalog/src/wire/attributes.rs
View file

@ -93,6 +93,10 @@ attributes! {
search_free_requests: SearchFreeRequests,
/// URL of the provider pricing/model page this entry was taken from.
source: String,
/// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists.
successor_model: String,
/// URL of the provider page or feed `deprecation_date` was read from.
deprecation_source: String,
/// Audio container formats the model can return.
supported_audio_formats: Vec<AudioFormat>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
@ -303,6 +307,10 @@ mod tests {
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::successor_model("successor_model", json!("gpt-6-luna"), true)]
#[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)]
#[case::deprecation_source("deprecation_source", json!("https://developers.openai.com/api/docs/deprecations"), true)]
#[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)]
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]

2
litellm-rust/crates/model-catalog/src/wire/entry.rs
View file

@ -24,6 +24,7 @@ pub struct EntryWire {
pub regional_endpoint_uplift: Option<Multiplier>,
pub residency_uplift_eu: Option<Multiplier>,
pub residency_uplift_us: Option<Multiplier>,
pub fedramp_uplift: Option<Multiplier>,
pub unknown: Fields,
pub problems: Vec<Error>,
}
@ -57,6 +58,7 @@ impl EntryWire {
"regional_processing_uplift_multiplier_us" => {
self.residency_uplift_us = reader.value(path(), value)
}
"fedramp_uplift_multiplier" => self.fedramp_uplift = reader.value(path(), value),
_ => self.read_open_key(reader, key, value),
}
}

586
litellm/blog_posts.json
View file

@ -1,10 +1,588 @@
{
"posts": [
{
"title": "Realtime WebRTC HTTP Endpoints",
"description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.",
"date": "2026-03-12",
"url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints"
"title": "Release cadence changes and an LTS version pilot",
"description": "Starting the week of November 2, 2026, release candidates and stable releases ship on Tuesday mornings, and LiteLLM is piloting a long-term support line on 1.108 for one month.",
"date": "2026-11-02",
"url": "https://docs.litellm.ai/blog/release-cadence-and-lts-pilot",
"tags": [
"release",
"support"
]
},
{
"title": "How to build an agent with grounding on LiteLLM",
"description": "Build a grounded chat agent in four steps with LiteLLM's AI Gateway. Connect your evidence, call a grounding guardrail, and show its verdict in chat.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/grounded-agent",
"tags": [
"engineering",
"guardrails",
"agents"
]
},
{
"title": "Introducing LiteLLM’s Unified Cost Accounting Engine",
"description": "Consistent cost reporting, independently scalable accounting, and spend analysis by team, user, model, or custom dimensions.",
"date": "2026-10-31",
"url": "https://docs.litellm.ai/blog/cost-infrastructure-makeover",
"tags": [
"engineering",
"reliability"
]
},
{
"title": "What we are changing after October 21",
"description": "Why the controls built after March did not stop the v1.107.0 release, and the identity, credential and detection changes we are making in response.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/security-changes-after-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Use Hex to analyze LiteLLM costs, then set budgets and routing",
"description": "Connect Hex to LiteLLM’s Postgres data, import a cost dashboard, then use the LiteLLM UI to set a budget or change model routing.",
"date": "2026-10-30",
"url": "https://docs.litellm.ai/blog/hex-litellm-roi",
"tags": [
"observability",
"cost tracking",
"ai-gateway"
]
},
{
"title": "Introducing the LiteLLM Private Registry",
"description": "LiteLLM is committing to a private registry to provide early access to fixes for the most severe security vulnerabilities, ahead of public release.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/litellm-private-registry",
"tags": [
"security"
]
},
{
"title": "Morph open-weight models on LiteLLM",
"description": "Use Morph's Kimi, GLM, and DeepSeek models through LiteLLM with tool calling, structured output, streaming, and prompt-cache cost tracking.",
"date": "2026-10-27",
"url": "https://docs.litellm.ai/blog/morph-provider",
"tags": []
},
{
"title": "Abliterated models on LiteLLM",
"description": "Call abliterated models from Refuseless and Abliteration through LiteLLM, with gateway access controls, streaming, tool calls, and cached-token pricing.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/abliterated-models",
"tags": [
"engineering"
]
},
{
"title": "GPT-6.1 Terra API Pricing and Day 0 Support on LiteLLM",
"description": "GPT-6.1 Terra costs $1 per 1M input tokens and $5 per 1M output tokens, half the price of GPT-6.1 Sol. LiteLLM supports it on day 0.",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/gpt-6-1-terra",
"tags": [
"openai",
"gpt-6.1",
"gpt-6.1-terra",
"pricing",
"day 0 support",
"card_title:GPT-6.1 Terra",
"card_summary:Half the price of GPT-6.1 Sol at $1 / $5 per 1M tokens, 50.3% on Terminal-Bench 4.0, 1.05M context."
]
},
{
"title": "Rust Migration: /v1/messages end-to-end in Rust by Dec 1",
"description": "By December 1st, the proxy will handle Anthropic messages requests 20x faster",
"date": "2026-10-26",
"url": "https://docs.litellm.ai/blog/rust-migration-rerouted",
"tags": [
"rust",
"rust-migration",
"ai-gateway",
"engineering"
]
},
{
"title": "OpenCode Go is now a LiteLLM provider",
"description": "Use your OpenCode Go subscription through LiteLLM. Connect once, route models to the right API, preserve sessions, and track estimated usage.",
"date": "2026-10-24",
"url": "https://docs.litellm.ai/blog/opencode-go",
"tags": [
"providers",
"tutorial"
]
},
{
"title": "The attack chain behind the v1.107.0 LiteLLM release",
"description": "How an internal key committed to a public repository in May led to three stolen GitHub tokens and the compromised litellm v1.107.0 release.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain",
"tags": [
"security",
"incident-report"
]
},
{
"title": "Tokenize text across vLLM and Watsonx",
"description": "Choose token counting for prompt budgets or /tokenize to inspect tokens. Use one request format across vLLM and Watsonx through LiteLLM.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/unified-tokenize",
"tags": [
"engineering"
]
},
{
"title": "October Townhall Updates: 426 Bug Fixes, 88.4% Coverage, and the Decisions API",
"description": "A recap of the October LiteLLM town hall: security changes that need action before you upgrade, 426 bug fixes, test coverage at 88.4%, the Decisions API, Lens, Moyai, and LiteLLM Relay.",
"date": "2026-10-23",
"url": "https://docs.litellm.ai/blog/october-townhall-updates",
"tags": [
"townhall",
"security",
"reliability",
"product"
]
},
{
"title": "Contained: Oct 21, 2026 LiteLLM release v1.107.0 Compromised",
"description": "LiteLLM release v1.107.0 was compromised by a malicious actor.",
"date": "2026-10-21",
"url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026",
"tags": [
"security",
"incident-report"
]
},
{
"title": "JevRank: decision models through LiteLLM’s rerank API",
"description": "Preview JevRank for low-cost reranking with Perplexity and TypeSafe decision models. Explore SciFact results, reproducible data and LiteLLM setup.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/jevrank",
"tags": [
"engineering",
"reranking",
"decisions"
]
},
{
"title": "Cache-Aware Routing for Self-Hosted Models",
"description": "Route each agent session to the same vLLM pod on Kubernetes to reuse cached prompt prefixes.",
"date": "2026-10-20",
"url": "https://docs.litellm.ai/blog/cache-aware-routing",
"tags": [
"routing",
"kubernetes",
"vllm",
"ai-gateway"
]
},
{
"title": "Day 0 Support: TypeSafe Jev 1.14: Adds images, box questions, and long context",
"description": "TypeSafe Jev 1.14 reads images, answers box questions with coordinates and takes long-context requests. LiteLLM serves all three on /v1/systemone and /v1/decisions behind TypeSafe's opt-in headers.",
"date": "2026-10-15",
"url": "https://docs.litellm.ai/blog/typesafe_jev_images",
"tags": [
"typesafe",
"jev",
"decisions",
"day 0 support",
"card_title:Jev 1.14",
"card_summary:Jev 1.14 reads PNG and JPEG images, answers box questions with coordinates, and takes long-context requests on LiteLLM."
]
},
{
"title": "Decision models on LiteLLM: /v1/decisions and /v1/systemone",
"description": "Decision models return probabilities instead of text, which makes routing, triage and guardrail checks cheap and deterministic. LiteLLM serves nine providers of them behind one request shape.",
"date": "2026-10-14",
"url": "https://docs.litellm.ai/blog/decision-models",
"tags": [
"openai",
"typesafe",
"decisions",
"day 0 support",
"card_title:Decision Models",
"card_summary:Typed yes/no, choice and score answers as probabilities, from 9 providers on /v1/decisions and /v1/systemone."
]
},
{
"title": "Workshop: Build a self-hosted cloud agent.",
"description": "Build a self-hosted cloud agent with the LiteLLM team. A two-week remote workshop, October 19–30, to build a working v0 and deploy an internal beta on your infrastructure.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/cloud-agent-workshop",
"tags": [
"agents",
"infrastructure",
"announcement"
]
},
{
"title": "Gemini 4 Argon API Pricing and Day 0 Support on LiteLLM",
"description": "Gemini 4 Argon costs $2.00 per 1M input tokens and $10.00 per 1M output tokens until Nov 30. LiteLLM supports it on day 0 on AI Studio and Google Cloud.",
"date": "2026-10-13",
"url": "https://docs.litellm.ai/blog/gemini-4-argon",
"tags": [
"gemini",
"gemini 4",
"gemini-4-argon",
"pricing",
"day 0 support",
"card_title:Gemini 4 Argon",
"card_summary:Google's new flagship: 77.9% on DeepSWE v1.1, 1M context, thinking levels, $2 / $10 per 1M tokens until Nov 30."
]
},
{
"title": "October Townhall: Product + Roadmap Updates",
"description": "Join the LiteLLM October townhall on Friday, 23 October at 7:30 AM PT to learn about LiteLLM's product updates and roadmap.",
"date": "2026-10-10",
"url": "https://docs.litellm.ai/blog/october-townhall-announcement",
"tags": [
"announcement",
"townhall"
]
},
{
"title": "Introducing Microsoft 365 Copilot in LiteLLM",
"description": "Use Microsoft 365 Copilot from your applications through LiteLLM, with access to your Microsoft 365 work data.",
"date": "2026-10-09",
"url": "https://docs.litellm.ai/blog/microsoft-365-copilot",
"tags": [
"announcement",
"ai-gateway"
]
},
{
"title": "Introducing litellm-core: the first step toward a leaner SDK",
"description": "Ongoing work to reduce LiteLLM's dependencies and installation footprint, improve runtime loading, and make imports faster",
"date": "2026-10-08",
"url": "https://docs.litellm.ai/blog/introducing-litellm-core",
"tags": []
},
{
"title": "Claude Haiku 5.5 API Pricing and Day 0 Support on LiteLLM",
"description": "Claude Haiku 5.5 costs $0.10 per 1M input tokens and $0.50 per 1M output tokens. LiteLLM supports it on day 0 on Anthropic, AWS, Google Cloud and Azure.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/claude-haiku-5-5",
"tags": [
"anthropic",
"claude",
"claude haiku",
"haiku 5.5",
"pricing",
"day 0 support",
"card_title:Claude Haiku 5.5",
"card_summary:90% cheaper than Haiku 4.5 on short prompts, 1M context, 72.4% on OSWorld 2.1, $0.10 / $0.50 per 1M tokens."
]
},
{
"title": "Open Sourcing Moyai: Self-Hosted Cloud Coding Agent",
"description": "Run cloud coding agents on your own infrastructure with Moyai. Choose your agent and models through LiteLLM, track costs, and follow the setup guide.",
"date": "2026-10-07",
"url": "https://docs.litellm.ai/blog/moyai-open-source",
"tags": [
"agents",
"open-source",
"infrastructure"
]
},
{
"title": "Day 0 Support: Nano Banana 2.1",
"description": "Day 0 support for Gemini Nano Banana 2.1 on LiteLLM, on Google AI Studio and Gemini Enterprise Agent Platform, with image output at half Nano Banana 2's price.",
"date": "2026-10-06",
"url": "https://docs.litellm.ai/blog/nano_banana_2_1",
"tags": [
"gemini",
"gemini enterprise agent platform",
"nano banana",
"image generation",
"day 0 support",
"card_title:Nano Banana 2.1",
"card_summary:Gemini image model with image output at half Nano Banana 2's price: $30 / MTok image out, $1.50 text in."
]
},
{
"title": "How LiteLLM Lens finds repeated failures across 1,000s of agent traces",
"description": "Inside Lens: parallel trace review, Python tools for large traces, and investigations that connect repeated failures to source evidence.",
"date": "2026-10-05",
"url": "https://docs.litellm.ai/blog/lens-failure-patterns",
"tags": [
"lens",
"agent-tracing",
"engineering"
]
},
{
"title": "How we cut time to first byte by 94% for long prompts",
"description": "A 440k-token benchmark went from 553 ms to 35 ms median time to first byte. We removed unnecessary prompt-cache routing work before the model call.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/long-prompt-latency",
"tags": [
"performance",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we built our own internal Devin in 2 days",
"description": "How we built Moyai Devin with Render, Modal, Hermes, Temporal, and LiteLLM: durable sessions, parallel agents, Slack, and shared organization connections.",
"date": "2026-10-03",
"url": "https://docs.litellm.ai/blog/internal-devin-two-days",
"tags": [
"engineering",
"agents",
"infrastructure",
"slack"
]
},
{
"title": "Adding Self-hosted Auto Router Classifiers: Laya & Nimble",
"description": "Use Laya or Bespoke Nimble to classify Auto Router requests on your own infrastructure. Control where classification runs, which model you serve, and how you provision it.",
"date": "2026-10-02",
"url": "https://docs.litellm.ai/blog/laya-nimble-classifiers",
"tags": [
"auto-router",
"product",
"ai-gateway"
]
},
{
"title": "Launching LiteLLM Lens",
"description": "LiteLLM Lens turns the traces flowing through your gateway into findings your agents can act on. Built for agent swarms generating 200K+ traces.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/litellm-lens-launch",
"tags": [
"lens",
"agent-tracing"
]
},
{
"title": "How we cut LiteLLM's Redis round trips per request by 64%",
"description": "A governed request to the LiteLLM AI Gateway waited on Redis 22 times. It now waits 8 times: one pipeline per Redis backend before the model call, one after.",
"date": "2026-10-01",
"url": "https://docs.litellm.ai/blog/redis-request-pipelines",
"tags": [
"performance",
"redis",
"proxy",
"engineering",
"ai-gateway"
]
},
{
"title": "How we made the LiteLLM Usage page 120x faster",
"description": "From six minutes to 3.2 seconds: how moving aggregation into Postgres made LiteLLM's Usage page roughly 120x faster in our benchmark.",
"date": "2026-09-30",
"url": "https://docs.litellm.ai/blog/usage-page-bounded-loading",
"tags": [
"performance",
"admin-ui",
"postgres",
"engineering",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6.1 Sol",
"description": "Day 0 support for GPT-6.1 Sol on LiteLLM, with cached input at half GPT-6 Sol's price.",
"date": "2026-09-29",
"url": "https://docs.litellm.ai/blog/gpt_6_1_sol",
"tags": [
"openai",
"gpt-6",
"gpt-6.1-sol",
"completion",
"day 0 support",
"card_title:GPT-6.1 Sol",
"card_summary:GPT-6 Sol successor with cached input at half GPT-6 Sol's price, same request family and params."
]
},
{
"title": "Learn the LiteLLM gateway with a guided course",
"description": "Follow a request through the LiteLLM gateway, Router, and SDK. A guided course for teams deploying the gateway and contributors making code changes.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/litellm-gateway-course",
"tags": [
"proxy",
"product"
]
},
{
"title": "Day 0 Support: Claude Sonnet 5.5",
"description": "Day 0 support for Claude Sonnet 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-28",
"url": "https://docs.litellm.ai/blog/claude_sonnet_5_5",
"tags": [
"anthropic",
"claude",
"sonnet 5.5",
"day 0 support",
"card_title:Claude Sonnet 5.5",
"card_summary:Same $2 / $10 price as Sonnet 5, 30%+ faster, 70.6% on Terminal-Bench 4.0 (Sonnet 5: 10.3%), 1M context."
]
},
{
"title": "Introducing the LiteLLM ROI Calculator",
"description": "Compare each engineer's LiteLLM gateway spend with the estimated effort of their merged pull requests. Open source and self-hosted.",
"date": "2026-09-26",
"url": "https://docs.litellm.ai/blog/litellm-roi-calculator",
"tags": [
"cost",
"product",
"announcement"
]
},
{
"title": "Introducing LiteAgents",
"description": "Switch agent harnesses without rewriting your agent. Keep your tools and model configuration, use native harness controls, and add Temporal when you need durable runs.",
"date": "2026-09-25",
"url": "https://docs.litellm.ai/blog/liteagents-sdk",
"tags": [
"agents",
"sdk"
]
},
{
"title": "September Townhall Updates: 583 Bug Fixes, OCR on Rust, and 83.5% Coverage",
"description": "A recap of the September LiteLLM town hall: security and stability updates, OCR running on Rust by default, test coverage past the 80% target, and new Fusion and liteagents launches.",
"date": "2026-09-24",
"url": "https://docs.litellm.ai/blog/september-townhall-updates",
"tags": [
"townhall",
"reliability",
"product"
]
},
{
"title": "Day 0 support: Gemini 3.8 Flash TTS and Flash-Lite TTS",
"description": "Day 0 support for Gemini 3.8 Flash TTS and Flash-Lite TTS on LiteLLM's /v1/audio/speech, with launch pricing tracked through 2026.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/gemini_3_8_flash_tts",
"tags": [
"gemini",
"gemini-3.8-flash-tts",
"text to speech",
"day 0 support",
"card_title:Gemini 3.8 Flash TTS and Flash-Lite TTS",
"card_summary:Gemini speech models on /v1/audio/speech, Flash and Flash-Lite, with launch pricing tracked through 2026."
]
},
{
"title": "Introducing LiteAdmin MCP",
"description": "Give your agent tools to create keys, add models, and manage budgets with LiteAdmin MCP. Use the same connector through LiteAdmin, our Slack admin agent.",
"date": "2026-09-23",
"url": "https://docs.litellm.ai/blog/litellm-admin-mcp",
"tags": [
"mcp",
"agents",
"ai-gateway"
]
},
{
"title": "Day 0 Support: GPT-6 Sol and GPT-6 Luna",
"description": "Day 0 support for GPT-6 Sol and GPT-6 Luna on LiteLLM, at half the price of their GPT-5.6 counterparts.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/gpt_6_sol_luna",
"tags": [
"openai",
"gpt-6",
"gpt-6-sol",
"gpt-6-luna",
"completion",
"day 0 support",
"card_title:GPT-6 Sol and GPT-6 Luna",
"card_summary:Two GPT-6 variants at half the price of their GPT-5.6 counterparts, routed as the GPT-5 request family."
]
},
{
"title": "Day 0 Support: Claude Opus 5.5",
"description": "Day 0 support for Claude Opus 5.5 on the LiteLLM AI Gateway. Use it across Anthropic, Bedrock, Gemini Enterprise Agent Platform, and Azure.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/claude_opus_5_5",
"tags": [
"anthropic",
"claude",
"opus 5.5",
"day 0 support",
"card_title:Claude Opus 5.5",
"card_summary:20% cheaper than Opus 5 at $4 / $20 per 1M, 30% faster output, Terminal-Bench 4.0 66.4%, thinking always on."
]
},
{
"title": "Getting Started with Fireworks AI on LiteLLM",
"description": "A beginner tutorial: run an open model on Fireworks AI behind a LiteLLM gateway, call it with the OpenAI SDK you already use, then add a second model and a fallback.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/fireworks-getting-started",
"tags": [
"tutorial",
"providers",
"fireworks",
"getting-started"
]
},
{
"title": "Day 0 Support: Xiaomi MiMo V2.6",
"description": "Day 0 support for Xiaomi MiMo V2.6 Pro and Flash on LiteLLM, priced on the native route for the first time.",
"date": "2026-09-22",
"url": "https://docs.litellm.ai/blog/mimo_v2_6",
"tags": [
"xiaomi",
"mimo",
"mimo-v2.6",
"day 0 support",
"card_title:Xiaomi MiMo V2.6",
"card_summary:Xiaomi's Pro ($0.435 / $0.87 per 1M) and Flash ($0.14 / $0.28), same rates as V2.5, priced on the native route."
]
},
{
"title": "Claude Code server-side auto mode through LiteLLM",
"description": "Anthropic is moving Claude Code auto mode's safety classifier server-side. LiteLLM's native /v1/messages route now forwards the safeguards contract to the Anthropic API, Bedrock InvokeModel, Bedrock Mantle and Vertex AI. Here is the contract, what changed, which releases carry it, and how to verify it.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/claude-code-server-side-auto-mode",
"tags": [
"announcement",
"claude-code",
"anthropic",
"ai-gateway"
]
},
{
"title": "Day 0 Support: Grok 4.7",
"description": "Day 0 support for Grok 4.7 on LiteLLM, at the same price as Grok 4.6.",
"date": "2026-09-21",
"url": "https://docs.litellm.ai/blog/grok_4_7",
"tags": [
"xai",
"grok",
"grok-4.7",
"day 0 support",
"card_title:Grok 4.7",
"card_summary:SpaceXAI's update to Grok 4.6 at the same price and limits, better at checking its own work and long context."
]
},
{
"title": "JEV Classifier: 5.43x as Fast as Haiku, 96% Lower Cost",
"description": "JEV classified requests 5.43x as fast as Haiku by median latency in our AI Gateway benchmark. Explore the setup, cost savings and methodology.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/jev-auto-router-benchmark",
"tags": [
"routing",
"complexity-router",
"engineering",
"ai-gateway"
]
},
{
"title": "TypeSafe Jev on LiteLLM",
"description": "TypeSafe AI's Jev lands in LiteLLM v1.103.0-rc: call it through the proxy with logging and cost tracking.",
"date": "2026-09-20",
"url": "https://docs.litellm.ai/blog/typesafe_jev",
"tags": [
"typesafe",
"jev",
"product"
]
}
]
}

43
litellm/cost_calculator.py
View file

@ -1102,20 +1102,19 @@ def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_nam
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
def declared_cost_map_pricing(key: str, model: str, custom_llm_provider: str | None) -> Mapping[str, object]:
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return unmapped_model_generalization(model, custom_llm_provider) or MappingProxyType({})
fill_missing: Final = mapped_model_fill_missing(key, entry)
return entry if fill_missing is None else MappingProxyType({**fill_missing, **entry})
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
return key is not None and _cost_map_entry_prices_anything(
declared_cost_map_pricing(key, model, custom_llm_provider)
)
def _served_or_priced_requested_model(
@ -2420,18 +2419,20 @@ def completion_cost(
raise e
def get_response_cost_from_hidden_params(
hidden_params: dict | BaseModel,
) -> float | None:
if isinstance(hidden_params, BaseModel):
_hidden_params_dict = hidden_params.model_dump()
else:
_hidden_params_dict = hidden_params
_ADDITIONAL_HEADERS: Final = TypeAdapter(Mapping[str, object])
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def get_response_cost_from_hidden_params(
hidden_params: Mapping[str, object] | BaseModel,
) -> float | None:
params: Final[Mapping[str, object]] = (
hidden_params.model_dump() if isinstance(hidden_params, BaseModel) else hidden_params
)
additional_headers: Final = params.get("additional_headers")
if not isinstance(additional_headers, Mapping):
return None
headers: Final = _ADDITIONAL_HEADERS.validate_python(additional_headers)
return _billable_header_cost(headers.get("llm_provider-x-litellm-response-cost"))
def _billable_header_cost(value: object) -> float | None:

10
litellm/litellm_core_utils/deployment_pricing.py
View file

@ -1,4 +1,4 @@
from collections.abc import Iterator, Mapping
from collections.abc import Iterable, Iterator, Mapping
from types import MappingProxyType
from typing import Final
@ -25,6 +25,14 @@ def deployment_pricing_models(deployment: object) -> tuple[str, ...]:
return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model"))
def deployment_backend_model(deployment: object) -> str:
return next(iter(deployment_pricing_models(deployment)), "")
def deployment_backend_models(deployments: Iterable[object]) -> tuple[str, ...]:
return tuple(map(deployment_backend_model, deployments))
def _named_models(*candidates: object) -> Iterator[str]:
return (candidate for candidate in candidates if isinstance(candidate, str) and candidate)

36
litellm/litellm_core_utils/litellm_logging.py
View file

@ -56,6 +56,8 @@ from litellm.constants import (
from litellm.cost_calculator import (
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
declared_cost_map_pricing,
get_response_cost_from_hidden_params,
get_usage_object,
pricing_entry_for_cost_calc,
select_model_name_for_cost_calc,
@ -74,6 +76,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.langtrace import langtrace_trace_endpoint
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.batch_utils import (
POLLED_BATCH_COST_IS_FINAL_KWARG,
POLLED_BATCH_ID_KWARG,
@ -221,7 +224,10 @@ from litellm.types.utils import (
)
from litellm.types.videos.main import VideoObject
from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG
from litellm.utils import get_base_model_from_metadata, print_verbose
from litellm.utils import (
get_base_model_from_metadata,
print_verbose,
)
from ..integrations.argilla import ArgillaLogger
from ..integrations.arize.arize_phoenix import ArizePhoenixLogger
@ -2588,6 +2594,33 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def _response_price_is_known(self, result: object, cost: float, payload: StandardLoggingPayload) -> bool:
if (
cost != 0
or self._provider_reported_cost == 0
or get_response_cost_from_hidden_params(get_hidden_params_dict(result)) == 0
):
return True
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
return True
params: Final = _LITELLM_PARAMS.validate_python(self._model_call_details_view.get("litellm_params") or {})
if is_unbilled_non_inference_call(
payload["call_type"], StandardLoggingPayloadSetup.merge_litellm_metadata(params), result
):
return True
if self.custom_cost_per_token is not None or self.custom_cost_per_second is not None:
return True
passthrough: Final = _LITELLM_PARAMS.validate_python(
self._model_call_details_view.get("passthrough_logging_payload") or {}
)
if passthrough.get("cost_per_request") is not None:
return True
model_info: Final = payload["model_map_information"]["model_map_value"]
if model_info is None:
return False
prices: Final = declared_cost_map_pricing(model_info["key"], model_info["key"], payload["custom_llm_provider"])
return any(value is not None and ("cost" in key or "pricing" in key) for key, value in prices.items())
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
@ -2640,6 +2673,7 @@ class Logging(LiteLLMLoggingBaseClass):
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
price_known=self._response_price_is_known(prepared[2], raw_cost, payload),
)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]

1
litellm/litellm_core_utils/llm_cost_calc/finalization.py
View file

@ -17,6 +17,7 @@ class PreparedCost:
amount: float
payload: StandardLoggingPayload
usage: Usage
price_known: bool = True
@dataclass(frozen=True, slots=True)

68
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -399,6 +399,74 @@ def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, objec
return projection.content, projection.media
_TPM_EMBEDDING_FILE_MIME_PREFIXES: Final = ("application/pdf", "video/")
def _tpm_embedding_file(data_uri: str) -> tuple[Mapping[str, object], ...]:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
mime: Final = (parse_mime_type(data_uri[:256]) or "").strip().lower()
if not mime.startswith(_TPM_EMBEDDING_FILE_MIME_PREFIXES):
return ()
return ({"type": "file", "file": {"file_data": data_uri, "format": mime}},)
_BASE64_PAYLOAD: Final = re.compile(r"[A-Za-z0-9+/]*={0,2}")
def _is_base64_data_uri(value: str) -> bool:
header_end: Final = value.find(",", 0, 256)
if header_end < 0 or not value[:header_end].endswith(";base64"):
return False
payload_start: Final = header_end + 1
if (len(value) - payload_start) % 4 or _BASE64_PAYLOAD.fullmatch(value, payload_start) is None:
return False
last_quantum: Final = value[max(payload_start, len(value) - 4) :]
return base64.b64encode(base64.b64decode(last_quantum)).decode("ascii") == last_quantum
def _tpm_embedding_file_block(part: Mapping[str, object]) -> _TPMMediaProjection:
projection: Final = _tpm_file_media(part)
fields: Final = as_str_mapping(part.get("file"))
data: Final = fields.get("file_data") if fields is not None else None
if projection.media or fields is None or not isinstance(data, str) or not _is_base64_data_uri(data):
return projection
media: Final = _tpm_embedding_file(data)
return _TPMMediaProjection({**part, "file": _tpm_blank_fields(fields, ("file_data",))} if media else part, media)
def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is not None and part.get("type") == "file":
return _tpm_embedding_file_block(part)
if not isinstance(value, str) or not value.startswith("data:"):
return _tpm_media_leaf(value)
if not _is_base64_data_uri(value):
return _TPMMediaProjection(value)
media: Final = _tpm_file_media({"file_data": value}).media or _tpm_embedding_file(value)
return _TPMMediaProjection("" if media else value, media)
def _is_json_array(value: object) -> bool:
return isinstance(value, (list, tuple))
def _tpm_embedding_item(value: object) -> _TPMMediaProjection:
return _tpm_media_list(value, _tpm_embedding_leaf) if _is_json_array(value) else _tpm_embedding_leaf(value)
def split_embedding_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = (
_tpm_media_list(value, _tpm_embedding_item) if _is_json_array(value) else _tpm_embedding_leaf(value)
)
return projection.content, projection.media
def embedding_input_is_all_media(value: object) -> bool:
items: Final = cast(Sequence[object], value) if _is_json_array(value) else (value,) # cast-ok: JSON input array
return all(_tpm_embedding_leaf(item).media for item in items)
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
recursion; an Anthropic tool_result nests its image parts exactly one level down."""

121
litellm/litellm_core_utils/token_counter.py
View file

@ -610,6 +610,127 @@ def estimate_image_tokens(
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
_MEDIA_FILE_CHARS_PER_TOKEN: Final = 4
_EmbeddingInputT: Final = TypeVar("_EmbeddingInputT")
def _max_input_tokens(model: str) -> int | None:
try:
limit: Final = litellm.get_model_info(model).get("max_input_tokens")
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
return None
return limit if isinstance(limit, int) and limit > 0 else None
def _media_file_tokens(data: str, models: Sequence[str]) -> int:
text_estimate: Final = len(data) // _MEDIA_FILE_CHARS_PER_TOKEN
ceilings: Final = tuple(map(_max_input_tokens, models))
known_ceilings: Final = tuple(ceiling for ceiling in ceilings if ceiling is not None)
if not known_ceilings or len(known_ceilings) != len(ceilings):
return text_estimate
return min(text_estimate, max(known_ceilings))
def _block_str_field(block: Mapping[str, object], container: str, field: str) -> str | None:
inner: Final = as_str_mapping(block.get(container))
value: Final = inner.get(field) if inner is not None else None
return value if isinstance(value, str) else None
def estimate_media_block_tokens(block: Mapping[str, object], models: Sequence[str], fallback_model: str | None) -> int:
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
match block.get("type"):
case "input_audio":
return estimate_audio_tokens(
_block_str_field(block, "input_audio", "data"), models, fallback_model=fallback_model
)
case "file":
return _media_file_tokens(_block_str_field(block, "file", "file_data") or "", models)
case _:
detail: Final = _block_str_field(block, "image_url", "detail")
return estimate_image_tokens(
models,
_block_str_field(block, "image_url", "url") or "",
"auto" if detail is None else detail,
fallback_model=fallback_model,
)
_PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS: Final = frozenset({"vertex_ai", "gemini", "azure_ai"})
_DataUriMediaScope: TypeAlias = Literal["per_item", "whole_batch"]
def _is_pdf_block(block: Mapping[str, object]) -> bool:
return (_block_str_field(block, "file", "file_data") or "").startswith("data:application/pdf")
def _bedrock_data_uri_media_scope(
resolved_model: str, media: Sequence[Mapping[str, object]]
) -> _DataUriMediaScope | None:
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.utils import supports_embedding_image_input
base_model: Final = BedrockModelInfo.get_base_model(resolved_model.removeprefix("async_invoke/"))
if not supports_embedding_image_input(f"bedrock/{base_model}"):
return None
match BaseAWSLLM.get_bedrock_embedding_provider(base_model):
case "cohere":
return "whole_batch"
case "twelvelabs":
return None
case "nova" if any(map(_is_pdf_block, media)):
return None
case _:
return "per_item"
def _data_uri_media_scope(model: str, media: Sequence[Mapping[str, object]]) -> _DataUriMediaScope | None:
from litellm.utils import supports_embedding_image_input
try:
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model)
except litellm.exceptions.BadRequestError:
return None
if provider == "bedrock":
return _bedrock_data_uri_media_scope(resolved_model, media)
if not supports_embedding_image_input(model):
return None
if provider == "cohere":
return "whole_batch"
return "per_item" if provider in _PER_ITEM_DATA_URI_EMBEDDING_PROVIDERS else None
def split_embedding_input_tokens(
value: _EmbeddingInputT, media_models: Callable[[], Sequence[str]], fallback_model: str | None = None
) -> tuple[_EmbeddingInputT, int]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
embedding_input_is_all_media,
split_embedding_media_for_tpm,
)
text, media = split_embedding_media_for_tpm(value)
if not media:
return value, 0
models: Final = tuple(media_models())
scopes: Final = frozenset(_data_uri_media_scope(model, media) for model in models)
if not models or None in scopes or ("whole_batch" in scopes and not embedding_input_is_all_media(value)):
return value, 0
media_tokens: Final = sum(estimate_media_block_tokens(block, models, fallback_model) for block in media)
return cast(_EmbeddingInputT, text), media_tokens # cast-ok: the projection keeps the input's container shape
def embedding_aware_token_count(
messages: Sequence[Mapping[str, str]] | None,
text: str | list[str] | None, # mutable-ok: litellm.token_counter only takes text as list[str]
media_models: Callable[[], Sequence[str]],
) -> int:
embedding_text, media_tokens = split_embedding_input_tokens(text, media_models)
return litellm.token_counter(messages=messages, text=embedding_text) + media_tokens
TokenCounterFunction = Callable[[str], int]
"""
Type for a function that counts tokens in a string.

38
litellm/llms/cohere/embed/v1_transformation.py
View file

@ -9,6 +9,7 @@ import httpx
from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.custom_httpx.http_handler import header_value
from litellm.types.llms.bedrock import (
CohereEmbeddingRequest,
CohereEmbeddingRequestWithModel,
@ -23,6 +24,11 @@ class _SupportsEncode(Protocol):
def encode(self, text: str, /) -> Sized: ...
def _bedrock_billed_input_tokens(response: httpx.Response) -> int | None:
count: Final = header_value(response.headers, "x-amzn-bedrock-input-token-count")
return int(count) if count is not None and count.isdigit() else None
class CohereEmbeddingConfig:
"""
Reference: https://docs.cohere.com/v2/reference/embed
@ -76,32 +82,30 @@ class CohereEmbeddingConfig:
) -> CohereEmbeddingRequestWithModel:
return self._transform_request(model, input, inference_params)
def _calculate_usage(self, input: list[str], encoding: _SupportsEncode, meta: dict) -> Usage:
input_tokens = 0
def _calculate_usage(
self, input: list[str], encoding: _SupportsEncode, meta: dict, billed_input_tokens: int | None = None
) -> Usage:
text_tokens: Final[int | None] = meta.get("billed_units", {}).get("input_tokens")
image_tokens: Final[int | None] = meta.get("billed_units", {}).get("images")
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
if image_tokens is None and text_tokens is None:
for text in input:
input_tokens += len(encoding.encode(text))
else:
prompt_tokens_details = PromptTokensDetailsWrapper(
image_tokens=image_tokens,
text_tokens=text_tokens,
counted_tokens: Final = (
billed_input_tokens
if billed_input_tokens is not None
else sum(len(encoding.encode(text)) for text in input)
)
if image_tokens:
input_tokens += image_tokens
if text_tokens:
input_tokens += text_tokens
return Usage(prompt_tokens=counted_tokens, completion_tokens=0, total_tokens=counted_tokens)
input_tokens: Final = (image_tokens or 0) + (text_tokens or 0)
return Usage(
prompt_tokens=input_tokens,
completion_tokens=0,
total_tokens=input_tokens,
prompt_tokens_details=prompt_tokens_details,
prompt_tokens_details=PromptTokensDetailsWrapper(
image_tokens=image_tokens,
text_tokens=text_tokens,
),
)
def _transform_response(
@ -129,6 +133,7 @@ class CohereEmbeddingConfig:
model=model,
encoding=encoding,
input=input,
billed_input_tokens=_bedrock_billed_input_tokens(response) if "images" in data else None,
)
def transform_response(
@ -151,6 +156,7 @@ class CohereEmbeddingConfig:
model: str,
encoding: _SupportsEncode,
input: list,
billed_input_tokens: int | None = None,
) -> EmbeddingResponse:
"""
Parse a Cohere embed response body into an OpenAI-style EmbeddingResponse.
@ -195,7 +201,7 @@ class CohereEmbeddingConfig:
setattr(
model_response,
"usage",
self._calculate_usage(input, encoding, response_json.get("meta", {})),
self._calculate_usage(input, encoding, response_json.get("meta", {}), billed_input_tokens),
)
return model_response

1596
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

113
litellm/proxy/auth/model_checks.py
View file

@ -23,7 +23,12 @@ from litellm.router import Router
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
from litellm.utils import (
AZURE_MODEL_PLACEHOLDER,
ProviderConfigManager,
get_valid_models,
potential_model_names_in_lookup_order,
)
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models(
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
if name_pattern.count("*") > deployment_pattern.count("*"):
return _catalog_ids_a_wider_name_matches(
name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments(
name_pattern, deployment_pattern
):
@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models(
)
def _catalog_ids_a_wider_name_matches(
name_pattern: str,
deployment_pattern: str,
provider: str,
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
name_regex: Final = _wildcard_name_regex(name_pattern)
return tuple(
catalog_id
for catalog_id in _ids_routing_to_catalog_models(
deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_regex.fullmatch(catalog_id)
)
def _model_info_field(deployment: Mapping[str, object], field: str) -> object:
model_info: Final = deployment.get("model_info")
return model_info.get(field) if is_str_keyed_mapping(model_info) else None
@ -456,11 +482,12 @@ def _routes_to_model(
return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id)
def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None:
def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model):
return None
name_suffix: Final = wildcard_model.split("/", 1)[-1]
deployment_suffix: Final = litellm_params.model.partition("/")[2]
return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None
return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None
def get_known_models_from_wildcard(
@ -468,6 +495,43 @@ def get_known_models_from_wildcard(
litellm_params: LiteLLM_Params | None = None,
routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True,
) -> list[str]:
provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params)
if litellm_params is None or provider_id_prefix is None:
return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model))
return [
listed_id.removeprefix(provider_id_prefix)
for listed_id in _known_models_from_wildcard(
wildcard_model,
litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}),
lambda listed_id, model: routes_to_model(
listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix)
),
)
]
def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if (
litellm_params is None
or not litellm_params.custom_llm_provider
or "/" in wildcard_model
or wildcard_model == "*"
or "/" in litellm_params.model
or not _check_wildcard_routing(litellm_params.model)
):
return None
return f"{litellm_params.custom_llm_provider}/"
def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]:
return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER)
def _known_models_from_wildcard(
wildcard_model: str,
litellm_params: LiteLLM_Params | None,
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
wildcard_model_to_expand: Final = (
litellm_params.model
if wildcard_model == "*"
@ -476,15 +540,14 @@ def get_known_models_from_wildcard(
and "/" in litellm_params.model
else wildcard_model
)
try:
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
except ValueError: # safely fail
return []
deployment_pattern_naming_listed_ids: Final = (
wildcard_model_to_expand
if wildcard_model_to_expand != wildcard_model
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params)
)
wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/")
if not has_provider_prefix and deployment_pattern_naming_listed_ids is None:
return ()
# Use provider from litellm_params when available, otherwise from wildcard prefix
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
@ -501,16 +564,14 @@ def get_known_models_from_wildcard(
wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params)
if wildcard_models is None:
return []
return ()
if deployment_pattern_naming_listed_ids is not None:
return list(
_ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models,
routes_to_model=routes_to_model,
)
return _ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models),
routes_to_model=routes_to_model,
)
if wildcard_suffix != "*":
model_prefix: Final = wildcard_suffix.replace("*", "")
@ -526,7 +587,7 @@ def get_known_models_from_wildcard(
# add model prefix to wildcard models
wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
return list(
return tuple(
dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
)
@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name(
return tuple(name for name in names if not deployed_by_name(name, team_id))
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
if not _check_wildcard_routing(model_name):
return None
if "/" in model_name or model_name == "*":
return model_name
if _check_wildcard_routing(litellm_model):
return litellm_model
return model_name
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
if allowed_model_names is None:
return True
@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info(
raw_params = deployment.get("litellm_params")
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
litellm_model = str(litellm_params_dict.get("model") or "")
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
if not _check_wildcard_routing(model_name):
expanded.append(deployment)
continue
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
if not _can_expand_to_an_allowed_name(model_name, allowed_model_names):
continue
try:
@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info(
expanded.append(deployment)
continue
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
wildcard_model=model_name,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
)

7
litellm/proxy/common_request_processing.py
View file

@ -1887,6 +1887,13 @@ def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float |
def cost_headers(
response_cost: float | str | None, litellm_logging_obj: LiteLLMLoggingObj | None
) -> Mapping[str, str | None]:
if (
response_cost == 0
and litellm_logging_obj is not None
and litellm_logging_obj.prepared_cost is not None
and not litellm_logging_obj.prepared_cost.price_known
):
return {}
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=litellm_logging_obj, response_cost=response_cost
)

90
litellm/proxy/common_utils/fips.py Normal file
View file

@ -0,0 +1,90 @@
from collections.abc import Mapping
from typing import Final
from litellm.secret_managers.main import str_to_bool
FIPS_MODE_SETTING: Final = "fips_mode"
_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}"
class FipsModeError(Exception):
pass
def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None:
"""general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including
a key with no value."""
if FIPS_MODE_SETTING not in general_settings:
return False
value: Final = general_settings[FIPS_MODE_SETTING]
if isinstance(value, bool):
return value
return str_to_bool(value) if isinstance(value, str) else None
def openssl_enforces_fips() -> bool:
"""Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5."""
try:
import _hashlib
if _hashlib.get_fips_mode() != 1:
return False
except (ImportError, AttributeError, ValueError):
return False
try:
_hashlib.new("md5", b"", usedforsecurity=True)
except ValueError:
return True
return False
def cryptography_enforces_fips() -> bool:
"""The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256."""
try:
from cryptography.exceptions import InternalError, UnsupportedAlgorithm
from cryptography.hazmat.bindings._rust import openssl as rust_openssl
from cryptography.hazmat.primitives import hashes
except ImportError:
return False
try:
if not rust_openssl.is_fips_enabled():
return False
hashes.Hash(hashes.SHA256()).finalize()
except (AttributeError, InternalError, UnsupportedAlgorithm):
return False
return True
def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None:
"""Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the
cryptography library's OpenSSL does not enforce FIPS. None when it may start."""
setting: Final = _fips_mode_setting(general_settings)
if setting is None:
value: Final = general_settings.get(FIPS_MODE_SETTING)
if value is None:
return (
f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. "
"Set it to true or false, or remove it."
)
return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it."
if not setting:
return None
if not openssl_enforces_fips():
return (
f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS "
f"provider active and default_properties = fips=yes, or turn {_SETTING} off."
)
if not cryptography_enforces_fips():
return (
f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it "
"against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn "
f"{_SETTING} off."
)
return None
def enforce_fips_mode(general_settings: Mapping[str, object]) -> None:
"""Raise FipsModeError with the fips_mode_refusal reason, if there is one."""
refusal: Final = fips_mode_refusal(general_settings)
if refusal is not None:
raise FipsModeError(refusal)

80
litellm/proxy/common_utils/model_deprecation.py
View file

@ -32,6 +32,7 @@ class _ResolvedDeprecation:
litellm_model: str | None
litellm_provider: str | None
copied_from_another_row: bool = False
successor_model: str | None = None
def _parse_deprecation_date(raw_value: object) -> date | None:
@ -58,10 +59,11 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
entry: Final = litellm.model_cost.get(model_key)
if not isinstance(entry, Mapping):
return None
parsed: Final = _parse_deprecation_date(entry.get("deprecation_date"))
fields: Final[Mapping[str, object]] = entry
parsed: Final = _parse_deprecation_date(fields.get("deprecation_date"))
if parsed is None:
return None
provider: Final = entry.get("litellm_provider")
provider: Final = fields.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=parsed,
litellm_model=model_key,
@ -69,6 +71,7 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
copied_from_another_row=isinstance(entry, AliasedCostEntry)
and "deprecation_date" not in entry.registered_fields
and model_key not in catalog,
successor_model=_successor_model(fields.get("successor_model")),
)
@ -127,15 +130,6 @@ def _resolve_deployment_deprecation(
litellm_params: Final = _mapping_field(deployment, "litellm_params")
model_key: Final = _as_model_key(litellm_params.get("model"))
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
if override is not None:
override_provider: Final = model_info.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=override,
litellm_model=model_key,
litellm_provider=override_provider if isinstance(override_provider, str) else None,
)
provider: Final = _deployment_provider(litellm_params)
base_model: Final = _as_model_key(model_info.get("base_model"))
model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key
@ -150,7 +144,18 @@ def _resolve_deployment_deprecation(
if pick is not None
)
own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row)
return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
from_catalog: Final = min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
if override is None:
return from_catalog
override_provider: Final = model_info.get("litellm_provider")
return _ResolvedDeprecation(
deprecation_date=override,
litellm_model=model_key,
litellm_provider=override_provider if isinstance(override_provider, str) else None,
successor_model=from_catalog.successor_model if from_catalog is not None else None,
)
def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus:
@ -182,7 +187,7 @@ def _build_info(
today: date,
warn_within_days: int,
catalog: Mapping[str, object],
) -> ModelDeprecationInfo | None:
) -> _DeploymentInfo | None:
model_name: Final = deployment.get("model_name")
if not isinstance(model_name, str) or not model_name:
return None
@ -192,21 +197,40 @@ def _build_info(
return None
days_until: Final = (resolved.deprecation_date - today).days
return ModelDeprecationInfo(
model_name=model_name,
litellm_model=resolved.litellm_model,
deprecation_date=resolved.deprecation_date,
days_until_deprecation=days_until,
status=_classify(days_until, warn_within_days),
litellm_provider=resolved.litellm_provider,
successor_model=_checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router),
configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
return _DeploymentInfo(
info=ModelDeprecationInfo(
model_name=model_name,
litellm_model=resolved.litellm_model,
deprecation_date=resolved.deprecation_date,
days_until_deprecation=days_until,
status=_classify(days_until, warn_within_days),
litellm_provider=resolved.litellm_provider,
successor_model=configured or resolved.successor_model,
),
configured_successor=configured is not None,
)
def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names"""
first: Final = group[0]
successors: Final = tuple(model.successor_model for model in group if model.successor_model is not None)
@dataclass(frozen=True, slots=True)
class _DeploymentInfo:
info: ModelDeprecationInfo
configured_successor: bool
def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names,
a successor configured in model_info ahead of one the cost map supplies"""
first: Final = group[0].info
tiers: Final = tuple(
tuple(
deployment.info.successor_model
for deployment in group
if deployment.configured_successor is configured and deployment.info.successor_model is not None
)
for configured in (True, False)
)
successors: Final = next((tier for tier in tiers if tier), ())
if not successors:
return first
if len(frozenset(successors)) > 1:
@ -220,13 +244,13 @@ def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
def _dedupe(
models: Sequence[ModelDeprecationInfo],
models: Sequence[_DeploymentInfo],
) -> tuple[ModelDeprecationInfo, ...]:
"""Report a model group carrying the same date on several deployments once"""
ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date))
ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date))
return tuple(
_merge_group(tuple(group))
for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date))
for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date))
)

13
litellm/proxy/db/exception_handler.py
View file

@ -34,6 +34,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014"
_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout"
def exception_chain(e: BaseException) -> Iterator[BaseException]:
@ -302,6 +304,17 @@ class PrismaDBExceptionHandler:
return True
return _LOCK_TIMEOUT_PHRASE in str(e).lower()
@staticmethod
def is_statement_timeout_error(e: Exception) -> bool:
"""True iff Postgres cancelled a statement for exceeding ``statement_timeout`` (SQLSTATE 57014)."""
import prisma
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if PrismaDBExceptionHandler.postgres_sqlstate(e) == _STATEMENT_TIMEOUT_SQLSTATE:
return True
return _STATEMENT_TIMEOUT_PHRASE in str(e).lower()
@staticmethod
def postgres_sqlstate(e: Exception) -> str | None:
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""

71
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -55,7 +55,6 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
response_has_hidden_params,
)
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
@ -64,7 +63,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.litellm_core_utils.token_counter import (
estimate_media_block_tokens,
offload_token_count,
split_embedding_input_tokens,
)
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -156,6 +159,12 @@ def _resolve_media_models(router: "Router | None", model: str, team_id: str | No
return tuple(dict.fromkeys(router.registry().names(model, kind="litellm_base_models", team_id=team_id))) or (model,)
def _media_models(
resolver: Callable[[str, str | None], tuple[str, ...]] | None, model: str, team_id: str | None
) -> tuple[str, ...]:
return resolver(model, team_id) if resolver is not None else (model,)
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
from litellm.proxy.proxy_server import llm_router
@ -1378,20 +1387,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
media_source: Final = messages if messages is not None else original_input
is_embedding_call: Final = call_type in EMBEDDING_API_CALL_TYPES
projected, media = (
split_media_for_tpm(media_source)
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
else (original_input, ())
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
fallback_model: Final = request_model if isinstance(request_model, str) else None
resolved_models: Final = (
media_model_resolver(fallback_model or "", team_id)
if media and media_model_resolver is not None
else (fallback_model or "",)
_media_models(media_model_resolver, fallback_model or "", team_id) if media else (fallback_model or "",)
)
media_tokens: Final = sum(
self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
embedding_text, embedding_media_tokens = (
split_embedding_input_tokens(
original_input,
lambda: _media_models(media_model_resolver, fallback_model or "", team_id),
fallback_model,
)
if include_media and is_embedding_call
else (original_input, 0)
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else embedding_text
media_tokens: Final = embedding_media_tokens + sum(
estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
)
total_chars: Final = (
@ -1436,24 +1453,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
@staticmethod
def _estimate_media_block_tokens(
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
) -> int:
if block.get("type") == "input_audio":
audio: Final = _callback_kwargs_mapping(block.get("input_audio"))
data: Final = audio.get("data")
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
image: Final = _callback_kwargs_mapping(block.get("image_url"))
url: Final = image.get("url")
detail: Final = image.get("detail")
return estimate_image_tokens(
models,
url if isinstance(url, str) else "",
detail if isinstance(detail, str) else "auto",
fallback_model=fallback_model,
)
def _is_redis_cluster(self) -> bool:
"""
Check if the dual cache is using Redis cluster.
@ -4098,12 +4097,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
None if is_responses_request else self._translate_google_genai_native_request(data, call_type)
)
is_embedding_request: Final = self._is_embedding_request(data, call_type)
embedding_text: Final = data.get("input") if is_embedding_request else None
request: Final = as_str_mapping(data)
raw_embedding_input: Final = request.get("input") if is_embedding_request and request is not None else None
pretokenized_input_tokens: Final = (
self._count_pretokenized_embedding_input(embedding_text) if is_embedding_request else None
self._count_pretokenized_embedding_input(raw_embedding_input) if is_embedding_request else None
)
if pretokenized_input_tokens is not None:
return pretokenized_input_tokens
embedding_text, embedding_media_tokens = (
split_embedding_input_tokens(
raw_embedding_input, lambda: self._media_model_resolver(model or "", team_id), model
)
if is_embedding_request
else (None, 0)
)
prompt: Final = data.get("prompt")
fallback_text: Final = prompt if prompt is not None else data.get("input")
@ -4153,12 +4160,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",))
),
)
return estimate + audio_token_estimate
return estimate + audio_token_estimate + embedding_media_tokens
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data=data, call_type=call_type, include_media=False
data=data,
call_type=call_type,
include_media=is_embedding_request,
team_id=team_id,
media_model_resolver=self._media_model_resolver,
)
return estimated_input_tokens + audio_token_estimate

29
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -47,7 +47,7 @@ from litellm.proxy.auth.password_policy import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.openai_error_payload import openai_shaped_proxy_exception, proxy_exception_for
from litellm.proxy.common_utils.user_api_key_cache import (
AUTH_OBJECTS_TARGET,
object_permission_cache_key,
@ -1048,6 +1048,7 @@ async def _check_user_info_v2_access(
1. Proxy admins / proxy admin viewers can access any user
2. User can access their own info
3. Team admins can access info of users in their teams
4. Org admins can access info of users in their organizations
Raises on unexpected DB errors so they surface as 500s, not silent 404s.
"""
@ -1090,9 +1091,33 @@ async def _check_user_info_v2_access(
if team.team_id in (target_user.teams or []):
return target_user
if user_api_key_dict.user_id is not None and await _target_in_caller_admin_org(
prisma_client=prisma_client,
caller_user_id=user_api_key_dict.user_id,
target_user_id=target_user_id,
):
return await _fetch_target_user()
return None
async def _target_in_caller_admin_org(
prisma_client: "PrismaClient",
caller_user_id: str,
target_user_id: str,
) -> bool:
admin_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
where={"user_id": caller_user_id, "user_role": LitellmUserRoles.ORG_ADMIN.value}
)
if not admin_memberships:
return False
admin_org_ids: Final = {membership.organization_id for membership in admin_memberships}
target_memberships: Final = await prisma_client.db.litellm_organizationmembership.find_many(
where={"user_id": target_user_id}
)
return any(membership.organization_id in admin_org_ids for membership in target_memberships)
@router.get(
"/v2/user/info",
tags=["Internal User management"],
@ -1197,6 +1222,8 @@ async def user_info_v2(
),
)
except Exception as e:
if isinstance(e, HTTPException) and e.status_code < 500:
raise openai_shaped_proxy_exception(e, str(e.detail), e.status_code)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)

39
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -140,6 +140,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
team_membership_reservation_cache_key,
)
from litellm.proxy.db.db_span import db_span
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.hooks.model_max_budget_limiter import (
RoutedModels,
@ -7473,6 +7474,19 @@ class _TeamUserSpendDbRow(TypedDict):
failed_requests: ReadOnly[int]
_TEAM_USER_SPEND_STATEMENT_TIMEOUT_SECONDS: Final = 30
async def _team_user_spend_rows(
prisma_client: PrismaClient, sql: str, params: tuple[str, ...]
) -> Sequence[_TeamUserSpendDbRow]:
async with prisma_client.read_db.tx(
timeout=timedelta(seconds=_TEAM_USER_SPEND_STATEMENT_TIMEOUT_SECONDS + 5)
) as tx:
await tx.execute_raw(f"SET LOCAL statement_timeout = {_TEAM_USER_SPEND_STATEMENT_TIMEOUT_SECONDS * 1000}")
return await tx.query_raw(sql, *params)
@router.get(
"/team/spend/by_user",
response_model=TeamUserSpendResponse,
@ -7522,14 +7536,23 @@ async def get_team_spend_by_user(
own_user_only: Final = scope.api_key_filter is not None
user_param: Final = (user_api_key_dict.user_id or "",) if own_user_only else ()
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
rows: Final[Sequence[_TeamUserSpendDbRow]] = await prisma_client.db.query_raw(
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
start_date,
end_date,
*scoped_team_ids,
*user_param,
)
try:
async with db_span("team_user_spend", "LiteLLM_SpendLogs"):
rows: Final = await _team_user_spend_rows(
prisma_client,
_team_user_spend_sql(team_count=len(scoped_team_ids), restrict_to_user=own_user_only),
(start_date, end_date, *scoped_team_ids, *user_param),
)
except Exception as e:
if PrismaDBExceptionHandler.is_statement_timeout_error(e):
raise _daily_activity_error(
status_code=503,
message=(
f"Spend per user took longer than {_TEAM_USER_SPEND_STATEMENT_TIMEOUT_SECONDS}s. "
"Pick a shorter date range or fewer teams"
),
) from e
raise
results: Final = tuple(
TeamUserSpendRow(
team_id=row["team_id"],

8
litellm/proxy/proxy_server.py
View file

@ -530,6 +530,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
encrypt_value_helper,
)
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
from litellm.proxy.common_utils.fips import enforce_fips_mode
from litellm.proxy.common_utils.healthy_model_filter import (
get_hidden_unhealthy_model_names,
is_healthy_only_listing_default,
@ -1673,6 +1674,8 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener
if decoded_worker_config is not None:
await initialize_from_worker_config(decoded_worker_config)
enforce_fips_mode(general_settings_view())
async with AsyncExitStack() as boot_database:
enforce_master_key_boot_verdict(
await with_stored_secrets_counted(
@ -18611,8 +18614,9 @@ async def model_deprecations(
Reads `deprecation_date` metadata from `model_prices_and_context_window.json`
(and any per-deployment `model_info.deprecation_date` overrides) for the
models configured on this proxy. An entry also names its `successor_model`
when the deployment sets `model_info.successor_model`.
models configured on this proxy. An entry also names its `successor_model`:
`model_info.successor_model` when the deployment sets one, else the
replacement the cost map records from the provider's deprecation notice.
Parameters:
warn_within_days: Window (in days) used to bucket "imminent" models,

221
litellm/proxy/spend_tracking/session_page.py Normal file
View file

@ -0,0 +1,221 @@
"""Which sessions a logs page shows and how many match, read from the newest rows instead of every row. Reads grow
until one settles both, so the result is what grouping every row would give."""
import math
from collections.abc import Awaitable, Callable, Sequence
from dataclasses import dataclass
from typing import Final, Protocol, TypeAlias
from typing_extensions import ReadOnly, TypedDict
MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
SESSION_REPRESENTATIVE_ORDER_SQL: Final = (
f"(call_type = {AGENT_CALL_TYPE_SQL}) DESC, "
f'CASE WHEN call_type = {AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, '
f'call_type IN {MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id'
)
class _ReadRow(TypedDict):
session_key: ReadOnly[str | None]
api_key: ReadOnly[str | None]
last_activity: ReadOnly[str | None]
settled: ReadOnly[bool | None]
rows_read: ReadOnly[int]
session_count: ReadOnly[int]
class _SupportsQueryRaw(Protocol):
async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ...
@dataclass(frozen=True, slots=True)
class Session:
key: str
api_key: str
last_activity: str
@dataclass(frozen=True, slots=True)
class SessionPage:
sessions: tuple[Session, ...]
has_more: bool
next_cursor: str | None
total: int
total_is_capped: bool
@dataclass(frozen=True, slots=True)
class _Request:
cursor: tuple[str, str, str] | None
offset: int
page_limit: int
sort_desc: bool
status: str | None
cap: int
def first_read_size(self) -> int | None:
"""Every row when oldest first, or when filtered by outcome, which any of a session's rows can decide. Otherwise
the fewest that could settle a page: a session past the cap and an older row."""
return self.cap + 2 if self.sort_desc and self.status is None else None
@dataclass(frozen=True, slots=True)
class _Read:
candidates: tuple[Session, ...]
last_settled: bool
rows_read: int
session_count: int
saw_every_row: bool
def settles(self, request: _Request) -> bool:
page_settled: Final = (
request.page_limit <= 0
or self.saw_every_row
or (len(self.candidates) > request.page_limit and self.last_settled)
)
return page_settled and (self.saw_every_row or self.session_count > request.cap)
def next_size(self, request: _Request) -> int:
rows_per_session: Final = self.rows_read / self.session_count
return max(2 * self.rows_read, math.ceil(1.25 * (request.cap + 1) * rows_per_session))
_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]]
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
if session_cursor is None or session_cursor.count("|") < 2:
return None
last_activity, _, rest = session_cursor.partition("|")
api_key, _, session_key = rest.partition("|")
if not last_activity or not session_key:
return None
return (last_activity, session_key, api_key)
async def read_session_page(
db: _SupportsQueryRaw,
*,
where_clause: str,
params: Sequence[object],
status: str | None,
page: int,
page_size: int,
sort_desc: bool,
session_cursor: str | None,
cap: int,
) -> SessionPage:
"""One page of sessions after ``session_cursor``, or after ``(page - 1) * page_size`` sessions, keeping only those
whose outcome is ``status`` when given. ``where_clause`` reads ``params`` as ``$1`` onward."""
cursor: Final = _parse_session_cursor(session_cursor)
offset: Final = (page - 1) * page_size if cursor is None else 0
request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, status, cap)
return await _session_page(_sql_reader(db, where_clause, params, request), request)
async def _session_page(read: _Reader, request: _Request) -> SessionPage:
last = await read(request.first_read_size()) # rebind-ok: each read replaces the last until one settles the page
while not last.settles(request):
last = await read(last.next_size(request))
has_more: Final = len(last.candidates) > request.page_limit
sessions: Final = last.candidates[: request.page_limit]
final: Final = sessions[-1] if has_more and sessions else None
return SessionPage(
sessions=sessions,
has_more=has_more,
next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None,
total=min(last.session_count, request.cap),
total_is_capped=last.session_count > request.cap,
)
def _sessions_sql(request: _Request, status_index: int) -> str:
"""Each session's last activity, from its rows in ``scanned``, keeping only sessions whose outcome is the status."""
if request.status is None:
return (
'SELECT session_key, api_key, MAX("startTime") AS last_activity FROM scanned GROUP BY session_key, api_key'
)
return f"""
SELECT session_key, api_key, last_activity
FROM (
SELECT DISTINCT ON (session_key, api_key)
session_key,
api_key,
status,
MAX("startTime") OVER (PARTITION BY session_key, api_key) AS last_activity
FROM scanned
ORDER BY session_key, api_key, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS outcomes
WHERE COALESCE(status, 'success') = ${status_index}
"""
def _sql_reader(db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], request: _Request) -> _Reader:
status_index: Final = len(params) + 1
status_args: Final = () if request.status is None else (request.status,)
cursor_index: Final = status_index + len(status_args)
after_cursor: Final = (
f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} "
f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})"
if request.cursor
else ""
)
outcome_columns: Final = ', status, call_type, "endTime", request_id' if request.status is not None else ""
direction: Final = "DESC" if request.sort_desc else "ASC"
page_rows: Final = request.page_limit + 1 if request.page_limit > 0 else 0
skip: Final = f"OFFSET {request.offset}" if page_rows and request.offset else ""
async def read(size: int | None) -> _Read:
# LEFT JOIN keeps the summary row when no session is on the page.
rows: Final = await db.query_raw(
f"""
WITH scanned AS (
SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"{outcome_columns}
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
{f'ORDER BY "startTime" DESC LIMIT {size}' if size is not None else ""}
),
sessions AS ({_sessions_sql(request, status_index)}),
summary AS (
SELECT COUNT(*) AS rows_read,
MIN("startTime") AS oldest,
(SELECT COUNT(*) FROM sessions) AS session_count
FROM scanned
)
SELECT page.session_key,
page.api_key,
page.last_activity::text AS last_activity,
page.last_activity > summary.oldest AS settled,
summary.rows_read,
summary.session_count
FROM summary
LEFT JOIN LATERAL (
SELECT session_key, api_key, last_activity
FROM sessions
{after_cursor}
ORDER BY last_activity {direction}, session_key {direction}, api_key {direction}
LIMIT {page_rows} {skip}
) AS page ON TRUE
ORDER BY page.last_activity {direction}, page.session_key {direction}, page.api_key {direction}
""",
*params,
*status_args,
*(request.cursor or ()),
)
rows_read: Final = int(rows[0]["rows_read"])
return _Read(
candidates=tuple(
Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"]))
for row in rows
if row["session_key"] is not None
),
last_settled=bool(rows[-1]["settled"]),
rows_read=rows_read,
session_count=int(rows[0]["session_count"]),
saw_every_row=size is None or rows_read < size,
)
return read

210
litellm/proxy/spend_tracking/spend_management_endpoints.py
View file

@ -52,6 +52,13 @@ from litellm.proxy.auth.authorization_dependencies import (
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.spend_tracking.session_page import (
AGENT_CALL_TYPE_SQL,
MCP_CALL_TYPES_SQL,
SESSION_KEY_EXPR,
SESSION_REPRESENTATIVE_ORDER_SQL,
read_session_page,
)
from litellm.proxy.spend_tracking.spend_capture_rate import (
ProviderBillingCredentialMissing,
ProviderBillingRequestFailed,
@ -87,23 +94,15 @@ router: Final = APIRouter()
SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000
_DECODED_JSON: Final = TypeAdapter(object)
_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key"
_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
_AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
_SESSION_REPRESENTATIVE_ORDER_SQL: Final = (
f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, "
f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, '
f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id'
)
_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key"
_BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')"
_SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType(
{
"mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}",
"agent": f"call_type = {_AGENT_CALL_TYPE_SQL}",
"mcp": f"call_type IN {MCP_CALL_TYPES_SQL}",
"agent": f"call_type = {AGENT_CALL_TYPE_SQL}",
"batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}",
"llm": (
f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} "
f"(call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} "
f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})"
),
}
@ -2948,23 +2947,6 @@ async def ui_view_spend_logs(
sql_params.append(json.dumps(used_client_oauth_token))
p += 1
if status_filter is not None and group_by_session is True and not is_search_lookup:
session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE"
sql_conditions.append(
f"""({_SESSION_GROUP_KEY_SQL}) IN (
SELECT session_key, api_key FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{_SESSION_KEY_EXPR} AS session_key, api_key, status
FROM "LiteLLM_SpendLogs"
WHERE {session_filter_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_outcomes
WHERE COALESCE(status, 'success') = ${p}
)"""
)
sql_params.append(status_filter)
p += 1
if (
group_by_session is True
and not is_v2
@ -2977,12 +2959,30 @@ async def ui_view_spend_logs(
sql_conditions=sql_conditions,
sql_params=sql_params,
next_param_index=p,
status_filter=status_filter,
page=page,
page_size=page_size,
sort_desc=order_direction != "asc",
session_cursor=session_cursor,
)
if status_filter is not None and group_by_session is True and not is_search_lookup:
session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE"
sql_conditions.append(
f"""({_SESSION_GROUP_KEY_SQL}) IN (
SELECT session_key, api_key FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{SESSION_KEY_EXPR} AS session_key, api_key, status
FROM "LiteLLM_SpendLogs"
WHERE {session_filter_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_outcomes
WHERE COALESCE(status, 'success') = ${p}
)"""
)
sql_params.append(status_filter)
p += 1
# Build the ORDER BY expression. ttft_ms is computed from
# completionStartTime - startTime; non-streaming rows (where
# completionStartTime is null or equals endTime) yield NULL, so we
@ -3031,7 +3031,7 @@ async def ui_view_spend_logs(
{_SPEND_LOG_LIST_COLUMNS}
FROM "LiteLLM_SpendLogs"
WHERE {joined_conditions}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_representatives
ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id
LIMIT ${p} OFFSET ${p + 1}
@ -3080,22 +3080,6 @@ async def ui_view_spend_logs(
raise handle_exception_on_proxy(e)
class _SessionPageRow(TypedDict):
session_key: ReadOnly[str]
api_key: ReadOnly[str]
last_activity: ReadOnly[str]
def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None:
if session_cursor is None or session_cursor.count("|") < 2:
return None
last_activity, _, rest = session_cursor.partition("|")
api_key, _, session_key = rest.partition("|")
if not last_activity or not session_key:
return None
return (last_activity, session_key, api_key)
async def _fetch_session_representatives(
prisma_client: "PrismaClient",
where_clause: str,
@ -3104,16 +3088,18 @@ async def _fetch_session_representatives(
session_keys: Sequence[tuple[str, str]],
) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row
"""Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order."""
# Lets Postgres use the session_id and request_id indexes.
rep_query: Final = f"""
SELECT * FROM (
SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL})
{_SPEND_LOG_LIST_COLUMNS}
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
AND (session_id = ANY(${next_param_index}::text[]) OR request_id = ANY(${next_param_index}::text[]))
AND ({_SESSION_GROUP_KEY_SQL}) IN (
SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[])
)
ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL}
ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL}
) AS session_representatives
"""
rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place
@ -3129,117 +3115,32 @@ async def _fetch_session_representatives(
return [rep_by_key[key] for key in session_keys if key in rep_by_key]
async def _count_grouped_sessions(
prisma_client: "PrismaClient",
where_clause: str,
sql_params: Sequence[object],
next_param_index: int,
) -> tuple[int, bool]:
"""Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap."""
count_query: Final = f"""
SELECT COUNT(*) AS total_count
FROM (
SELECT 1
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
GROUP BY {_SESSION_GROUP_KEY_SQL}
LIMIT ${next_param_index}
) AS bounded_sessions
"""
count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw(
prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1
)
raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0
return (
(SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False)
)
async def _ui_session_grouped_spend_logs(
prisma_client: "PrismaClient",
sql_conditions: Sequence[str],
sql_params: Sequence[object],
next_param_index: int,
status_filter: str | None,
page: int,
page_size: int,
sort_desc: bool,
session_cursor: str | None,
) -> Mapping[str, object]:
"""
One row per session, keyset-paginated by session last activity.
Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra
table): rows sharing a ``session_id`` and ``api_key`` form a session, rows
without a session id are singletons keyed by ``request_id``. A page is the
next ``page_size`` sessions ordered by ``(MAX(startTime), session_key,
api_key)``, resumed from the ``session_cursor`` keyset
``'<last_activity>|<api_key>|<session_key>'`` instead of an OFFSET, so
page depth does not degrade the query plan. A request for ``page > 1``
without a cursor (the UI jumping straight to the last page, or back to a
page it never walked through) falls back to ``OFFSET (page - 1) *
page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP``
window the capped ``total`` promises, so a page never runs past that total
and one starting at or past it returns no rows without a query. Each session is represented
by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response``
exactly like the flat listing, and the response carries
``next_session_cursor`` / ``has_more`` while ``total`` counts sessions
(capped like the flat total). A page that runs out of sessions while still
holding some is itself the end of the list, so its ``total`` is
``offset + len(page)`` and the grouped count query is skipped; a page that
starts past the end says nothing about the total, so that one is counted.
"""
"""One row per session, keyset-paginated by last activity. ``read_session_page`` picks and counts the sessions,
and each is shown by its final agent outcome or newest non-MCP row."""
where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE"
cmp_op: Final = "<" if sort_desc else ">"
direction: Final = "DESC" if sort_desc else "ASC"
cursor: Final = _parse_session_cursor(session_cursor)
having_clause: Final = (
f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} '
f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})"
if cursor
else ""
session_page: Final = await read_session_page(
prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__
where_clause=where_clause,
params=sql_params,
status=status_filter,
page=page,
page_size=page_size,
sort_desc=sort_desc,
session_cursor=session_cursor,
cap=SPEND_LOGS_PAGINATION_COUNT_CAP,
)
cursor_params: Final[tuple[object, ...]] = cursor if cursor else ()
limit_index: Final = next_param_index + len(cursor_params)
offset: Final = (page - 1) * page_size if cursor is None else 0
page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset)
offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else ()
offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else ""
page_query: Final = f"""
SELECT {_SESSION_KEY_EXPR} AS session_key,
api_key,
MAX("startTime")::text AS last_activity
FROM "LiteLLM_SpendLogs"
WHERE {where_clause}
GROUP BY {_SESSION_GROUP_KEY_SQL}
{having_clause}
ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction}
LIMIT ${limit_index} {offset_clause}
"""
page_rows: Final[Sequence[_SessionPageRow]] = (
()
if page_limit <= 0
else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params)
)
has_more: Final = len(page_rows) > page_limit
visible_rows: Final = page_rows[:page_limit]
next_cursor: Final = (
f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}"
if has_more and visible_rows
else None
)
page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0
page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list
total_records, total_is_capped = (
(offset + len(page_rows), False)
if page_ends_the_list
else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index)
)
session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows)
session_keys: Final = tuple((session.key, session.api_key) for session in session_page.sessions)
data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row
await _fetch_session_representatives(
prisma_client=prisma_client,
@ -3253,18 +3154,17 @@ async def _ui_session_grouped_spend_logs(
)
_hydrate_spend_log_metadata(data)
total_pages: Final = (total_records + page_size - 1) // page_size
response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response(
prisma_client,
data,
total_records,
session_page.total,
page,
page_size,
total_pages,
(session_page.total + page_size - 1) // page_size,
enrich_session_counts=True,
total_is_capped=total_is_capped,
total_is_capped=session_page.total_is_capped,
)
return {**response, "next_session_cursor": next_cursor, "has_more": has_more}
return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more}
class RequestResponsePayload(NamedTuple):
@ -4720,16 +4620,16 @@ async def _build_ui_spend_logs_response(
)
), 0)::bigint AS session_total_duration_ms,
COUNT(*) FILTER (
WHERE call_type IN {_MCP_CALL_TYPES_SQL}
WHERE call_type IN {MCP_CALL_TYPES_SQL}
)::int AS mcp_tool_call_count,
COALESCE(SUM(spend) FILTER (
WHERE call_type IN {_MCP_CALL_TYPES_SQL}
WHERE call_type IN {MCP_CALL_TYPES_SQL}
), 0)::double precision AS mcp_tool_call_spend,
COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count,
COUNT(*) FILTER (
WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL}
WHERE call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL}
)::int AS session_llm_count,
COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
COUNT(*) FILTER (WHERE call_type = {AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens,
COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens,
COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens

11
litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
View file

@ -6,6 +6,7 @@ from collections import Counter
from collections.abc import Mapping, MutableMapping, Sequence
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
Final,
Literal,
@ -16,7 +17,6 @@ from typing import (
from urllib.parse import urlparse
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
from prisma.errors import PrismaError
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError, create_model, field_validator
from pydantic.fields import FieldInfo, PydanticUndefined
from typing_extensions import NotRequired, ReadOnly, TypedDict
@ -63,6 +63,9 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
SSOConfig,
)
if TYPE_CHECKING:
from prisma.errors import PrismaError
router: Final = APIRouter()
_DECODED_JSON: Final = TypeAdapter(object)
@ -1260,7 +1263,9 @@ def _parse_environment_variables_param(param_value: str | Mapping[str, object] |
async def _load_environment_variables_param(
repo: ConfigRepository,
) -> Mapping[str, object] | None | PrismaError | ValueError:
) -> "Mapping[str, object] | None | PrismaError | ValueError":
from prisma.errors import PrismaError
try:
entry: Final = await _config_param_db(repo).find_unique(where={"param_name": "environment_variables"})
param_value: Final = None if entry is None else _parse_environment_variables_param(entry.param_value)
@ -1314,6 +1319,8 @@ async def update_sso_settings(
detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."},
)
from prisma.errors import PrismaError
# Read the existing SSO row first so the audit log captures a real
# before/after diff. Stored values are encrypted; decrypt them so the
# before-snapshot has the same shape as after_value, and rely on

11
litellm/router_python_core.py
View file

@ -135,7 +135,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models, deployment_pricing_models
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
@ -14866,6 +14866,7 @@ class PythonCore(DeprecatedRouterMembers):
input: str | list[object] | None,
request_kwargs: Mapping[str, object] | None = None,
tokenizer_model: str = "",
deployments: Sequence[Mapping[str, object]] = (),
) -> int:
"""
Count input tokens for context-window pre-call checks.
@ -14900,11 +14901,15 @@ class PythonCore(DeprecatedRouterMembers):
if input is not None:
from openai.types.responses.response_create_params import ResponseInputParam
from litellm.litellm_core_utils.token_counter import split_embedding_input_tokens
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
typed_input: Final = cast(str | ResponseInputParam, input) # cast-ok: str | list matches transform input
text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments))
typed_input: Final = cast( # cast-ok: str | list matches transform input
str | ResponseInputParam, text_input
)
input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=typed_input,
responses_api_request={"instructions": instructions} if instructions is not None else {},
@ -14972,6 +14977,7 @@ class PythonCore(DeprecatedRouterMembers):
self.served_openai_models(model) or served_openai_models(healthy_deployments)
)
or "",
deployments=healthy_deployments,
)
except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request
verbose_router_logger.error(
@ -15002,6 +15008,7 @@ class PythonCore(DeprecatedRouterMembers):
self.served_openai_models(model) or served_openai_models(deployments)
)
or "",
deployments=deployments,
)
except Exception as e:
verbose_router_logger.error(

14
litellm/router_strategy/lowest_cost.py
View file

@ -2,16 +2,22 @@
# picks based on response time (for streaming, this is time to first token)
from collections.abc import Mapping
from datetime import datetime
from functools import partial
from typing import Final
import litellm
from litellm import ModelResponse, token_counter, verbose_logger
from litellm import ModelResponse, verbose_logger
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
from litellm.litellm_core_utils.deployment_pricing import (
deployment_backend_models,
deployment_pricing_models,
str_keyed,
)
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]:
@ -235,7 +241,9 @@ class LowestCostLoggingHandler(CustomLogger):
}
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0

9
litellm/router_strategy/lowest_latency.py
View file

@ -3,18 +3,21 @@
import random
from collections.abc import Sequence
from datetime import datetime, timedelta
from functools import partial
from math import ceil, isfinite
from typing import TYPE_CHECKING, Any, Final
from pydantic import Field
import litellm
from litellm import ModelResponse, token_counter, verbose_logger
from litellm import ModelResponse, verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import LiteLLMPydanticObjectBase
@ -439,7 +442,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
}
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0

8
litellm/router_strategy/lowest_tpm_rpm.py
View file

@ -2,14 +2,16 @@
# identifies lowest tpm deployment
import traceback
from datetime import datetime
from functools import partial
from typing import Final
from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.types.utils import LiteLLMPydanticObjectBase
from litellm.utils import print_verbose
@ -174,7 +176,9 @@ class LowestTPMLoggingHandler(CustomLogger):
verbose_router_logger.debug("tpm_key=%s, tpm_dict: %s, rpm_dict: %s", tpm_key, tpm_dict, rpm_dict)
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0
verbose_router_logger.debug("input_tokens=%s", input_tokens)

8
litellm/router_strategy/lowest_tpm_rpm_v2.py
View file

@ -5,18 +5,20 @@ from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, Final
import httpx
import litellm
from litellm import token_counter
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger, verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models
from litellm.litellm_core_utils.token_counter import embedding_aware_token_count
from litellm.types.router import RouterErrors
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
from litellm.utils import get_utc_datetime, print_verbose
@ -422,7 +424,9 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
rpm_dict[rpm_keys[idx].split(":")[0]] = rpm_values[idx]
try:
input_tokens = token_counter(messages=messages, text=input)
input_tokens = embedding_aware_token_count(
messages, input, partial(deployment_backend_models, healthy_deployments)
)
except Exception:
input_tokens = 0
verbose_router_logger.debug("input_tokens=%s", input_tokens)

5
litellm/types/proxy/model_deprecation.py
View file

@ -34,7 +34,10 @@ class ModelDeprecationInfo(LiteLLMBaseModel):
litellm_provider: str | None = Field(default=None, description="The provider this model belongs to.")
successor_model: str | None = Field(
default=None,
description=("The model to migrate to, as set in model_info.successor_model on the deployment."),
description=(
"The model to migrate to: model_info.successor_model on the deployment when set, "
"else the successor_model the cost map names for the deprecated model."
),
)

5
litellm/utils.py
View file

@ -9069,6 +9069,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]:
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM"
def get_valid_models(
check_provider_endpoint: bool | None = None,
custom_llm_provider: str | None = None,
@ -9124,7 +9127,7 @@ def get_valid_models(
continue
if provider == "azure":
valid_models.append("Azure-LLM")
valid_models.append(AZURE_MODEL_PLACEHOLDER)
elif provider_config is not None and check_provider_endpoint and provider is not None:
valid_models.extend(
_get_valid_models_from_provider_api(

1596
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

1
scripts/check_mcp_sdk_install.py
View file

@ -13,6 +13,7 @@ IMPORTED_MODULES: Final[tuple[str, ...]] = (
"litellm.proxy.mcp_server.server",
"litellm.proxy.mcp_server.mcp_server_manager",
"litellm.proxy.mcp_server.rest_endpoints",
"litellm.proxy.proxy_server",
)

20
tests/code_coverage_tests/test_provider_cache.py
View file

@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi
assert len(provider.hits) == (2 if truncated else 1)
def test_store_outage_preserves_provider_success(provider: Provider) -> None:
@pytest.mark.parametrize("cluster_mode", ("0", "1"))
def test_store_outage_preserves_provider_success(
provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str,
) -> None:
monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode)
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
port: Final = probe.getsockname()[1]
@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None
assert isinstance(hit, CacheHit) and hit.payload == b"new"
def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None:
other: Final = replace(store, namespace=store.namespace + "-other")
first_lease: Final = store.lookup("shared-key")
assert isinstance(first_lease, CaptureLease)
assert store.publish("shared-key", first_lease, b"first namespace")
other_lease: Final = other.lookup("shared-key")
assert isinstance(other_lease, CaptureLease)
assert other.publish("shared-key", other_lease, b"other namespace")
first_hit: Final = store.lookup("shared-key")
other_hit: Final = other.lookup("shared-key")
assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace"
assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace"
def test_identity_preserves_values_and_never_contains_credentials() -> None:
variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'')
url: Final = "https://example.invalid/v1/chat/completions"

10
tests/e2e/AGENTS.md
View file

@ -32,10 +32,10 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `migrations/` - isolated Docker startup, concurrent migration, crash recovery, and legacy database compatibility. The CircleCI migration workflow enables `LITELLM_MIGRATION_TESTS=1`. These tests own their proxy containers and databases, so they do not use the shared proxy preflight or shared database cleanup
- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
- `llm_translation/` - Azure realtime translation is explicitly skipped pending investigation of intermittent missing transcripts and remains unqualified. OpenAI translation and other Azure realtime cases remain enabled. LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
- `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation
- `embeddings/` - the `/embeddings` endpoint across providers
- `batches/` - the `/batches` endpoint (placeholder until the first test lands)
- `batches/` - the `/batches` endpoint. GovCloud batch coverage is explicitly skipped until its infrastructure is provisioned. Other missing credentials and provider errors remain failures
- `realtime/` - realtime websocket sessions, including the pipecat audio path
- `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
- `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface, not Playwright)
@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `gateway/` - proxy configuration only (`litellm-config.yml`). No tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it
- `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py <npm package>`, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, and auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. The Bedrock Runtime setup is explicitly skipped until stage IAM permits `bedrock:InvokeModel` on its default project. The other setups still run, and skipped Runtime cases do not qualify that route. Covered by `tests/e2e_harness/codex/`
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md`
## MCP suite: real Datadog only
@ -329,3 +329,7 @@ other.<area>.<case>.<assertion>
- the `codex/` suite is the one exception: a single setup that a LiteLLM or provider bug breaks may carry `pytest.mark.xfail(strict=True, reason=...)` with a plain one-sentence reason on that setup's `pytest.param`, so it keeps running and goes red once the bug is fixed. Strict only, and never on a flaky setup (one that passes some runs and fails others)
- a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod
## Provider response cache
The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace

5
tests/e2e/batches/COVERAGE.md
View file

@ -11,8 +11,9 @@ cost write-back via a cross-run marker baton (design below).
Only supported cells are tested. The capability table in `capabilities.py` holds one
row per supported (provider, scenario) pair, so there are no skipped cells in the
parametrized run. The batches suite never skips: missing provider creds or upstream
failures are hard test failures (see `tests/e2e/AGENTS.md`).
parametrized run. Missing provider creds or upstream failures are hard test failures
(see `tests/e2e/AGENTS.md`). GovCloud is explicitly skipped until its infrastructure
is provisioned and does not count as qualified coverage.
| Provider | create | retrieve | cancel | list | content download | file backing |
|-----------|--------|----------|--------|------|------------------|--------------|

1
tests/e2e/batches/test_batches_e2e.py
View file

@ -1210,6 +1210,7 @@ def _govcloud_params() -> LiteLLMParamsBody:
)
@pytest.mark.skip(reason="GovCloud E2E infrastructure is not provisioned")
class TestBedrockBatchGovCloud:
"""Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1).

7
tests/e2e/codex/setups.py
View file

@ -13,7 +13,7 @@ from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from models import LiteLLMParamsBody
GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol"
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol"
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/global.openai.gpt-6.1-sol"
GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol"
GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol"
CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5"
@ -60,7 +60,10 @@ SETUPS: Final = (
RateBucket.BEDROCK_CONVERSE,
),
id="bedrock_runtime",
marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
marks=(
_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
pytest.mark.skip(reason="Stage IAM lacks bedrock:InvokeModel on the default Bedrock Runtime project"),
),
),
pytest.param(
Deployment(

2
tests/e2e/guardrails/guardrails_client.py
View file

@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
BlockedWordAction = Literal["BLOCK", "MASK"]

2
tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py
View file

@ -128,7 +128,7 @@ class TestOpenAIModerationGuardrail:
)
allowed = unwrap(
client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=64)
client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=512)
)
assert allowed.content or allowed.choices, (
"the same moderation guardrail must let a benign /v1/messages prompt through, but "

5
tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md
View file

@ -57,8 +57,9 @@ Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy b
kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by
uncommenting their entry.
Every provider is provisioned and asserted; the suite never skips a provider. Per
`tests/e2e/AGENTS.md` there is no sanctioned skip: the whole-suite proxy-liveness
Azure realtime translation is explicitly skipped pending investigation of intermittent
missing transcripts and does not count as qualified coverage. OpenAI translation and
other Azure realtime cases remain enabled. Per `tests/e2e/AGENTS.md`, the whole-suite proxy-liveness
probe hard-fails when no proxy answers, and a provider whose credentials or upstream
realtime model are missing on the gateway is likewise a hard failure, not a skip.
Give the gateway each provider's credentials to turn its tests green.

13
tests/e2e/llm_translation/realtime/test_realtime_e2e.py
View file

@ -360,7 +360,18 @@ LIVE_TRANSCRIPTION_PROVIDERS: Final = (
)
TRANSLATION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "translation")) for p in TRANSLATION_PROVIDERS
pytest.param(
p,
id=p.id,
marks=(
*audio_marks(p, "translation"),
pytest.mark.skipif(
p.id == "azure",
reason="Azure realtime translation intermittently returns no transcript; qualification deferred",
),
),
)
for p in TRANSLATION_PROVIDERS
)
LIVE_TRANSCRIPTION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS

4
tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
View file

@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(

10
tests/e2e/llm_translation/test_custom_pricing_e2e.py
View file

@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@ -37,10 +37,8 @@ from models import (
pytestmark = pytest.mark.e2e
BACKEND_MODEL = "gemini/gemini-2.5-flash"
BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
@ -78,7 +76,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@ -251,7 +249,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token

2
tests/e2e/llm_translation/test_google_native_e2e.py
View file

@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):

2
tests/e2e/llm_translation/test_image_edits_e2e.py
View file

@ -24,7 +24,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
_TEST_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

2
tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py
View file

@ -41,7 +41,7 @@ from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
FLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5"
UNFLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"

6
tests/e2e/llm_translation/test_passthrough_e2e.py
View file

@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)

2
tests/e2e/llm_translation/test_responses_e2e.py
View file

@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"

46
tests/e2e/logging/logging_client.py
View file

@ -18,10 +18,11 @@ import json
import os
import time
from dataclasses import dataclass
from typing import Callable, Literal
from datetime import datetime, timedelta, timezone
from typing import Callable, Final, Literal
import pytest
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
from e2e_metadata import step
@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
guardrail_name: str | None = None
_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
class LangfuseObservation(BaseModel):
model_config = ConfigDict(extra="ignore", populate_by_name=True)
model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
id: str
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
type: str | None = None
calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
calculated_total_cost: float | None = Field(
default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
)
level: str | None = None
input: object | None = None
output: object | None = None
metadata: object | None = None
input: JsonValue = None
output: JsonValue = None
metadata: JsonValue = None
usage: object | None = None
usage_details: object | None = Field(default=None, alias="usageDetails")
model: str | None = None
@field_validator("input", "output", "metadata", mode="before")
@classmethod
def decode_json_fields(cls, value: object) -> JsonValue:
if not isinstance(value, str):
return _LANGFUSE_JSON.validate_python(value)
try:
return _LANGFUSE_JSON.validate_json(value)
except ValidationError:
return value
class LangfuseObservationList(BaseModel):
model_config = ConfigDict(extra="ignore")
@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
class LangfuseListParams(BaseModel):
model_config = ConfigDict(populate_by_name=True)
model_config = ConfigDict(populate_by_name=True, frozen=True)
limit: int = 100
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
from_start_time: str | None = Field(default=None, alias="fromStartTime")
from_start_time: str = Field(alias="fromStartTime")
to_start_time: str = Field(alias="toStartTime")
fields: str = "core,basic,io,metadata,model,usage"
expand_metadata: str = Field(default="attributes", alias="expandMetadata")
@dataclass(frozen=True, slots=True)
@ -580,14 +600,16 @@ class LoggingClient:
name: str | None = None,
from_start_time: str | None = None,
) -> list[LangfuseObservation]:
result = get(
URL(f"{creds.host}/api/public/observations"),
now: Final = datetime.now(timezone.utc)
result: Final = get(
URL(f"{creds.host}/api/public/v2/observations"),
headers=creds.auth_headers,
params=LangfuseListParams(
limit=100,
traceId=trace_id,
name=name,
fromStartTime=from_start_time,
fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
toStartTime=now.isoformat(),
),
response_type=LangfuseObservationList,
timeout=30.0,

10
tests/e2e/logging/test_langsmith_batch_serialization_e2e.py
View file

@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e
class LangsmithHeaders(Headers):
x_api_key: str = Field(serialization_alias="x-api-key")
x_tenant_id: str | None = Field(default=None, serialization_alias="x-tenant-id")
class LangsmithRunExtra(BaseModel):
@ -50,6 +51,7 @@ class LangsmithCreds:
api_key: str
base_url: str
project: str
tenant_id: str | None
def load_langsmith_creds() -> LangsmithCreds:
@ -62,6 +64,7 @@ def load_langsmith_creds() -> LangsmithCreds:
api_key=api_key,
base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"),
project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e",
tenant_id=os.getenv("LANGSMITH_TENANT_ID"),
)
@ -69,7 +72,7 @@ def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None:
result = get_external(
f"{creds.base_url}/runs/{run_id}",
response_type=LangsmithRun,
headers=LangsmithHeaders(x_api_key=creds.api_key),
headers=LangsmithHeaders(x_api_key=creds.api_key, x_tenant_id=creds.tenant_id),
)
match result:
case Success(data=run):
@ -102,7 +105,10 @@ class TestLangsmithBatchSerialization:
async def test_non_json_native_metadata_reaches_langsmith(self) -> None:
creds: Final = load_langsmith_creds()
logger: Final = LangsmithLogger(
langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url
langsmith_api_key=creds.api_key,
langsmith_project=creds.project,
langsmith_base_url=creds.base_url,
langsmith_tenant_id=creds.tenant_id,
)
assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API"
marker: Final = unique_marker()

37
tests/e2e/logging/test_otel_trace_e2e.py
View file

@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
return False
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
"""The enforced behavior: the destination holds exactly one call-id-tagged
trace for the call, rooted at the SERVER span, with auth/db children and
the gen-AI span all connected into that one tree - no dangling parent
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
roots = [span for span in trace.spans if not span.references]
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
root = roots[0]
assert root.operation_name == f"POST {route}", (
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
)
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
return otel_reader.poll_traces_for_call(
call_id=call_id,
settled_names={f"POST {route}", f"auth {route}", genai_span},
settled_names={f"auth {route}", genai_span},
settled_prefixes={DB_SPAN_PREFIX},
)
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
"""The enforced behavior: the gen-AI span for the attempt that served the
stream records a TTFT that is a real measurement - present, numeric,
positive, and strictly less than that span's own total duration. A TTFT of
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
@meta(
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
"""This test verifies that one successful non-streaming /v1/messages request
produces exactly one complete OTEL trace.
The trace must have a single root span named "POST /v1/messages". The
The trace must have a single root SERVER span identifying /v1/messages. The
authentication, database, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
@meta(
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
one_served_genai_span(traces.hits[0], genai_span)
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
@meta(
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
@meta(
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (

2
tests/e2e/management/test_budget_customer_user_org_e2e.py
View file

@ -418,7 +418,7 @@ class TestBudgetListV1:
@pytest.mark.covers("mgmt.budget.list_v1.happy_path")
@meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT))
def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None:
assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400
assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400
assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400
@pytest.mark.covers("mgmt.budget.list_v1.admin_only")

17
tests/e2e/otel_client.py
View file

@ -28,11 +28,10 @@ from dataclasses import dataclass
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
from e2e_metadata import step
from pydantic import BaseModel, ConfigDict, Field
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
JAEGER_SERVICE = "litellm"
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
time.sleep(POLL_INTERVAL)
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
present = set(trace.span_names())
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
root: Final = root_span(trace)
present: Final = frozenset(trace.span_names())
return (
root is not None
and root.kind == "server"
and names.issubset(present)
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
)
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@ -172,7 +177,7 @@ class _Observation:
def settled(self, names: set[str], prefixes: set[str]) -> bool:
hits: Final = self.traces.hits
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
@dataclass(frozen=True, slots=True)

34
tests/e2e/provider_cache_redis.py
View file

@ -14,8 +14,8 @@ from typing import Final, Protocol, cast
from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease
from pydantic import TypeAdapter, ValidationError
from redis import Redis
from redis.exceptions import RedisError
from redis import Redis, RedisCluster
from redis.exceptions import RedisClusterException, RedisError
REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes])
@ -83,7 +83,7 @@ class RedisResponseStore:
started: Final = time.monotonic()
try:
result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return CacheUnavailable()
try:
parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True))
@ -102,28 +102,46 @@ class RedisResponseStore:
result: Final = self.client.eval(
PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms,
)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def release(self, key: str, lease: CaptureLease) -> bool:
try:
result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def discard(self, key: str, payload: bytes) -> bool:
try:
result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload)
except (RedisError, OSError):
except (RedisError, RedisClusterException, OSError):
return False
return result == 1
@functools.lru_cache(maxsize=16)
def _redis_commands(url: str, cluster: bool) -> RedisCommands:
if cluster:
return cast(
RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
)
return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False))
@dataclass(frozen=True, slots=True)
class LazyRedisCommands:
url: str
cluster: bool
def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object:
return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args)
def redis_store(url: str, namespace: str) -> RedisResponseStore:
client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
return RedisResponseStore(cast(RedisCommands, client), namespace)
client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1")
return RedisResponseStore(client, namespace)
def write_metrics(cache: CacheEdge) -> None:

2
tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py
View file

@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"

6
tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py
View file

@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
GEMINI_MODEL = "gemini-2.5-flash"
GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
"/spend/calculate returned 0 for gemini-2.5-flash; "
"/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)

2
tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py
View file

@ -30,7 +30,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, ValidationError
pytestmark = pytest.mark.e2e
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
SEARCH_CALL_TYPE: Final = "asearch"

13
tests/e2e/router/reliability_support.py
View file

@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
COOLDOWN_SECONDS = 30.0
REPLICA_PROPAGATION_SECONDS = 15.0
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
# past that limit comes back as a real `context_length_exceeded` 400, which is
# what litellm maps to ContextWindowExceededError.
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
SMALL_CONTEXT_LIMIT_TOKENS = 16385
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
def oversized_prompt(marker: str) -> str:
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""Register a deployment on the smallest-context model OpenAI still serves, so an
"""Register a deployment with a smaller context window than the backup, so an
oversized prompt earns a real context-window refusal from the provider."""
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""The always-picked half of a retry pair on the smallest-context model OpenAI
still serves: it holds all of the model group's shuffle weight, so an oversized
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
prompt opens on it and earns a real context-window refusal, which never benches
a deployment, so only the retry itself can steer the request off it."""
return proxy.register_model(

2
tests/e2e/ui/helpers/navigation.ts
View file

@ -37,7 +37,7 @@ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Prom
// assert against the default page.
for (let attempt = 0; attempt < 3; attempt++) {
await page.goto(`/ui?page=${pageEnum}`);
await page.waitForLoadState("networkidle");
await expect(page.getByRole("complementary")).toBeVisible();
const url = new URL(page.url());
const onLegacyRoot = url.pathname.replace(/\/+$/, "").endsWith("/ui");
if (!onLegacyRoot || url.searchParams.get("page") === pageEnum) {

6
tests/e2e/ui/tests/modelsPage/addModel.spec.ts
View file

@ -391,7 +391,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the model we just added
await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5");
@ -470,7 +470,7 @@ test.describe("Add Model", () => {
// The Models table renders team-scoped models with the team id in the row.
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
await page.getByPlaceholder("Search model names").fill("cohere");
@ -515,7 +515,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
await page.waitForLoadState("networkidle");
await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the wildcard model
await page.getByPlaceholder("Search model names").fill("cohere");

2
tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts
View file

@ -27,7 +27,7 @@ async function readStoredParams(
}
function paramsEditor(page: PlaywrightPage) {
return page.getByPlaceholder('"rpm": 100');
return page.getByRole("textbox", { name: "LiteLLM Params", exact: true });
}
async function editParams(

54
tests/e2e/ui/tests/navigation/commandPalette.spec.ts Normal file
View file

@ -0,0 +1,54 @@
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { dismissFeedbackPopup, navigateToPage } from "../../helpers/navigation";
import { createVirtualKey, deleteVirtualKey, uniqueSuffix } from "../../helpers/traffic";
const paletteSearch = (page: PlaywrightPage) => page.getByRole("combobox", { name: "Search" });
const headerTrigger = (page: PlaywrightPage) => page.getByRole("button", { name: "Search keys…", exact: true });
test.describe("Command palette", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test.beforeEach(async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
await expect(headerTrigger(page)).toBeVisible();
});
test("opens a key found by its alias", async ({ page }) => {
const alias = `e2e-palette-key-${uniqueSuffix()}`;
const created = await createVirtualKey(page.request, { key_alias: alias });
try {
await page.keyboard.press("ControlOrMeta+K");
await paletteSearch(page).fill(alias);
await expect(page.getByRole("option", { name: new RegExp(`^${alias}`) })).toBeVisible();
await page.keyboard.press("Enter");
await expect(page).toHaveURL(new RegExp(`[?&]key=${created.token}`));
await expect(paletteSearch(page)).toBeHidden();
} finally {
await deleteVirtualKey(page.request, created.token);
}
});
test("does not open over the Create New Key dialog", async ({ page }) => {
await page.getByRole("button", { name: /Create New Key/i }).click();
const keyName = page.getByLabel(/Key Name/);
await keyName.click();
await page.keyboard.press("ControlOrMeta+K");
await page.keyboard.type("still-in-the-dialog");
await expect(keyName).toHaveValue("still-in-the-dialog");
await expect(paletteSearch(page)).toBeHidden();
});
test("opens from the header trigger and closes with Escape", async ({ page }) => {
await headerTrigger(page).click();
await expect(paletteSearch(page)).toBeFocused();
await page.keyboard.press("Escape");
await expect(paletteSearch(page)).toBeHidden();
});
});

49
tests/e2e_harness/logging/test_logging_client.py Normal file
View file

@ -0,0 +1,49 @@
import json
from typing import Final
import pytest
from pydantic import JsonValue
from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
@pytest.mark.parametrize("encoded", [False, True])
@pytest.mark.parametrize("cost", [0.0, 0.000023])
def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
metadata: Final[JsonValue] = {
"attributes": {
"langfuse.observation.type": "generation",
"litellm.metadata.user_api_key_alias": "test-key-alias",
}
}
usage: Final[JsonValue] = {"input": 17, "output": 3}
observation: Final = LangfuseObservation.model_validate(
{
"id": "observation",
"input": json.dumps(content) if encoded else content,
"output": json.dumps(output) if encoded else output,
"metadata": json.dumps(metadata) if encoded else metadata,
"usageDetails": usage,
"totalCost": cost,
}
)
assert observation.input == content
assert observation.output == output
assert observation.usage_details == usage
assert observation_spend(observation) == cost
assert is_otel_v2_generation(observation, key_alias="test-key-alias")
assert not is_otel_v2_generation(observation, key_alias="another-key")
def test_observation_preserves_plain_text_and_legacy_cost() -> None:
cost: Final = 0.000031
observation: Final = LangfuseObservation.model_validate(
{"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
)
assert observation.input == "plain prompt"
assert observation.output == "plain reply"
assert observation_spend(observation) == cost

85
tests/e2e_harness/logging/test_span_selection.py
View file

@ -8,8 +8,21 @@ helper is worth pinning here.
from __future__ import annotations
from typing import Final
import pytest
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
from otel_client import (
TTFT_TAG,
CallTraces,
JaegerReference,
JaegerSpan,
JaegerTag,
JaegerTrace,
one_served_genai_span,
served_genai_spans,
trace_settled,
)
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
GENAI_SPAN = "chat claude-haiku-4-5"
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
def _request_trace(
*,
root_name: str = "POST",
root_kind: str = "server",
route: str = "/v1/messages",
genai_parent: str = "root",
ttft: float = 0.2,
) -> JaegerTrace:
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
return JaegerTrace(
traceID="request",
spans=[
JaegerSpan(
spanID="root",
operationName=root_name,
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
),
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
JaegerSpan(
spanID="genai",
operationName=GENAI_SPAN,
duration=1_000_000,
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
),
],
)
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
trace: Final = _request_trace(root_name=root_name)
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
trace: Final = _request_trace()
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
@pytest.mark.parametrize("route", ("/v1/responses", ""))
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
with pytest.raises(AssertionError, match="route"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
with pytest.raises(AssertionError, match="kind=server"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
trace: Final = _request_trace(genai_parent="absent")
with pytest.raises(AssertionError, match="orphaned trace"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
@pytest.mark.parametrize("ttft", (0.0, 1.0))
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
trace: Final = _request_trace(ttft=ttft)
with pytest.raises(AssertionError, match="TTFT"):
assert_real_ttft((trace,), genai_span=GENAI_SPAN)

3
tests/e2e_harness/pytest.ini
View file

@ -5,5 +5,8 @@
addopts = --strict-markers --strict-config -p no:cacheprovider
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
markers =
e2e: live assertion modules imported by harness regression tests
meta(subject): metadata on live assertion modules imported by harness regression tests
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
cli_determinism: drives the real claude CLI for several seconds

39
tests/integration/configuration/test_fips_mode_boot.py Normal file
View file

@ -0,0 +1,39 @@
import _hashlib
import json
from pathlib import Path
from typing import Final
import pytest
from tests.integration._support.client import Gateway
from tests.integration._support.process import refused_boot_log
def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path:
config: Final = directory / "fips_boot.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "fips-model",
"litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"},
}
],
"general_settings": {"fips_mode": fips_mode},
}
)
)
return config
def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None:
if _hashlib.get_fips_mode() == 1:
pytest.skip("this runner's OpenSSL is in FIPS mode")
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True))
assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log
def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None:
log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced"))
assert "general_settings.fips_mode='enforced' is not a boolean" in log, log

6
tests/integration/mcp/test_mcp_transports.py
View file

@ -669,7 +669,7 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
import httpx2
import yaml
from integration._support.asgi import asgi_server
from integration._support.mcp import _drain_sse_streams
from integration._support.mcp import _acknowledge_notifications_after_dispatch, _drain_sse_streams
from integration._support.process import owned_proxy
from mcp import ClientSession, MCPError
from mcp.client.streamable_http import streamable_http_client
@ -769,7 +769,9 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session(
watchers = tuple(task for task in asyncio.all_tasks() if "_shutdown_watcher" in repr(task.get_coro()))
await asyncio.wait_for(asyncio.gather(*watchers), 2)
hosted: Final = Starlette(routes=[Mount("/", app=app)], lifespan=lifespan)
hosted: Final = Starlette(
routes=[Mount("/", app=_acknowledge_notifications_after_dispatch(app))], lifespan=lifespan
)
with asgi_server(hosted) as upstream_url, httpx.Client() as client:
config: Final = tmp_path / "proxy.yaml"
config.write_text(

75
tests/integration/pricing/test_price_data_reload_prices.py
View file

@ -25,9 +25,16 @@ _GEO: Final = f"au.{_ROW}"
_FINETUNE: Final = "ft:gpt-audit-reload-2025-01-01"
_CLAUDE: Final = "claude-audit-reload-sonnet"
_MINI: Final = "gpt-audit-reload-mini"
_FREE: Final = "gpt-audit-reload-free"
_RULE_FREE: Final = "gpt-audit-rule-free"
_FILL_FREE: Final = "gpt-audit-fill-free"
_RULE_UNPRICED: Final = "gpt-audit-rule-unpriced"
_REPORTED_FREE: Final = "gpt-audit-reported-free"
_USAGE: Final = re.compile(r"bill (\d+) (\d+) (\d+)")
_CUSTOM: Final = MappingProxyType({"input_cost_per_token": 1.1e-06, "output_cost_per_token": 2.3e-06})
_COST_MAP_LIMITS: Final = MappingProxyType({"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"})
_COST_MAP_LIMITS: Final = MappingProxyType(
{"MODEL_COST_MAP_MIN_MODEL_COUNT": "1", "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0"}
)
_PUBLISHED_SCALE: Final = 2.0
_CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
{
@ -60,6 +67,13 @@ _CATALOG: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
"output_cost_per_token": 1.5e-05,
"cache_read_input_token_cost": 3e-07,
},
_FILL_FREE: {"litellm_provider": "openai", "mode": "chat"},
_FREE: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
_MINI: {
"litellm_provider": "openai",
"mode": "chat",
@ -99,6 +113,8 @@ class _Deployment:
prices: Mapping[str, float]
usage: _Usage
rates: Callable[[float], _Rates | None]
reported_cost: float | None = None
listed_rates: Callable[[float], _Rates | None] | None = None
@dataclass(frozen=True, slots=True)
@ -107,10 +123,23 @@ class _Charge:
expected: float
def _catalog(scale: float) -> dict[str, dict[str, JsonValue]]:
def _catalog(scale: float) -> dict[str, JsonValue]:
return {
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
for model, row in _CATALOG.items()
**{
model: {name: value * scale if isinstance(value, float) else value for name, value in row.items()}
for model, row in _CATALOG.items()
},
"fallback_generalizations": {
"rules": [
{
"name": "free-pricing",
"pattern": f"({_RULE_FREE}|{_FILL_FREE})",
"fill_missing_for_providers": ["openai"],
"model_info": {"mode": "chat", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
},
{"name": "unpriced-capabilities", "pattern": _RULE_UNPRICED, "model_info": {"mode": "chat"}},
]
},
}
@ -240,6 +269,7 @@ def _reply(request: Request) -> Reply:
"completion_tokens": usage.completion,
"total_tokens": usage.prompt + usage.completion,
"prompt_tokens_details": {"cached_tokens": usage.cached},
**({"cost": 0.0} if object_value(json.loads(request.body))["model"] == _REPORTED_FREE else {}),
},
}
).encode()
@ -281,10 +311,16 @@ def _billed(client: httpx.Client, key: str, deployment: _Deployment) -> tuple[st
def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) -> _Charge:
rates: Final = deployment.rates(scale)
assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name
assert _listed_rates(proxy, deployment.name) == _expected_listing(
deployment.listed_rates(scale) if deployment.listed_rates is not None else rates
), deployment.name
request_id, charged = _billed(proxy.client, proxy.key, deployment)
expected: Final = _expected_cost(rates, deployment.usage)
assert charged == pytest.approx(expected, rel=1e-6), deployment.name
expected: Final = (
deployment.reported_cost if deployment.reported_cost is not None else _expected_cost(rates, deployment.usage)
)
assert charged == (
None if rates is None and deployment.reported_cost is None else pytest.approx(expected, rel=1e-6)
), deployment.name
return _Charge(request_id=request_id, expected=expected)
@ -352,8 +388,31 @@ def test_price_data_reload_reprices_catalog_deployments_and_keeps_custom_prices(
_Usage(2000, 1000, 1000),
_catalog_rates(_FINETUNE),
),
_Deployment("finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates),
_Deployment(
"finetune-custom", f"openai/{_FINETUNE}:org:suffix:z", _CUSTOM, _Usage(2000, 0, 1000), _custom_rates
),
_Deployment("cross-provider-unpriced", f"azure/{_CLAUDE}", {}, _Usage(2000, 1000, 1000), _unpriced),
_Deployment("catalog-free", f"openai/{_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
_Deployment("rule-free", f"openai/{_RULE_FREE}", {}, _Usage(2000, 1000, 1000), _catalog_rates(_FREE)),
_Deployment(
"fill-rule-free",
f"openai/{_FILL_FREE}",
{},
_Usage(2000, 1000, 1000),
_catalog_rates(_FREE),
listed_rates=_unpriced,
),
_Deployment("rule-unpriced", f"openai/{_RULE_UNPRICED}", {}, _Usage(2000, 1000, 1000), _unpriced),
_Deployment(
"provider-reported-free", f"openrouter/{_REPORTED_FREE}", {}, _Usage(2000, 1000, 1000), _unpriced, 0.0
),
_Deployment(
"cross-provider-free",
f"azure/{_CLAUDE}",
{"input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
_Usage(2000, 1000, 1000),
_catalog_rates(_FREE),
),
_Deployment(
"region-custom-above-threshold",
f"bedrock/ap-northeast-1.{_ROW}",

455
tests/integration/spend/test_session_grouped_logs_pages.py Normal file
View file

@ -0,0 +1,455 @@
import itertools
import json
import os
import uuid
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import psycopg
import pytest
from prisma import Prisma
from psycopg import sql
from pydantic import BaseModel, ConfigDict, Field
from litellm.proxy.spend_tracking.session_page import read_session_page
from litellm.proxy.spend_tracking.spend_management_endpoints import _fetch_session_representatives
from tests.integration._support.client import Gateway
from tests.integration._support.database import write_rows
_URL: Final = "/spend/logs/ui"
_WINDOW_SECONDS: Final = 3600
@dataclass(frozen=True, slots=True)
class _Row:
request: str
session: str | None
api_key: str
second: int
status: str = "success"
model: str = "m-a"
_ROWS: Final = (
_Row("z1", "z", "key-1", -1),
_Row("a1", "a", "key-1", 10),
_Row("a2", "a", "key-1", 50),
_Row("b1", "b", "key-1", 20, model="m-b"),
_Row("b2", "b", "key-1", 60),
_Row("b3", "b", "key-1", 30),
_Row("c1", "c", "key-2", 60),
_Row("d1", "b", "key-2", 40, model="m-b"),
_Row("e1", None, "key-1", 55, model="m-b"),
_Row("f1", "", "key-1", 45),
_Row("g1", "g", "key-3", 5, status="failure"),
_Row("g2", "g", "key-3", 15, status="failure"),
*(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)),
_Row("i1", "i", "key-3", 35, status="failure"),
_Row("i2", "i", "key-3", _WINDOW_SECONDS + 1),
)
_NEWEST_FIRST: Final = (
("h", "key-1"),
("c", "key-2"),
("b", "key-1"),
("e1", "key-1"),
("a", "key-1"),
("f1", "key-1"),
("b", "key-2"),
("i", "key-3"),
("g", "key-3"),
)
class _LogRow(BaseModel):
model_config = ConfigDict(frozen=True)
request_id: str
session_id: str | None
api_key: str
session_total_count: int
class _Page(BaseModel):
model_config = ConfigDict(frozen=True)
data: tuple[_LogRow, ...]
total: int
total_is_capped: bool
has_more: bool
next_session_cursor: str | None
@dataclass(frozen=True, slots=True)
class _Seeded:
prefix: str
window: Mapping[str, str]
def session(self, row: _LogRow) -> tuple[str, str]:
key: Final = row.session_id or row.request_id
assert key.startswith(self.prefix), key
return key.removeprefix(self.prefix), row.api_key
@pytest.fixture
def seeded(gateway: Gateway) -> Iterator[_Seeded]:
prefix: Final = f"sess{uuid.uuid4().hex[:12]}-"
start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000)
rows: Final = [
{
"request_id": f"{prefix}{row.request}",
"session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "",
"api_key": row.api_key,
"start_time": (start + timedelta(seconds=row.second)).isoformat(),
"status": row.status,
"model": row.model,
}
for row in _ROWS
]
with gateway.scenario() as scenario:
scenario.cleanups.callback(
write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",)
)
write_rows(
'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, '
"session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, "
"status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, "
"start_time timestamp, status text, model text)",
(json.dumps(rows),),
)
yield _Seeded(
prefix,
{
"start_date": f"{start:%Y-%m-%d %H:%M:%S}",
"end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}",
"group_by_session": "true",
},
)
def _page(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str]) -> _Page:
return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params}))
def _cursor_walk(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str], number: int = 1) -> tuple[_Page, ...]:
page: Final = _page(gateway, seeded, {**params, "page_size": "2", "page": str(number)})
if page.next_session_cursor is None or number > len(_NEWEST_FIRST):
return (page,)
following: Final = {**params, "session_cursor": page.next_session_cursor}
return (page, *_cursor_walk(gateway, seeded, following, number + 1))
def _rows(pages: tuple[_Page, ...]) -> Iterator[_LogRow]:
return itertools.chain.from_iterable(page.data for page in pages)
def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]:
return tuple(seeded.session(row) for row in _rows(pages))
def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row(
gateway: Gateway, seeded: _Seeded
) -> None:
pages: Final = _cursor_walk(gateway, seeded, {})
assert _sessions(seeded, pages) == _NEWEST_FIRST
assert [(page.total, page.total_is_capped) for page in pages] == [(9, False)] * 5
assert [page.has_more for page in pages] == [True, True, True, True, False]
representatives: Final = tuple(
(row.request_id.removeprefix(seeded.prefix), row.session_total_count) for row in _rows(pages)
)
assert representatives == (
("h6", 6),
("c1", 1),
("b2", 3),
("e1", 1),
("a2", 2),
("f1", 1),
("d1", 1),
("i1", 2),
("g2", 2),
)
@pytest.mark.parametrize(
"sort_order, expected",
[("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))],
ids=["newest-first", "oldest-first"],
)
def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk(
gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...]
) -> None:
pages: Final = tuple(
_page(gateway, seeded, {"page_size": "2", "page": str(number), "sort_order": sort_order})
for number in range(1, 7)
)
assert _sessions(seeded, pages) == expected
assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0]
assert [page.has_more for page in pages] == [True, True, True, True, False, False]
assert {page.total for page in pages} == {len(expected)}
def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None:
pages: Final = _cursor_walk(gateway, seeded, {"status_filter": "failure"})
assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3"))
assert pages[0].total == 2
def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None:
pages: Final = _cursor_walk(gateway, seeded, {"model": "m-b"})
assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1"))
assert tuple(row.request_id.removeprefix(seeded.prefix) for row in _rows(pages)) == ("e1", "d1", "b1")
assert pages[0].total == 3
_COPIED_ROWS: Final = 20_000
_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC)
_COPY_WINDOW: Final = (
"\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')"
)
_COPY_WINDOW_PARAMS: Final = (_COPY_START, _COPY_START + timedelta(days=1))
def _generated_sessions(rows_per_session: int) -> tuple[sql.SQL, tuple[object, ...]]:
"""`_COPIED_ROWS` rows a second apart in sessions of `rows_per_session` rows, a one-row session keyed by request_id."""
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) '
"SELECT 'r' || n, 'acompletion', 'key-1', moment, moment, "
"CASE WHEN %s > 1 THEN 's' || (n - 1) / %s END FROM generate_series(1, %s) n, "
"LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds"
), (rows_per_session, rows_per_session, _COPIED_ROWS, _COPY_START.replace(tzinfo=None))
def _sessions_tied_at_the_newest_moment() -> tuple[sql.SQL, tuple[object, ...]]:
"""`_generated_sessions(1)` plus thirty one-row sessions sharing a newer moment, inserted and keyed against session
order so a read cut inside the tie misses the sessions that rank first."""
start: Final = _COPY_START.replace(tzinfo=None)
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) '
"SELECT request_id, 'acompletion', 'key-1', moment, moment, session_id FROM ("
"SELECT n AS ordinal, 'r' || n AS request_id, %s::timestamp + n * INTERVAL '1 second' AS moment, "
"NULL AS session_id FROM generate_series(1, %s) n UNION ALL "
"SELECT %s + 30 - n, 't' || lpad((29 - n)::text, 2, '0'), %s::timestamp + %s * INTERVAL '1 second', "
"'s' || lpad(n::text, 2, '0') FROM generate_series(0, 29) n) seeded ORDER BY ordinal"
), (start, _COPIED_ROWS, _COPIED_ROWS, start, _COPIED_ROWS + 1)
@dataclass(frozen=True, slots=True)
class _Call:
session: str
call_type: str
status: str | None
second: int
def _calls(calls: tuple[_Call, ...]) -> tuple[sql.SQL, tuple[object, ...]]:
"""One row per call, `second` seconds into the window, ending a second later."""
seeded: Final = tuple(
{
"request_id": f"{call.session}-{call.second}",
"call_type": call.call_type,
"session_id": call.session,
"status": call.status,
"start_time": (_COPY_START + timedelta(seconds=call.second)).replace(tzinfo=None).isoformat(),
}
for call in calls
)
return sql.SQL(
'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id, status) '
"SELECT request_id, call_type, 'key-1', start_time, start_time + INTERVAL '1 second', session_id, status "
"FROM jsonb_to_recordset(%s::jsonb) AS calls(request_id text, call_type text, session_id text, status text, "
"start_time timestamp)"
), (json.dumps(seeded),)
@asynccontextmanager
async def _spend_logs_copy(insert: sql.SQL, args: tuple[object, ...]) -> AsyncIterator[Prisma]:
"""A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded by `insert`."""
schema: Final = f"integration_{uuid.uuid4().hex}"
url: Final = os.environ["DATABASE_URL"]
table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs")
with psycopg.connect(url, autocommit=True) as setup:
setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
try:
setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table))
setup.execute(insert.format(table=table), args)
setup.execute(sql.SQL("ANALYZE {}").format(table))
parsed: Final = urlsplit(url)
query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema})
database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))})
await database.connect()
try:
yield database
finally:
await database.disconnect()
finally:
setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
class _PlanNode(BaseModel):
model_config = ConfigDict(frozen=True)
relation_name: str | None = Field(default=None, alias="Relation Name")
actual_rows: float = Field(alias="Actual Rows")
rows_removed_by_filter: float = Field(default=0, alias="Rows Removed by Filter")
actual_loops: int = Field(alias="Actual Loops")
plans: tuple["_PlanNode", ...] = Field(default=(), alias="Plans")
def spend_log_rows_scanned(self) -> int:
own: Final = (
(self.actual_rows + self.rows_removed_by_filter) * self.actual_loops
if self.relation_name == "LiteLLM_SpendLogs"
else 0
)
return round(own) + sum(child.spend_log_rows_scanned() for child in self.plans)
class _Explained(BaseModel):
model_config = ConfigDict(frozen=True)
plan: _PlanNode = Field(alias="Plan")
class _ExplainedReads:
"""Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans."""
def __init__(self, database: Prisma) -> None:
self.database: Final = database
self.rows_scanned: tuple[int, ...] = ()
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args)
plan: Final = _Explained.model_validate(explained[0]["QUERY PLAN"][0]).plan
self.rows_scanned = (*self.rows_scanned, plan.spend_log_rows_scanned())
return await self.database.query_raw(query, *args)
@pytest.mark.parametrize(
"rows_per_session, page, newest_session, rows_scanned_per_read, total",
[
pytest.param(1, 1, ("r", 20_000), (102,), (100, True), id="first-page"),
pytest.param(1, 3, ("r", 19_980), (102,), (100, True), id="third-page"),
pytest.param(10, 1, ("s", 1_999), (102, 1_171), (100, True), id="second-read-sized-by-the-first-reads-density"),
pytest.param(100, 1, ("s", 199), (102, 6_439, 12_878), (100, True), id="third-read-at-least-doubled"),
pytest.param(1_000, 1, ("s", 19), (102, 12_878, 20_000), (20, False), id="last-read-runs-out-of-rows"),
],
)
async def test_a_newest_first_page_scans_only_the_rows_its_reads_ask_for(
rows_per_session: int,
page: int,
newest_session: tuple[str, int],
rows_scanned_per_read: tuple[int, ...],
total: tuple[int, bool],
) -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session)) as database:
reads: Final = _ExplainedReads(database)
session_page: Final = await read_session_page(
reads,
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=page,
page_size=10,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
prefix, newest = newest_session
assert tuple(session.key for session in session_page.sessions) == tuple(
f"{prefix}{newest - n}" for n in range(10)
)
assert (session_page.total, session_page.total_is_capped) == total
assert reads.rows_scanned == rows_scanned_per_read
async def test_a_page_cut_inside_a_tie_reads_on_until_the_tie_is_whole() -> None:
async with _spend_logs_copy(*_sessions_tied_at_the_newest_moment()) as database:
reads: Final = _ExplainedReads(database)
session_page: Final = await read_session_page(
reads,
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=1,
page_size=5,
status=None,
sort_desc=True,
session_cursor=None,
cap=10,
)
assert tuple(session.key for session in session_page.sessions) == ("s29", "s28", "s27", "s26", "s25")
assert reads.rows_scanned == (12, 24, 48)
@dataclass(frozen=True, slots=True)
class _ExplainedPrisma:
db: _ExplainedReads
async def test_a_pages_representatives_are_found_by_session_not_by_scanning_the_window() -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session=10)) as database:
reads: Final = _ExplainedReads(database)
representatives: Final = await _fetch_session_representatives(
prisma_client=_ExplainedPrisma(reads), # pyright: ignore[reportArgumentType] # only .db is used
where_clause=_COPY_WINDOW,
sql_params=_COPY_WINDOW_PARAMS,
next_param_index=len(_COPY_WINDOW_PARAMS) + 1,
session_keys=(("s1999", "key-1"), ("s1998", "key-1")),
)
assert tuple(row["request_id"] for row in representatives) == ("r20000", "r19990")
assert reads.rows_scanned == (20,), "only the two sessions' rows, not the window's 20,000"
async def test_a_page_number_far_past_the_cap_shows_no_session() -> None:
async with _spend_logs_copy(*_generated_sessions(rows_per_session=1)) as database:
session_page: Final = await read_session_page(
_ExplainedReads(database),
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
page=9_300_000_000_000_000,
page_size=1000,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert (session_page.sessions, session_page.total, session_page.total_is_capped) == ((), 100, True)
_OUTCOME_CALLS: Final = (
_Call("agent", "asend_message", "failure", 10),
_Call("agent", "acompletion", "success", 20),
_Call("mcp", "acompletion", "failure", 5),
_Call("mcp", "call_mcp_tool", "success", 30),
_Call("unknown", "acompletion", None, 40),
_Call("failed", "acompletion", "success", 1),
_Call("failed", "acompletion", "failure", 50),
_Call("recovered", "acompletion", "failure", 2),
_Call("recovered", "acompletion", "success", 45),
)
@pytest.mark.parametrize(
"status, expected",
[("failure", ("failed", "mcp", "agent")), ("success", ("recovered", "unknown"))],
)
async def test_a_status_filter_keeps_sessions_whose_representative_call_has_that_status(
status: str, expected: tuple[str, ...]
) -> None:
"""An agent message stands for its session, then the newest call that is not an MCP tool call, and a missing status
counts as success. The page stays ordered by each session's newest call."""
async with _spend_logs_copy(*_calls(_OUTCOME_CALLS)) as database:
session_page: Final = await read_session_page(
_ExplainedReads(database),
where_clause=_COPY_WINDOW,
params=_COPY_WINDOW_PARAMS,
status=status,
page=1,
page_size=10,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert tuple(session.key for session in session_page.sessions) == expected
assert (session_page.total, session_page.total_is_capped) == (len(expected), False)

20
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
View file

@ -39,6 +39,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
schema_has_unresolvable_ref,
schema_property_count,
split_concatenated_json_objects,
split_embedding_media_for_tpm,
split_media_for_tpm,
strip_encrypted_reasoning_from_messages,
strip_litellm_response_only_fields,
@ -2709,3 +2710,22 @@ def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_unt
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
assert messages == messages_before
@pytest.mark.parametrize("mime", ["application/pdf", "video/mp4", "audio/wav", "image/png"])
def test_embedding_media_split_holds_at_most_one_copy_of_a_large_payload(mime: str) -> None:
import gc
import tracemalloc
uri: Final = f"data:{mime};base64," + base64.b64encode(bytes(3 * 1024 * 1024)).decode("ascii")
split_embedding_media_for_tpm(f"data:{mime};base64,AAAA")
gc.collect()
tracemalloc.start()
try:
_, media = split_embedding_media_for_tpm(uri)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert media
assert peak < 1.5 * len(uri)

1
tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py
View file

@ -22,6 +22,7 @@ TOKEN_PRICED_NAMES: Final = (
"cohere-command-a",
"grok-4-20-reasoning",
"grok-4-20-non-reasoning",
"claude-mythos-5-1",
)
GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning")
CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",)

5
tests/unit/llms/bedrock/embed/test_bedrock_embedding.py
View file

@ -61,6 +61,7 @@ def test_bedrock_embedding_with_api_key_bearer_token(model, input_type, embed_re
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = httpx.Headers()
mock_response.text = json.dumps(embed_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response
@ -774,6 +775,7 @@ def test_bedrock_embedding_header_forwarding(model, embed_response):
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = httpx.Headers()
mock_response.text = json.dumps(embed_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response
@ -909,6 +911,7 @@ def test_bedrock_cohere_v4_embedding_response_parsing():
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = httpx.Headers()
mock_response.text = json.dumps(cohere_v4_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response
@ -1317,6 +1320,7 @@ def test_bedrock_cohere_embedding_types_wrapped_as_list(
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = httpx.Headers()
mock_response.text = json.dumps(cohere_embedding_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response
@ -1848,6 +1852,7 @@ def test_bedrock_embedding_models(model, input_type, embed_response):
with patch.object(client, "post") as mock_post:
mock_response = Mock()
mock_response.status_code = 200
mock_response.headers = httpx.Headers()
mock_response.text = json.dumps(embed_response)
mock_response.json = lambda: json.loads(mock_response.text)
mock_post.return_value = mock_response

31
tests/unit/llms/cohere/embed/test_v1_transformation.py
View file

@ -1,5 +1,7 @@
from unittest.mock import MagicMock
import httpx
import pytest
from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig
from litellm.types.utils import EmbeddingResponse
@ -183,6 +185,7 @@ class TestCohereEmbeddingV1Transform:
"meta": {}, # No billed_units
}
mock_response.json = MagicMock(return_value=response_json)
mock_response.headers = httpx.Headers()
input_data = ["test text"]
data = {"texts": input_data, "input_type": "search_query"}
@ -208,3 +211,31 @@ class TestCohereEmbeddingV1Transform:
# Verify encoding was called
self.encoding.encode.assert_called()
@pytest.mark.parametrize(
("body_key", "item", "expected_tokens"),
[("images", "data:image/png;base64," + "iVBORw0KGgo" * 400, 2047), ("texts", "a short sentence", 5)],
)
def test_transform_response_counts_bedrock_billed_tokens_only_for_images(
self, body_key: str, item: str, expected_tokens: int
):
response = httpx.Response(
200,
json={"id": "e1", body_key: [{}], "embeddings": [[0.1, 0.2]], "response_type": "embeddings_floats"},
headers={"x-amzn-bedrock-input-token-count": "2047"},
)
result = self.config.transform_response(
response=response,
api_key=None,
logging_obj=self.logging_obj,
data={body_key: [item], "input_type": "image" if body_key == "images" else "search_document"},
model_response=EmbeddingResponse(),
model="cohere.embed-v4:0",
encoding=self.encoding,
input=[item],
)
assert result.usage is not None
assert result.usage.prompt_tokens == expected_tokens
assert result.usage.total_tokens == expected_tokens

145
tests/unit/proxy/auth/test_model_checks.py
View file

@ -19,6 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from litellm import Router
from litellm.utils import AZURE_MODEL_PLACEHOLDER
def test_get_team_models_for_all_models_and_team_only_models():
@ -1105,13 +1106,13 @@ def test_expand_wildcard_invalid_litellm_params_passthrough():
@pytest.mark.parametrize(
("allowed_model_names", "expected_names", "expected_listings"),
[
pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"),
pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"),
pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"),
pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"),
pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"),
pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"),
],
)
@respx.mock
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern(
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern(
monkeypatch: pytest.MonkeyPatch,
request: pytest.FixtureRequest,
allowed_model_names: list[str] | None,
@ -2099,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
pytest.param(
"*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all"
),
pytest.param(
"gpt-partial-route-*",
"openai/gpt-partial-route-*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-repeating-the-deployment-partial-word",
),
pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"),
pytest.param(
"team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all"
),
pytest.param(
"team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment"
),
pytest.param(
"*partial-route-9*",
"openai/gpt-partial-route-9*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-with-more-wildcards-than-its-deployment",
),
pytest.param(
"gemini/*partial-route-9*",
"gemini/gemini-*",
None,
{_ROUTE_GEMINI_ID},
id="slashed-name-with-more-wildcards-than-its-deployment",
),
pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"),
pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"),
pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"),
@ -2176,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names(
assert listed_model_info_names == sorted(reachable)
@pytest.mark.parametrize(
("model_name", "expected_v1_models"),
[
pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"),
pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"),
pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"),
],
)
def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder(
model_name: str, expected_v1_models: list[str]
) -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}])
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_names: Final = [
str(row["model_name"])
for row in expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
]
assert v1_models == expected_v1_models
assert model_info_names == (expected_v1_models or [model_name])
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"),
pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"),
],
)
def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog(
model_name: str, deployment_model: str
) -> None:
from fnmatch import fnmatchcase
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
from litellm.utils import get_valid_models
router: Final = Router(
model_list=[
{
"model_name": model_name,
"litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"},
}
]
)
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_rows: Final = expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai"))
routed_models: Final = {
listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)]
for listed_id in v1_models
}
assert v1_models
assert [
listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name)
] == []
assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {}
assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models)
assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == []
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"),
pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"),
pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"),
],
)
def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments(
model_name: str, deployment_model: str
) -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
with_provider: Final = get_known_models_from_wildcard(
model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai")
)
without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model))
assert with_provider == without_provider
def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params

120
tests/unit/proxy/common_utils/test_fips.py Normal file
View file

@ -0,0 +1,120 @@
import _hashlib
from collections.abc import Mapping
from contextlib import nullcontext
from typing import Final
from unittest.mock import Mock
import pytest
from cryptography.exceptions import InternalError
from cryptography.hazmat.bindings._rust import openssl as rust_openssl
from cryptography.hazmat.primitives import hashes
from litellm.proxy.common_utils.fips import (
FipsModeError,
cryptography_enforces_fips,
enforce_fips_mode,
openssl_enforces_fips,
)
def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None:
real_new: Final = _hashlib.new
def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH:
if name == "md5" and md5_refused:
raise _hashlib.UnsupportedDigestmodError("unsupported hash type")
return real_new(name, data, usedforsecurity=usedforsecurity)
monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag)
monkeypatch.setattr(_hashlib, "new", new)
@pytest.mark.parametrize(
"fips_flag, md5_refused, enforces",
[(1, True, True), (0, True, False), (1, False, False)],
ids=["fips", "fips-flag-off", "md5-allowed"],
)
def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(
monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool
) -> None:
_fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused)
assert openssl_enforces_fips() is enforces
def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delattr(_hashlib, "get_fips_mode")
assert openssl_enforces_fips() is False
@pytest.mark.parametrize(
"fips_enabled, sha256_broken, enforces",
[(True, False, True), (False, False, False), (True, True, False)],
ids=["fips", "fips-flag-off", "sha256-broken"],
)
def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256(
monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool
) -> None:
monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled)
if sha256_broken:
monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", [])))
assert cryptography_enforces_fips() is enforces
def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delattr(rust_openssl, "is_fips_enabled")
assert cryptography_enforces_fips() is False
def test_this_runners_python_openssl_does_not_enforce_fips() -> None:
if _hashlib.get_fips_mode() == 1:
pytest.skip("this runner's Python OpenSSL is in FIPS mode")
assert openssl_enforces_fips() is False
def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None:
if rust_openssl.is_fips_enabled():
pytest.skip("this runner's cryptography OpenSSL is in FIPS mode")
assert cryptography_enforces_fips() is False
@pytest.mark.parametrize(
"general_settings, openssl_fips, cryptography_fips, refusal",
[
({}, False, False, None),
({"fips_mode": False}, False, False, None),
({"fips_mode": "false"}, False, False, None),
({"fips_mode": True}, True, True, None),
({"fips_mode": "true"}, True, True, None),
({"fips_mode": True}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"),
({"fips_mode": " TRUE "}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"),
({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"),
({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"),
({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"),
({"fips_mode": None}, True, True, "general_settings.fips_mode has no value"),
],
ids=[
"unset",
"false",
"false-from-a-string",
"fips",
"fips-from-a-string",
"openssl-not-fips",
"openssl-not-fips-from-a-padded-string",
"cryptography-not-fips",
"not-a-boolean",
"a-number",
"no-value",
],
)
def test_enforce_fips_mode(
monkeypatch: pytest.MonkeyPatch,
general_settings: Mapping[str, object],
openssl_fips: bool,
cryptography_fips: bool,
refusal: str | None,
) -> None:
_fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips)
monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips)
with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext():
enforce_fips_mode(general_settings)

15
tests/unit/proxy/db/test_exception_handler.py
View file

@ -953,6 +953,21 @@ def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None
assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False
@pytest.mark.parametrize(
("error", "expected"),
[
(_lock_timeout_error(message="canceling statement due to statement timeout", code="57014"), True),
(_lock_timeout_error(message="canceling statement due to statement timeout", code=None), True),
(_lock_timeout_error(), False),
(_capacity_error("sorry, too many clients already"), False),
(RuntimeError("canceling statement due to statement timeout"), False),
],
ids=["meta_57014", "message_only", "lock_timeout_55P03", "capacity_53300", "not_prisma"],
)
def test_is_statement_timeout_error(error: Exception, expected: bool) -> None:
assert PrismaDBExceptionHandler.is_statement_timeout_error(error) is expected
@pytest.mark.parametrize(
("body", "retryable"),
[

331
tests/unit/proxy/hooks/test_tpm_concurrent.py
View file

@ -20,6 +20,7 @@ import copy
import io
import json
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import Any, Dict, Final
import pytest
@ -27,7 +28,7 @@ import soundfile as sf
from fastapi import HTTPException
from PIL import Image
from litellm import token_counter
from litellm import get_model_info, token_counter
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, estimate_image_tokens
@ -3971,6 +3972,334 @@ async def test_media_pre_call_rejects_tight_quota_and_reserves_sufficient_quota(
assert expected[0] > 1000 and expected[1] >= expected[0]
_EMBEDDING_MODEL: Final = "gemini-embedding-2"
_EMBEDDING_CAPTION: Final = "a red bicycle leaning on a wall"
def _embedding_media_uri(kind: str, compressed: bool) -> str:
mime: Final = "image/png" if kind == "image" else "audio/wav"
return f"data:{mime};base64,{_quota_media_payload(kind, compressed)}"
@pytest.mark.parametrize("kind", ["image", "audio"])
@pytest.mark.parametrize(
"shape",
["string", "list", "file_block", "with_caption", "combined"],
)
def test_embedding_media_input_is_estimated_independently_of_encoding(
rate_limiter: tuple[RateLimitHandler, DualCache], kind: str, shape: str
) -> None:
handler, _ = rate_limiter
def embedding_request(uri: str) -> dict[str, object]:
shapes: Final[dict[str, object]] = {
"string": uri,
"list": [uri],
"file_block": [{"type": "file", "file": {"file_data": uri}}],
"with_caption": [_EMBEDDING_CAPTION, uri],
"combined": [[_EMBEDDING_CAPTION, uri]],
}
return {"model": _EMBEDDING_MODEL, "input": shapes[shape]}
uncompressed: Final = embedding_request(_embedding_media_uri(kind, False))
original: Final = copy.deepcopy(uncompressed)
estimate: Final = handler._estimate_input_and_output_tokens(uncompressed, call_type="aembedding")
assert estimate == handler._estimate_input_and_output_tokens(
embedding_request(_embedding_media_uri(kind, True)), call_type="aembedding"
)
assert 0 < estimate[0] < 2_000 and estimate[1] == 0
assert uncompressed == original
@pytest.mark.parametrize("kind", ["image", "audio"])
def test_embedding_media_input_costs_the_same_as_chat_media(
rate_limiter: tuple[RateLimitHandler, DualCache], kind: str
) -> None:
handler, _ = rate_limiter
uri: Final = _embedding_media_uri(kind, False)
chat_block: Final = (
{"type": "image_url", "image_url": {"url": uri}}
if kind == "image"
else {"type": "input_audio", "input_audio": {"data": _quota_media_payload(kind, False), "format": "wav"}}
)
chat_input_tokens, _ = handler._estimate_input_and_output_tokens(
{"model": _EMBEDDING_MODEL, "messages": [{"role": "user", "content": [chat_block]}]},
call_type="acompletion",
)
caption_tokens, _ = handler._estimate_input_and_output_tokens(
{"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION]}, call_type="aembedding"
)
assert handler._estimate_input_and_output_tokens(
{"model": _EMBEDDING_MODEL, "input": uri}, call_type="aembedding"
) == (chat_input_tokens, 0)
assert handler._estimate_input_and_output_tokens(
{"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, uri]}, call_type="aembedding"
) == (caption_tokens + chat_input_tokens, 0)
@pytest.mark.asyncio
@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"])
async def test_project_token_limit_admits_image_embedding_and_still_rejects_long_text(
rate_limiter: tuple[RateLimitHandler, DualCache], limit_field: str
) -> None:
handler, cache = rate_limiter
auth: Final = UserAPIKeyAuth(
api_key=hash_token(f"sk-embedding-image-{limit_field}"),
project_id=f"proj-embedding-image-{limit_field}",
project_metadata={limit_field: {_EMBEDDING_MODEL: 75_000}},
)
image_uri: Final = _embedding_media_uri("image", False)
long_text: Final = "word " * 80_000
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={"model": _EMBEDDING_MODEL, "input": image_uri},
call_type="aembedding",
)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={"model": _EMBEDDING_MODEL, "input": long_text},
call_type="aembedding",
)
assert error.value.status_code == 429
assert len(image_uri) > len(long_text)
_EMBEDDING_FILE_MIMES: Final = MappingProxyType({"pdf": "application/pdf", "video": "video/mp4"})
def _embedding_file_uri(kind: str) -> str:
return f"data:{_EMBEDDING_FILE_MIMES[kind]};base64,{base64.b64encode(bytes(range(256)) * 1200).decode()}"
def _embedding_uri(kind: str) -> str:
return _embedding_file_uri(kind) if kind in _EMBEDDING_FILE_MIMES else _embedding_media_uri(kind, False)
@pytest.mark.asyncio
@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"])
@pytest.mark.parametrize("kind", ["pdf", "video"])
@pytest.mark.parametrize("shape", ["data_uri", "file_block"])
async def test_project_token_limit_admits_document_embedding_up_to_the_model_input_limit(
rate_limiter: tuple[RateLimitHandler, DualCache], limit_field: str, kind: str, shape: str
) -> None:
handler, cache = rate_limiter
uri: Final = _embedding_file_uri(kind)
model_input_limit: Final = get_model_info(_EMBEDDING_MODEL)["max_input_tokens"]
assert isinstance(model_input_limit, int) and len(uri) // 4 > 75_000 > model_input_limit
def project_auth(limit: int) -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key=hash_token(f"sk-embedding-{kind}-{shape}-{limit_field}-{limit}"),
project_id=f"proj-embedding-{kind}-{shape}-{limit_field}-{limit}",
project_metadata={limit_field: {_EMBEDDING_MODEL: limit}},
)
item: Final = uri if shape == "data_uri" else {"type": "file", "file": {"file_data": uri}}
request: Final = {"model": _EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, item]}
await handler.async_pre_call_hook(
user_api_key_dict=project_auth(75_000), cache=cache, data=dict(request), call_type="aembedding"
)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=project_auth(model_input_limit // 2),
cache=cache,
data=dict(request),
call_type="aembedding",
)
assert error.value.status_code == 429
_TEXT_ONLY_EMBEDDING_MODEL: Final = "text-embedding-3-small"
@pytest.mark.parametrize("kind", ["image", "audio", "pdf", "video"])
def test_text_only_embedding_model_counts_data_uri_as_text(
rate_limiter: tuple[RateLimitHandler, DualCache], kind: str
) -> None:
handler, _ = rate_limiter
uri: Final = _embedding_uri(kind)
assert handler._estimate_input_and_output_tokens(
{"model": _TEXT_ONLY_EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, uri]}, call_type="aembedding"
) == handler._estimate_input_and_output_tokens(
{"model": _TEXT_ONLY_EMBEDDING_MODEL, "input": [_EMBEDDING_CAPTION, "x" * len(uri)]},
call_type="aembedding",
)
def _project_limited_embedding_call(
limit_field: str, backends: tuple[str, ...], kind: str = "image"
) -> tuple[RateLimitHandler, DualCache, UserAPIKeyAuth, dict[str, object]]:
cache: Final = DualCache()
handler: Final = RateLimitHandler(
internal_usage_cache=InternalUsageCache(cache), media_model_resolver=lambda _model, _team_id: backends
)
scope: Final = f"{limit_field}-{kind}-{'-'.join(backends)}"
auth: Final = UserAPIKeyAuth(
api_key=hash_token(f"sk-embedding-backends-{scope}"),
project_id=f"proj-embedding-backends-{scope}",
project_metadata={limit_field: {"team-embedder": 75_000}},
)
return handler, cache, auth, {"model": "team-embedder", "input": _embedding_uri(kind)}
@pytest.mark.asyncio
@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"])
@pytest.mark.parametrize(
("backends", "kind"),
[
((_TEXT_ONLY_EMBEDDING_MODEL,), "image"),
((_EMBEDDING_MODEL, _TEXT_ONLY_EMBEDDING_MODEL), "image"),
(("voyage/voyage-multimodal-3.5",), "image"),
(("oci/cohere.embed-v4.0",), "image"),
(("bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",), "image"),
(("bedrock/twelvelabs.marengo-embed-3-0-v1:0",), "image"),
(("bedrock/amazon.nova-2-multimodal-embeddings-v1:0",), "pdf"),
],
ids=["text", "mixed", "voyage", "oci", "marengo_2_7", "marengo_3", "nova_pdf"],
)
async def test_project_token_limit_rejects_embedding_media_a_backend_reads_as_text(
limit_field: str, backends: tuple[str, ...], kind: str
) -> None:
handler, cache, auth, request = _project_limited_embedding_call(limit_field, backends, kind)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data=request, call_type="aembedding")
assert error.value.status_code == 429
@pytest.mark.asyncio
@pytest.mark.parametrize("limit_field", ["model_tpm_limit", "model_itpm_limit"])
@pytest.mark.parametrize(
"backend",
[
_EMBEDDING_MODEL,
"bedrock/amazon.titan-embed-image-v1",
"bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0",
],
ids=["gemini", "titan", "nova_async_invoke"],
)
async def test_project_token_limit_admits_image_embedding_through_an_alias_of_an_image_backend(
limit_field: str, backend: str
) -> None:
handler, cache, auth, request = _project_limited_embedding_call(limit_field, (backend,))
await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data=request, call_type="aembedding")
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={**request, "input": "word " * 100_000},
call_type="aembedding",
)
assert error.value.status_code == 429
def _malformed_embedding_uri(shape: str) -> str:
payload: Final = _quota_media_payload("image", False)
whole_quanta: Final = payload.rstrip("=")[: len(payload.rstrip("=")) // 4 * 4]
words: Final = "word " * 100_000
return {
"no_comma": f"data:audio/wav;base64{words}",
"text_payload": f"data:image/png;base64,{words}",
"no_base64_marker": f"data:image/png,{payload}",
"header_past_256_chars": f"data:image/png;{'x' * 300};base64,{payload}",
"noncanonical_padding": f"data:image/png;base64,{whole_quanta}AB==",
"padding_mid_payload": f"data:image/png;base64,{whole_quanta}AA==AAAA",
"length_not_multiple_of_4": f"data:image/png;base64,{whole_quanta}A",
}[shape]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"shape",
[
"no_comma",
"text_payload",
"no_base64_marker",
"header_past_256_chars",
"noncanonical_padding",
"padding_mid_payload",
"length_not_multiple_of_4",
],
)
async def test_project_token_limit_counts_a_malformed_data_uri_as_text(
rate_limiter: tuple[RateLimitHandler, DualCache], shape: str
) -> None:
handler, cache = rate_limiter
auth: Final = UserAPIKeyAuth(
api_key=hash_token(f"sk-embedding-malformed-{shape}"),
project_id=f"proj-embedding-malformed-{shape}",
project_metadata={"model_tpm_limit": {_EMBEDDING_MODEL: 75_000}},
)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={"model": _EMBEDDING_MODEL, "input": _malformed_embedding_uri(shape)},
call_type="aembedding",
)
assert error.value.status_code == 429
@pytest.mark.asyncio
@pytest.mark.parametrize("backend", ["cohere/embed-v4.0", "cohere.embed-v4:0"], ids=["cohere", "bedrock_cohere"])
async def test_project_token_limit_counts_a_mixed_batch_as_text_when_the_backend_types_the_whole_batch(
backend: str,
) -> None:
handler, cache, auth, request = _project_limited_embedding_call("model_tpm_limit", (backend,))
image_uri: Final = request["input"]
await handler.async_pre_call_hook(
user_api_key_dict=auth, cache=cache, data={**request, "input": [image_uri, image_uri]}, call_type="aembedding"
)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={**request, "input": [image_uri, _EMBEDDING_CAPTION]},
call_type="aembedding",
)
assert error.value.status_code == 429
@pytest.mark.asyncio
async def test_project_itpm_limit_reserves_media_tokens_for_an_image_embedding(
rate_limiter: tuple[RateLimitHandler, DualCache],
) -> None:
handler, cache = rate_limiter
image_uri: Final = _embedding_media_uri("image", False)
image_tokens, _ = handler._estimate_input_and_output_tokens(
{"model": _EMBEDDING_MODEL, "input": image_uri}, call_type="aembedding"
)
auth: Final = UserAPIKeyAuth(
api_key=hash_token("sk-embedding-image-itpm-media"),
project_id="proj-embedding-image-itpm-media",
project_metadata={"model_itpm_limit": {_EMBEDDING_MODEL: image_tokens // 2}},
)
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={"model": _EMBEDDING_MODEL, "input": _EMBEDDING_CAPTION},
call_type="aembedding",
)
with pytest.raises(HTTPException) as error:
await handler.async_pre_call_hook(
user_api_key_dict=auth,
cache=cache,
data={"model": _EMBEDDING_MODEL, "input": image_uri},
call_type="aembedding",
)
assert error.value.status_code == 429
@pytest.mark.asyncio
async def test_ptu_pre_call_weights_media_input_and_output() -> None:
cache: Final = DualCache()

112
tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py
View file

@ -3352,6 +3352,8 @@ async def test_user_info_v2_internal_user_cannot_query_other(mocker):
mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique)
mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(return_value=[])
mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_request = mocker.MagicMock(spec=Request)
@ -3456,6 +3458,40 @@ async def test_user_info_v2_nonexistent_user_returns_404(mocker):
assert "nonexistent-user-id" in str(exc_info.value.message)
@pytest.mark.parametrize(
("find_unique", "expected_code", "logs_traceback"),
[
(AsyncMock(return_value=None), "404", False),
(AsyncMock(side_effect=RuntimeError("db down")), "500", True),
],
)
@pytest.mark.asyncio
async def test_user_info_v2_logs_traceback_only_for_server_errors(
mocker: MockerFixture,
caplog: pytest.LogCaptureFixture,
find_unique: AsyncMock,
expected_code: str,
logs_traceback: bool,
) -> None:
from fastapi import Request
from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2
prisma: Final = mocker.MagicMock()
prisma.db.litellm_usertable.find_unique = find_unique
mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma)
with caplog.at_level(logging.ERROR), pytest.raises(ProxyException) as exc_info:
await user_info_v2(
request=mocker.MagicMock(spec=Request),
user_id="some-user",
user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert exc_info.value.code == expected_code
assert any(record.exc_info for record in caplog.records) is logs_traceback
@pytest.mark.asyncio
async def test_user_info_v2_response_shape(mocker):
"""
@ -3674,6 +3710,8 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker):
mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams)
mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(return_value=[])
mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
mock_request = mocker.MagicMock(spec=Request)
@ -3690,6 +3728,80 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker):
assert exc_info.value.code == "404"
_ORG_MEMBERSHIPS: Final = (
SimpleNamespace(user_id="org-admin-user", organization_id="org-a", user_role="org_admin"),
SimpleNamespace(user_id="org-admin-user", organization_id="org-c", user_role="internal_user"),
SimpleNamespace(user_id="member-of-a", organization_id="org-a", user_role="internal_user"),
SimpleNamespace(user_id="member-of-b", organization_id="org-b", user_role="internal_user"),
SimpleNamespace(user_id="member-of-c", organization_id="org-c", user_role="internal_user"),
)
def _membership_matches(membership: SimpleNamespace, where: Mapping[str, object]) -> bool:
return membership.user_id == where["user_id"] and (
"user_role" not in where or membership.user_role == where["user_role"]
)
def _org_admin_prisma(mocker: MockerFixture) -> MagicMock:
def user_row(user_id: str) -> MagicMock:
row: Final = mocker.MagicMock()
row.teams = []
row.model_dump.return_value = {"user_id": user_id, "user_email": f"{user_id}@example.com", "teams": []}
return row
async def find_unique(**kwargs: dict[str, str]) -> MagicMock | None:
user_id: Final = kwargs["where"]["user_id"]
return user_row(user_id) if user_id in {m.user_id for m in _ORG_MEMBERSHIPS} else None
async def find_many(**kwargs: Mapping[str, object]) -> list[SimpleNamespace]:
return [m for m in _ORG_MEMBERSHIPS if _membership_matches(m, kwargs["where"])]
prisma: Final = mocker.MagicMock()
prisma.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=find_unique)
prisma.db.litellm_organizationmembership.find_many = mocker.AsyncMock(side_effect=find_many)
return prisma
@pytest.mark.asyncio
async def test_user_info_v2_org_admin_can_query_member_of_their_org(mocker: MockerFixture) -> None:
from fastapi import Request
from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2
mocker.patch("litellm.proxy.proxy_server.prisma_client", _org_admin_prisma(mocker))
response: Final = await user_info_v2(
request=mocker.MagicMock(spec=Request),
user_id="member-of-a",
user_api_key_dict=UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER),
)
assert response.user_id == "member-of-a"
assert response.user_email == "member-of-a@example.com"
@pytest.mark.parametrize("target_user_id", ["member-of-b", "member-of-c"])
@pytest.mark.asyncio
async def test_user_info_v2_org_admin_cannot_query_user_outside_their_admin_orgs(
mocker: MockerFixture, target_user_id: str
) -> None:
from fastapi import Request
from litellm.proxy.management_endpoints.internal_user_endpoints import user_info_v2
mocker.patch("litellm.proxy.proxy_server.prisma_client", _org_admin_prisma(mocker))
with pytest.raises(ProxyException) as exc_info:
await user_info_v2(
request=mocker.MagicMock(spec=Request),
user_id=target_user_id,
user_api_key_dict=UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER),
)
assert exc_info.value.code == "404"
@pytest.mark.asyncio
async def test_user_info_v2_url_encoding_plus_character(mocker):
"""

94
tests/unit/proxy/management_endpoints/test_team_endpoints.py
View file

@ -13,6 +13,7 @@ import litellm
import pytest
from fastapi import HTTPException, Request
from fastapi.testclient import TestClient
from prisma.errors import DataError
from pydantic import ValidationError
from litellm._uuid import uuid
@ -17278,6 +17279,28 @@ def _team_spend_by_user_db_row(team_id: str, user_id: str, spend: float, request
}
class _RecordingTransaction:
def __init__(self, query_raw: AsyncMock) -> None:
self.statements: tuple[str, ...] = ()
self.query_raw: AsyncMock = query_raw
async def execute_raw(self, sql: str) -> int:
self.statements = (*self.statements, sql)
return 0
async def __aenter__(self) -> "_RecordingTransaction":
return self
async def __aexit__(self, *exc_info: object) -> None:
pass
def _team_spend_by_user_query(mock_db_client: MagicMock, query_raw: AsyncMock) -> _RecordingTransaction:
transaction: Final = _RecordingTransaction(query_raw)
mock_db_client.read_db.tx = MagicMock(return_value=transaction)
return transaction
@pytest.mark.asyncio
async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(mock_db_client):
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
@ -17286,12 +17309,15 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
beta = _team_spend_by_user_team("team-beta", "Team Beta", Member(user_id="alice", role="user"), [])
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha, beta])
mock_db_client.db.query_raw = AsyncMock(
return_value=[
_team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3),
_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2),
_team_spend_by_user_db_row("team-beta", "alice", 0.1, 1),
]
transaction = _team_spend_by_user_query(
mock_db_client,
AsyncMock(
return_value=[
_team_spend_by_user_db_row("team-alpha", "alice", 0.5, 3),
_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2),
_team_spend_by_user_db_row("team-beta", "alice", 0.1, 1),
]
),
)
response = await get_team_spend_by_user(
@ -17301,7 +17327,9 @@ async def test_get_team_spend_by_user_admin_groups_spend_logs_by_team_and_user(m
end_date="2026-09-04",
)
sql, *params = mock_db_client.db.query_raw.call_args.args
assert any("statement_timeout = 30000" in statement for statement in transaction.statements)
mock_db_client.db.tx.assert_not_called()
sql, *params = transaction.query_raw.call_args.args
assert params == ["2026-09-01", "2026-09-04", "team-alpha", "team-beta"]
assert 'FROM "LiteLLM_SpendLogs" sl' in sql
assert 'sl."startTime" >= $1::timestamp' in sql
@ -17328,7 +17356,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien
caller = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER)
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
mock_db_client.db.query_raw = AsyncMock(return_value=[])
transaction = _team_spend_by_user_query(mock_db_client, AsyncMock(return_value=[]))
mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock(
return_value=_team_spend_by_user_caller("alice", ["team-alpha"])
)
@ -17337,7 +17365,7 @@ async def test_get_team_spend_by_user_team_admin_sees_every_member(mock_db_clien
user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04"
)
sql, *params = mock_db_client.db.query_raw.call_args.args
sql, *params = transaction.query_raw.call_args.args
assert params == ["2026-09-01", "2026-09-04", "team-alpha"]
assert 'sl."user" = $' not in sql
@ -17350,7 +17378,9 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="bob", role="user"), ["/key/info"])
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
mock_db_client.db.query_raw = AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)])
transaction = _team_spend_by_user_query(
mock_db_client, AsyncMock(return_value=[_team_spend_by_user_db_row("team-alpha", "bob", 0.25, 2)])
)
mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock(
return_value=_team_spend_by_user_caller("bob", ["team-alpha"])
)
@ -17359,7 +17389,7 @@ async def test_get_team_spend_by_user_plain_member_only_sees_own_row(mock_db_cli
user_api_key_dict=caller, team_ids="team-alpha", start_date="2026-09-01", end_date="2026-09-04"
)
sql, *params = mock_db_client.db.query_raw.call_args.args
sql, *params = transaction.query_raw.call_args.args
assert params == ["2026-09-01", "2026-09-04", "team-alpha", "bob"]
assert "sl.team_id IN ($3)" in sql
assert 'AND sl."user" = $4' in sql
@ -17371,7 +17401,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
caller = UserAPIKeyAuth(user_id="bob", user_role=LitellmUserRoles.INTERNAL_USER)
mock_db_client.db.query_raw = AsyncMock(return_value=[])
transaction = _team_spend_by_user_query(mock_db_client, AsyncMock(return_value=[]))
mock_db_client.writer_db.litellm_usertable.find_unique = AsyncMock(
return_value=_team_spend_by_user_caller("bob", ["team-alpha"])
)
@ -17382,7 +17412,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie
)
assert exc_info.value.status_code == 404
mock_db_client.db.query_raw.assert_not_called()
transaction.query_raw.assert_not_called()
@pytest.mark.asyncio
@ -17400,7 +17430,7 @@ async def test_get_team_spend_by_user_member_of_other_team_gets_404(mock_db_clie
async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids, start_date, end_date, expected_error):
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
mock_db_client.db.query_raw = AsyncMock(return_value=[])
transaction = _team_spend_by_user_query(mock_db_client, AsyncMock(return_value=[]))
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
with pytest.raises(HTTPException) as exc_info:
@ -17410,7 +17440,41 @@ async def test_get_team_spend_by_user_rejects_bad_input(mock_db_client, team_ids
assert exc_info.value.status_code == 400
assert expected_error in str(exc_info.value.detail)
mock_db_client.db.query_raw.assert_not_called()
transaction.query_raw.assert_not_called()
def _statement_timeout_error() -> DataError:
message = "canceling statement due to statement timeout"
return DataError(
data={
"user_facing_error": {
"is_panic": False,
"message": f"Error querying the database: {message}",
"meta": {"code": "57014", "message": message},
}
}
)
@pytest.mark.asyncio
async def test_get_team_spend_by_user_statement_timeout_returns_503(mock_db_client):
from litellm.proxy.management_endpoints.team_endpoints import get_team_spend_by_user
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
alpha = _team_spend_by_user_team("team-alpha", "Team Alpha", Member(user_id="alice", role="admin"), [])
mock_db_client.db.litellm_teamtable.find_many = AsyncMock(return_value=[alpha])
_team_spend_by_user_query(mock_db_client, AsyncMock(side_effect=_statement_timeout_error()))
with pytest.raises(HTTPException) as exc_info:
await get_team_spend_by_user(
user_api_key_dict=admin,
team_ids="team-alpha",
start_date="2026-09-01",
end_date="2026-09-04",
)
assert exc_info.value.status_code == 503
assert "longer than 30s" in str(exc_info.value.detail)
class _TeamRowWithOrganization(LiteLLM_TeamTable):

3
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
View file

@ -69,7 +69,7 @@ def test_get_blog_posts_returns_every_fetched_post(client):
def test_get_blog_posts_returns_local_backup_on_failure(client):
"""Endpoint returns the local backup, with empty tags, when the fetcher raises."""
"""Endpoint returns the local backup when the fetcher raises."""
with patch(
"litellm.proxy.public_endpoints.public_endpoints.get_blog_posts",
side_effect=Exception("fetch failed"),
@ -79,4 +79,3 @@ def test_get_blog_posts_returns_local_backup_on_failure(client):
assert response.status_code == 200
backup = [post.model_dump(mode="json") for post in GetBlogPosts.load_local_blog_posts()]
assert response.json()["posts"] == backup
assert all(post["tags"] == [] for post in response.json()["posts"])

230
tests/unit/proxy/spend_tracking/test_cost_finalization.py
View file

@ -1,8 +1,9 @@
import asyncio
from datetime import datetime, timezone
from typing import Final
from typing import Final, Literal
import pytest
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
@ -450,18 +451,17 @@ async def test_completed_call_keeps_its_charge_when_post_call_processing_fails(
disable_spend_updates: bool,
) -> None:
from unittest.mock import create_autospec
import anyio
from fastapi import Request, Response
from litellm.proxy._types import ProxyException
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyException
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy import proxy_server
from litellm.proxy.utils import ProxyLogging
scope: Final = anyio.CancelScope()
@ -598,6 +598,7 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
logging_obj: Logging, monkeypatch: pytest.MonkeyPatch
) -> None:
import httpx
from litellm.caching.dual_cache import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy.utils import ProxyLogging
@ -633,3 +634,222 @@ async def test_guardrail_json_rejection_retains_completed_provider_usage(
assert (payload["prompt_tokens"], payload["completion_tokens"]) == (10, 5)
assert payload["response_cost"] == 0.5
ProxyLogging._callback_capabilities_cache.clear()
@pytest.mark.asyncio
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
@pytest.mark.parametrize("discount", [0.0, 1.0], ids=["no-discount", "fully-discounted"])
async def test_finalized_header_distinguishes_missing_prices_from_free_models(
monkeypatch: pytest.MonkeyPatch, price: float | None, discount: float
) -> None:
model: Final = "claude-header-pricing-test"
monkeypatch.setattr(litellm, "cost_discount_config", {"azure": discount})
monkeypatch.setitem(
litellm.model_cost,
f"azure/{model}",
{
"litellm_provider": "azure",
"mode": "chat",
"regional_endpoint_uplift_multiplier": 1.0,
**({"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}),
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="pricing-header",
function_id="pricing-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="azure", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None
assert finalized.amount == pytest.approx((price or 0.0) * 15 * (1 - discount))
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"source",
["unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough"],
)
@pytest.mark.parametrize("guardrail_cost", [0.0, 0.1])
async def test_zero_cost_headers_preserve_known_free_calls_and_guardrail_charges(
source: Literal[
"unpriced", "cache", "non-inference", "provider", "provider-header", "per-token", "per-second", "passthrough"
],
guardrail_cost: float,
) -> None:
model: Final = "unmapped-header-cost"
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="adelete_responses" if source == "non-inference" else "acompletion",
start_time=NOW,
litellm_call_id="zero-cost-header",
function_id="zero-cost-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params = {"response_cost": 0.0, "cache_hit": source == "cache"}
match source:
case "provider":
logging._provider_reported_cost = 0.0
case "provider-header":
result._hidden_params["additional_headers"] = {"llm_provider-x-litellm-response-cost": 0.0}
case "per-token":
logging.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}
case "per-second":
logging.custom_cost_per_second = 0.0
case "passthrough":
logging.model_call_details["passthrough_logging_payload"] = {"cost_per_request": 0.0}
case _:
pass
finalized: Final = await finalize_response_cost(logging, result, None, NOW, guardrail_cost=guardrail_cost)
assert finalized is not None
assert finalized.amount == guardrail_cost
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers.get("x-litellm-response-cost") == (
None if source == "unpriced" and guardrail_cost == 0 else str(guardrail_cost)
)
@pytest.mark.asyncio
async def test_free_model_priced_only_per_request_keeps_the_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "gpt-free-per-request-header"
monkeypatch.setitem(
litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", "input_cost_per_request": 0.0}
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="per-request-header",
function_id="per-request-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None and finalized.amount == 0.0
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers["x-litellm-response-cost"] == "0.0"
@pytest.mark.asyncio
async def test_free_off_peak_pricing_keeps_the_finalized_zero_cost_header(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "gpt-free-off-peak-header"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0},
},
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="off-peak-header",
function_id="off-peak-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
result._hidden_params["response_cost"] = 0.0
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None and finalized.amount == 0.0
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
assert headers["x-litellm-response-cost"] == "0.0"
@pytest.mark.asyncio
@pytest.mark.parametrize("mapped", [False, True], ids=["unmapped-rule", "fill-missing-rule"])
@pytest.mark.parametrize("price", [None, 0.0, 0.02], ids=["unpriced", "free", "priced"])
async def test_rule_pricing_preserves_free_headers_without_treating_capabilities_as_prices(
monkeypatch: pytest.MonkeyPatch, mapped: bool, price: float | None
) -> None:
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
set_fallback_generalizations,
)
model: Final = "gpt-rule-header-model"
previous_rules: Final = TypeAdapter(list[object]).validate_python(get_fallback_generalization_rules())
if mapped:
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
try:
set_fallback_generalizations(
[
{
"name": "header-pricing",
"pattern": model,
"fill_missing_for_providers": ["openai"],
"model_info": {
"mode": "chat",
**(
{"input_cost_per_token": price, "output_cost_per_token": price} if price is not None else {}
),
},
}
]
)
logging: Final = Logging(
model=model,
messages=[],
stream=False,
call_type="acompletion",
start_time=NOW,
litellm_call_id="rule-header",
function_id="rule-header",
)
logging.update_environment_variables(
model=model, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
result: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
finalized: Final = await finalize_response_cost(logging, result, None, NOW)
assert finalized is not None
assert finalized.amount == pytest.approx((price or 0.0) * 15)
headers: Final = TypeAdapter(dict[str, str]).validate_python(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(),
response_cost=finalized.amount,
litellm_logging_obj=logging,
)
)
assert headers.get("x-litellm-response-cost") == (None if price is None else str(finalized.amount))
finally:
set_fallback_generalizations(previous_rules)

327
tests/unit/proxy/spend_tracking/test_session_page.py Normal file
View file

@ -0,0 +1,327 @@
import itertools
import random
import re
from collections.abc import Sequence
from typing import Final, TypeAlias
import pytest
from litellm.proxy.spend_tracking.session_page import (
Session,
_parse_session_cursor,
_Read,
_ReadRow,
_Request,
_session_page,
read_session_page,
)
_Row: TypeAlias = tuple[int, str, str] # (startTime, session key, api_key)
_Ranked: TypeAlias = tuple[int, str, str] # (last activity, session key, api_key)
def _session_of(row: _Row) -> tuple[str, str]:
return row[1], row[2]
def _ranked(rows: Sequence[_Row], request: _Request) -> tuple[_Ranked, ...]:
"""The sessions in ``rows`` in page order, after the request's cursor or offset."""
by_session: Final = itertools.groupby(sorted(rows, key=_session_of), key=_session_of)
ranked: Final = tuple(
sorted(
((max(start for start, _, _ in group), key, api_key) for (key, api_key), group in by_session),
reverse=request.sort_desc,
)
)
if request.cursor is None:
return ranked[request.offset :]
cursor: Final = (int(request.cursor[0]), request.cursor[1], request.cursor[2])
return tuple(session for session in ranked if (session < cursor if request.sort_desc else session > cursor))
class _Reads:
"""Reads like the page query, whose LIMIT cuts tied rows lowest key first, the opposite of page order. It fails a
read no larger than the one before, since only growing reads end the loop."""
def __init__(self, rows: Sequence[_Row], request: _Request, every_row: bool = False) -> None:
self.newest_first: Final = tuple(sorted(rows, key=lambda row: (-row[0], row[1], row[2])))
self.request: Final = request
self.every_row: Final = every_row
self.budgets: tuple[int | None, ...] = ()
async def __call__(self, budget: int | None) -> _Read:
assert not self.budgets or (self.budgets[-1] is not None and (budget is None or budget > self.budgets[-1])), (
self.budgets,
budget,
)
self.budgets = (*self.budgets, budget)
taken: Final = self.newest_first if budget is None or self.every_row else self.newest_first[:budget]
page: Final = _ranked(taken, self.request)[: self.request.page_limit + 1] if self.request.page_limit > 0 else ()
return _Read(
candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page),
last_settled=bool(page) and page[-1][0] > taken[-1][0],
rows_read=len(taken),
session_count=len(frozenset(_session_of(row) for row in taken)),
saw_every_row=budget is None or self.every_row or len(taken) < budget,
)
def _session_rows(rng: random.Random, index: int, clock: int) -> tuple[_Row, ...]:
start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40))
return tuple((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length))
def _sessions(rng: random.Random) -> tuple[_Row, ...]:
"""Sessions of one to forty rows over a short clock, so many rows share a time."""
clock: Final = rng.choice((3, 40, 400))
return tuple(
itertools.chain.from_iterable(_session_rows(rng, index, clock) for index in range(rng.randrange(1, 120)))
)
def _random_cursor(rng: random.Random, ranked: Sequence[_Ranked]) -> tuple[str, str, str]:
at, key, api_key = ranked[rng.randrange(len(ranked))]
return f"{at:06d}", key, api_key
def _request(rng: random.Random, rows: Sequence[_Row]) -> _Request:
cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8
mode: Final = rng.choice(("first", "offset", "cursor"))
full: Final = _ranked(rows, _Request(None, 0, 0, sort_desc, None, cap))
cursor: Final = _random_cursor(rng, full) if mode == "cursor" and full else None
offset: Final = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0
return _Request(cursor, offset, min(page_size, cap - offset), sort_desc, None, cap)
@pytest.mark.parametrize("seed", range(60))
@pytest.mark.asyncio
async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed: int) -> None:
rng: Final = random.Random(seed)
rows: Final = _sessions(rng)
for request in tuple(_request(rng, rows) for _ in range(12)):
grouped = await _session_page(_Reads(rows, request, every_row=True), request)
assert await _session_page(_Reads(rows, request), request) == grouped, request
@pytest.mark.parametrize(
"rows_per_session, expected_budgets",
[
pytest.param(1, (102,), id="one-read-holds-the-cap"),
pytest.param(2, (102, 253), id="sized-by-the-first-reads-density"),
pytest.param(50, (102, 4293, 8586), id="doubles-when-the-density-misleads"),
pytest.param(500, (102, 12878, 62533), id="re-sized-by-each-reads-density"),
],
)
@pytest.mark.asyncio
async def test_reads_grow_by_the_last_reads_rows_per_session_until_one_settles_the_page(
rows_per_session: int, expected_budgets: tuple[int, ...]
) -> None:
"""150 sessions a row per second apart. A cap of 100 starts at 100 + 2 rows."""
rows: Final = tuple((index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100)
reads: Final = _Reads(rows, request)
await _session_page(reads, request)
assert reads.budgets == expected_budgets
@pytest.mark.asyncio
async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted() -> None:
"""Thirty one-row sessions share one time and the first read takes 12 of them. The page's sixth session sits at the
read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on."""
rows: Final = tuple((0, f"s{index:02d}", "k") for index in range(30))
request: Final = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, status=None, cap=10)
reads: Final = _Reads(rows, request)
page: Final = await _session_page(reads, request)
assert tuple(session.key for session in page.sessions) == ("s29", "s28", "s27", "s26", "s25")
assert reads.budgets == (12, 24, 48)
@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))])
@pytest.mark.asyncio
async def test_the_total_is_capped_only_past_the_cap(session_count: int, expected_total: tuple[int, bool]) -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(session_count))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100)
page: Final = await _session_page(_Reads(rows, request), request)
assert (page.total, page.total_is_capped) == expected_total
@pytest.mark.asyncio
async def test_a_page_number_near_the_bigint_limit_reads_once() -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500))
offset: Final = (9_300_000_000_000_000 - 1) * 1000
request: Final = _Request(
cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, status=None, cap=100
)
reads: Final = _Reads(rows, request)
page: Final = await _session_page(reads, request)
assert reads.budgets == (102,)
assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True)
@pytest.mark.parametrize(
"sort_desc, status",
[pytest.param(False, None, id="oldest-first"), pytest.param(True, "failure", id="filtered-by-outcome")],
)
@pytest.mark.asyncio
async def test_a_page_that_needs_every_row_reads_them_once(sort_desc: bool, status: str | None) -> None:
rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500))
request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=sort_desc, status=status, cap=100)
reads: Final = _Reads(rows, request)
await _session_page(reads, request)
assert reads.budgets == (None,)
def test_parse_session_cursor() -> None:
assert _parse_session_cursor(None) is None
assert _parse_session_cursor("no-delimiter") is None
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
assert _parse_session_cursor("|hashed-key|sess-1") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
"2026-08-29 10:00:00",
"sess-1",
"hashed-key",
)
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
"2026-08-29 10:00:00.123",
"sess|with|pipes",
"hashed-key",
)
def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]:
"""The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``."""
found: Final = re.search(pattern, sql)
assert found, f"{pattern!r} not found in:\n{sql}"
assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}"
return tuple(args[int(index) - 1] for index in found.groups())
class _TableDb:
"""A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row."""
def __init__(self, rows: int, sessions: int) -> None:
self.rows: Final = rows
self.sessions: Final = sessions
self.calls: tuple[tuple[str, tuple[object, ...]], ...] = ()
async def query_raw(self, query: str, *args: object) -> tuple[_ReadRow, ...]:
self.calls = (*self.calls, (query, args))
budget: Final = re.search(r"DESC LIMIT (\d+)", query)
rows_read: Final = min(int(budget.group(1)), self.rows) if budget else self.rows
return (
_ReadRow(
session_key=None,
api_key=None,
last_activity=None,
settled=None,
rows_read=rows_read,
session_count=self.sessions,
),
)
def _page_bounds(sql: str) -> tuple[int, int]:
"""The LIMIT and OFFSET of the read's sessions."""
found: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s*\) AS page", sql)
assert found, sql
return int(found.group(1)), int(found.group(2) or 0)
def _read_size(sql: str) -> int | None:
found: Final = re.search(r"DESC LIMIT (\d+)", sql)
return int(found.group(1)) if found else None
@pytest.mark.asyncio
async def test_a_read_that_fills_its_size_is_not_taken_for_every_row() -> None:
"""A read that took every row it asked for may have stopped short, so its 50 sessions are no total: the reads
grow until one takes fewer rows than it asked for."""
db: Final = _TableDb(rows=9_000, sessions=50)
page: Final = await read_session_page(
db,
where_clause="TRUE",
params=(),
page=1,
page_size=10,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert tuple(_read_size(sql) for sql, _ in db.calls) == (102, 258, 652, 1647, 4159, 10502)
assert (page.total, page.total_is_capped) == (50, False)
@pytest.mark.parametrize("page_number", [pytest.param(5, id="at-the-cap"), pytest.param(6, id="past-the-cap")])
@pytest.mark.asyncio
async def test_a_page_starting_at_or_past_the_cap_reads_once_and_shows_no_session(page_number: int) -> None:
db: Final = _TableDb(rows=1_000, sessions=150)
page: Final = await read_session_page(
db,
where_clause="TRUE",
params=(),
page=page_number,
page_size=25,
status=None,
sort_desc=True,
session_cursor=None,
cap=100,
)
assert len(db.calls) == 1, "the first read already holds more sessions than the cap"
assert _page_bounds(db.calls[0][0])[0] == 0
assert (page.total, page.total_is_capped) == (100, True)
@pytest.mark.asyncio
async def test_the_read_binds_the_filters_and_the_cursor_to_their_placeholders() -> None:
db: Final = _TableDb(rows=50, sessions=40)
where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a")
for page, page_size, session_cursor in ((1, 3, "2026-10-28 09:00:00|key-a|sess|9"), (3, 1, None)):
await read_session_page(
db,
where_clause=where,
params=params,
page=page,
page_size=page_size,
status=None,
sort_desc=True,
session_cursor=session_cursor,
cap=100,
)
(cursor_sql, cursor_args), (offset_sql, _) = db.calls
for sql, args in db.calls:
assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)), (
"every argument needs a placeholder and every placeholder an argument"
)
assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params
assert _bound(
cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)"
) == ("2026-10-28 09:00:00", "sess|9", "key-a")
assert _page_bounds(cursor_sql) == (4, 0)
assert _page_bounds(offset_sql) == (2, 2)
@pytest.mark.asyncio
async def test_a_status_filtered_read_binds_the_status_between_the_filters_and_the_cursor() -> None:
db: Final = _TableDb(rows=50, sessions=40)
await read_session_page(
db,
where_clause="model = $1",
params=("gpt-4o",),
status="failure",
page=1,
page_size=3,
sort_desc=True,
session_cursor="2026-10-28 09:00:00|key-a|sess-9",
cap=100,
)
((sql, args),) = db.calls
assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1))
assert _bound(sql, args, r"model = \$(\d+)") == ("gpt-4o",)
assert _bound(sql, args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",)
assert _bound(sql, args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)") == (
"2026-10-28 09:00:00",
"sess-9",
"key-a",
)

103
tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -5,7 +5,7 @@ import hashlib
import json
import re
import sqlite3
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from datetime import timezone
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
@ -7151,37 +7151,19 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch):
app.dependency_overrides.pop(ps.user_api_key_auth, None)
def test_parse_session_cursor():
from litellm.proxy.spend_tracking.spend_management_endpoints import (
_parse_session_cursor,
)
assert _parse_session_cursor(None) is None
assert _parse_session_cursor("no-delimiter") is None
assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None
assert _parse_session_cursor("|hashed-key|sess-1") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None
assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == (
"2026-08-29 10:00:00",
"sess-1",
"hashed-key",
)
assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == (
"2026-08-29 10:00:00.123",
"sess|with|pipes",
"hashed-key",
)
SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
def _session_grouped_mock_prisma(session_page_rows, session_total, representative_rows):
def _session_grouped_mock_prisma(
session_page_rows: Sequence[Mapping[str, object]],
session_total: int,
representative_rows: Sequence[Mapping[str, object]],
) -> MagicMock:
"""Mock prisma dispatching the raw queries the keyset grouped path emits."""
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": session_total}]
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "rows_read" in sql_query:
return _session_read_rows(session_page_rows, session_total)
if "DISTINCT ON" in sql_query:
return representative_rows
if "COALESCE(SUM(spend)" in sql_query:
@ -7199,7 +7181,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ
"session_models": ["gpt-4o"],
}
]
return session_page_rows
raise AssertionError(f"unexpected query:\n{sql_query}")
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -7228,23 +7210,33 @@ def _session_page_row(session_key, last_activity):
return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity}
def _session_grouped_paginating_prisma(sessions, counted_total=None):
"""Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for."""
def _session_read_rows(
page_rows: Sequence[Mapping[str, object]], session_count: int
) -> tuple[Mapping[str, object], ...]:
"""The read query's rows for ``page_rows``, as if the read held every matching row."""
summary: Final = {"rows_read": 0, "session_count": session_count}
if not page_rows:
return ({"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary},)
return tuple({**row, "settled": True, **summary} for row in page_rows)
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}]
def _session_grouped_paginating_prisma(sessions: Sequence[tuple[str, str]]) -> MagicMock:
"""Mock prisma serving the grouped page read out of ``sessions``, honoring the LIMIT and OFFSET it asks for."""
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "DISTINCT ON" in sql_query:
return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]]
if "COALESCE(SUM(spend)" in sql_query:
return []
bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query)
limit = params[int(bounds.group(1)) - 1]
offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0
return [
_session_page_row(session_key, last_activity)
for session_key, last_activity in sessions[offset : offset + limit]
]
bounds: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s+\) AS page", sql_query)
limit, offset = int(bounds.group(1)), int(bounds.group(2) or 0)
return _session_read_rows(
[
_session_page_row(session_key, last_activity)
for session_key, last_activity in sessions[offset : offset + limit]
],
len(sessions),
)
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -7296,20 +7288,15 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
assert data["data"][1]["session_total_count"] == 1
emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list]
assert len(emitted) == 3, "one read picks the page and counts it, then the representatives and their totals"
page_query_sql = emitted[0][0]
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql
assert "OFFSET" not in page_query_sql
assert "HAVING" not in page_query_sql
assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more"
assert "LIMIT 3 " in page_query_sql, "the read fetches page_size + 1 sessions to detect has_more"
count_sql = emitted[1][0]
assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
rep_call = emitted[2]
rep_call: Final = emitted[1]
assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0]
assert (
f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
) in rep_call[0]
assert rep_call[-2] == ["sess-1", "req-solo"]
assert rep_call[-1] == ["hashed-key", "hashed-key"]
@ -7319,7 +7306,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc
@pytest.mark.asyncio
async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch):
"""A session_cursor becomes a HAVING keyset predicate instead of an OFFSET."""
"""A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET."""
page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")]
reps = [_session_representative_row("req-2", "sess-2")]
mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps)
@ -7352,7 +7339,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat
page_query_call = mock_prisma.db.query_raw.await_args_list[0]
page_query_sql = page_query_call.args[0]
assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql
assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql
assert "OFFSET" not in page_query_sql
cursor_index = page_query_call.args.index("2026-08-29 09:00:00")
assert page_query_call.args[cursor_index : cursor_index + 3] == (
@ -7403,9 +7390,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor(
@pytest.mark.asyncio
async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch):
"""A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted."""
"""A page that runs out of sessions has read every matching row, so that one read gives the total."""
sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10))
mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999)
mock_prisma: Final = _session_grouped_paginating_prisma(sessions)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma)
monkeypatch.setattr(
"litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe",
@ -7429,10 +7416,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie
)
assert response.status_code == 200, response.text
data = response.json()
assert data["total"] == 10, "the count query's 999 would have won if it had been asked"
assert data["total"] == 10
assert data["total_is_capped"] is False
assert data["total_pages"] == 1
assert len(data["data"]) == 10
reads: Final = tuple(call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0])
assert len(reads) == 1, "a page that ends the list is not read again for its total"
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
@ -7517,7 +7506,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty(
assert past_cap_data["data"] == []
assert past_cap_data["has_more"] is False
assert past_cap_data["total"] == cap
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window"
assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window"
finally:
app.dependency_overrides.pop(ps.user_api_key_auth, None)
@ -7820,14 +7809,14 @@ def test_session_representative_uses_completed_agent_outcome(parent_status, chil
)
result = connection.execute(
"SELECT request_id, status FROM logs ORDER BY "
+ spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
+ spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
+ " LIMIT 1"
).fetchone()
assert result == ("parent", expected)
connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'")
fallback = connection.execute(
"SELECT request_id, status FROM logs ORDER BY "
+ spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL
+ spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL
+ " LIMIT 1"
).fetchone()
assert fallback == ("llm", "success")

40
tests/unit/proxy/spend_tracking/test_spend_query_optimization.py
View file

@ -6,7 +6,9 @@ GitHub Issue: #17487
"""
import datetime
from collections.abc import Mapping, Sequence
from datetime import timezone
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -508,20 +510,10 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch):
@pytest.mark.asyncio
async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
"""
With group_by_session=true, /spend/logs/ui must page and count SESSIONS,
not raw calls: the page is one representative row per session (DISTINCT
ON the session group key, preferring non-MCP calls, newest first), the
bounded count counts groups, and on the default startTime sort the page is
selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the
UI collapses a server page of N calls into fewer visible rows while the
footer still claims N (issue #38060).
"""
"""With group_by_session=true, /spend/logs/ui pages and counts sessions, not calls, one representative row per
session, so the footer's total matches the rows shown (issue #38060)."""
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.spend_tracking.spend_management_endpoints import (
SPEND_LOGS_PAGINATION_COUNT_CAP,
ui_view_spend_logs,
)
from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs
group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key"
session_rows = [
@ -533,12 +525,12 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
{"request_id": "req-2", "api_key": "k", "metadata": "{}", "session_id": None},
]
async def mock_query_raw(sql_query, *params):
if "COUNT(*) AS total_count" in sql_query:
return [{"total_count": 60}]
async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]:
if "rows_read" in sql_query:
return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows]
if "AS session_representatives" in sql_query:
return representative_rows
return session_rows
return []
mock_prisma = MagicMock()
mock_prisma.db = MagicMock()
@ -565,22 +557,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch):
user_api_key_dict=auth,
group_by_session=True,
session_cursor=None,
status_filter=None,
)
emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list]
page_sql = emitted[0][0]
assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}"
assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected"
assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more"
assert "LIMIT 51 " in page_sql, "the page read fetches page_size + 1 sessions to detect has_more"
count_call = emitted[1]
count_sql = count_call[0]
assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}"
assert "COUNT(*) OVER ()" not in count_sql
assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded"
assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1
rep_sql = emitted[2][0]
rep_sql: Final = emitted[1][0]
assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}"
assert (
f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, "

73
tests/unit/proxy/test_model_deprecations_endpoint.py
View file

@ -64,12 +64,79 @@ def test_should_bucket_configured_models_by_urgency(authenticated_client):
def test_should_rebucket_with_warn_within_days_override(authenticated_client):
response = authenticated_client.get(
"/v1/model/deprecations", params={"warn_within_days": 40000}
)
response = authenticated_client.get("/v1/model/deprecations", params={"warn_within_days": 40000})
assert response.status_code == 200
payload = response.json()
assert [m["model_name"] for m in payload["imminent"]] == ["future-alias"]
assert payload["upcoming"] == []
assert payload["warn_within_days"] == 40000
@pytest.fixture
def client_with_cost_map_successors(monkeypatch):
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-9876"
)
monkeypatch.setattr(
litellm,
"model_cost",
{
"retiring-model": {
"deprecation_date": "2020-01-01",
"litellm_provider": "openai",
"successor_model": "fresh-model",
},
"orphan-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"},
},
)
router = route_entry_points_to_stubs(MagicMock())
router.get_model_list.return_value = [
{
"model_name": "retiring-alias",
"litellm_params": {"model": "retiring-model"},
"model_info": {"id": "1"},
},
{
"model_name": "retiring-overridden",
"litellm_params": {"model": "retiring-model"},
"model_info": {"id": "2", "successor_model": "my-own-pick"},
},
{
"model_name": "shared-name",
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
"model_info": {"id": "4"},
},
{
"model_name": "shared-name",
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
"model_info": {"id": "5", "successor_model": "my-own-pick"},
},
{
"model_name": "dated-by-hand",
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
"model_info": {"id": "6", "deprecation_date": "2020-06-01"},
},
{
"model_name": "orphan-alias",
"litellm_params": {"model": "orphan-model"},
"model_info": {"id": "3"},
},
]
monkeypatch.setattr(proxy_server, "llm_router", router)
yield client
app.dependency_overrides.pop(user_api_key_auth, None)
def test_cost_map_successor_is_reported_unless_the_deployment_names_its_own(client_with_cost_map_successors):
response = client_with_cost_map_successors.get("/model/deprecations")
assert response.status_code == 200, response.text
successors = {m["model_name"]: m["successor_model"] for m in response.json()["deprecated"]}
assert successors == {
"retiring-alias": "fresh-model",
"retiring-overridden": "my-own-pick",
"shared-name": "my-own-pick",
"dated-by-hand": "fresh-model",
"orphan-alias": None,
}

32
tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
View file

@ -1794,6 +1794,38 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database
assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database
@pytest.mark.asyncio
async def test_proxy_startup_reads_fips_mode_from_the_config_file(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
from litellm.proxy.common_utils.fips import FipsModeError
from litellm.proxy.proxy_server import proxy_startup_event
_boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": True})
monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: False)
with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode is on but OpenSSL does not enforce FIPS"):
async with proxy_startup_event(FastAPI()):
pass
@pytest.mark.asyncio
async def test_proxy_startup_refuses_a_fips_mode_reference_to_an_unset_variable(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
from litellm.proxy.common_utils.fips import FipsModeError
from litellm.proxy.proxy_server import proxy_startup_event
_boot_with_general_settings(
monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": "os.environ/FIPS_MODE_NOT_SET"}
)
monkeypatch.delenv("FIPS_MODE_NOT_SET", raising=False)
with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode has no value"):
async with proxy_startup_event(FastAPI()):
pass
class _DatabaseWithOneStoredCredential:
def __init__(self, ciphertext):
self._ciphertext = ciphertext

21
tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
View file

@ -18,6 +18,27 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
client = TestClient(app)
@pytest.mark.asyncio
async def test_sso_settings_without_prisma_reports_database_not_connected(monkeypatch: pytest.MonkeyPatch) -> None:
import runpy
import sys
from fastapi import HTTPException
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.ui_crud_endpoints import proxy_setting_endpoints
monkeypatch.setattr(proxy_server, "prisma_client", None)
with patch.dict(sys.modules, {"prisma": None, "prisma.errors": None}):
endpoints: Final = runpy.run_path(proxy_setting_endpoints.__file__)
with pytest.raises(HTTPException) as error:
await endpoints["update_sso_settings"](SSOConfig(), UserAPIKeyAuth())
assert error.value.status_code == 500
assert error.value.detail == {"error": "Database not connected. Please connect a database."}
@pytest.mark.asyncio
@pytest.mark.parametrize("config_value,stored_value,expected", [
(None, None, False), (None, False, False), (None, True, True),

54
tests/unit/router_strategy/test_lowest_tpm_rpm_v2.py
View file

@ -2,7 +2,9 @@
# This tests the router's ability to pick deployment with lowest tpm using 'usage-based-routing-v2-v2'
import asyncio
import base64
import importlib
import io
import os
import time
import traceback
@ -15,6 +17,7 @@ from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outco
from unittest.mock import patch
import pytest
from PIL import Image
import litellm
from litellm import Router
@ -23,7 +26,7 @@ from litellm.caching.caching import DualCache
from litellm.router_strategy.lowest_tpm_rpm_v2 import (
LowestTPMLoggingHandler_v2 as LowestTPMLoggingHandler,
)
from litellm.types.router import DeploymentTypedDict
from litellm.types.router import DeploymentTypedDict, RouterRateLimitError
from litellm.types.utils import StandardLoggingPayload
from litellm.utils import get_utc_datetime
@ -347,6 +350,55 @@ def test_router_skip_rate_limited_deployments():
print(f"An exception occurred! {str(e)}")
def _embedding_image_uri() -> str:
image: Final = io.BytesIO()
Image.new("RGB", (1024, 1024)).save(image, format="PNG", compress_level=0)
return f"data:image/png;base64,{base64.b64encode(image.getvalue()).decode()}"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"routing_strategy",
["usage-based-routing", "usage-based-routing-v2", "latency-based-routing", "cost-based-routing"],
)
@pytest.mark.parametrize(
("litellm_params", "model_info", "admitted"),
[
({"model": "vertex_ai/gemini-embedding-2"}, {}, True),
(
{"model": "openai/team-embedder", "api_base": "https://embedder.invalid"},
{"base_model": "vertex_ai/gemini-embedding-2"},
True,
),
({"model": "text-embedding-3-small"}, {}, False),
],
ids=["image-model", "image-base-model", "text-model"],
)
async def test_deployment_tpm_counts_embedding_image_as_media_only_on_image_models(
routing_strategy: str, litellm_params: Dict[str, str], model_info: Dict[str, str], admitted: bool
) -> None:
uri: Final = _embedding_image_uri()
assert len(uri) // 4 > 75_000
router: Final = Router(
model_list=[
{
"model_name": "embedder",
"litellm_params": {**litellm_params, "tpm": 75_000},
"model_info": {**model_info, "id": f"embedder-{index}"},
}
for index in range(2)
],
routing_strategy=routing_strategy,
)
if not admitted:
with pytest.raises((litellm.RateLimitError, RouterRateLimitError)):
await router.select("embedder", {}, input=[uri])
return
deployment: Final = await router.select("embedder", {}, input=[uri])
assert deployment["model_info"]["id"] in ("embedder-0", "embedder-1")
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_multiple_potential_deployments(sync_mode):

3
tests/unit/test_get_blog_posts.py
View file

@ -50,11 +50,10 @@ def reset_blog_posts_cache():
GetBlogPosts._last_fetch_time = 0.0
def test_load_local_blog_posts_returns_posts_without_tags():
def test_load_local_blog_posts_returns_posts():
posts = GetBlogPosts.load_local_blog_posts()
assert len(posts) > 0
assert all(isinstance(post, BlogPost) for post in posts)
assert posts[0].tags == ()
def test_parse_rss_to_posts():

72
tests/unit/test_router/test_router.py
View file

@ -5,6 +5,7 @@ import functools
import gc
import itertools
import io
import base64
import json
import linecache
import logging
@ -8174,6 +8175,77 @@ def test_pre_call_checks_count_a_group_mixing_in_another_vendor_by_its_own_name(
)
_EMBEDDING_CONTEXT_WINDOW: Final = 8_192
_EMBEDDING_IMAGE_INPUT: Final = f"data:image/png;base64,{base64.b64encode(bytes(range(256)) * 1024).decode()}"
def _embedding_context_router(backend: str, window: int = _EMBEDDING_CONTEXT_WINDOW) -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": "embedder",
"litellm_params": {"model": backend, "api_key": "sk-test"},
"model_info": {"max_input_tokens": window},
}
],
enable_pre_call_checks=True,
)
async def test_context_window_pre_call_check_counts_an_embedding_image_as_media_on_an_image_backend():
router: Final = _embedding_context_router("vertex_ai/gemini-embedding-2")
deployments: Final = router.registry().deployments("embedder")
kept: Final = router._pre_call_checks(model="embedder", healthy_deployments=deployments, input=_EMBEDDING_IMAGE_INPUT)
counted: Final = await router._acount_pre_call_check_tokens(
model="embedder", healthy_deployments=deployments, messages=None, input=_EMBEDDING_IMAGE_INPUT, request_kwargs=None
)
assert len(kept) == 1
assert counted is not None and 0 < counted < _EMBEDDING_CONTEXT_WINDOW
with pytest.raises(litellm.ContextWindowExceededError):
router._pre_call_checks(model="embedder", healthy_deployments=deployments, input="word " * 20_000)
@pytest.mark.parametrize(
("backend", "shape"),
[
("vertex_ai/gemini-embedding-2", "pdf"),
("vertex_ai/gemini-embedding-2", "pdf_file_block"),
("vertex_ai/gemini-embedding-2", "two_pdfs"),
("bedrock/amazon.titan-embed-image-v1", "image"),
("bedrock/amazon.titan-embed-image-v1", "two_images"),
],
)
def test_context_window_pre_call_check_leaves_embedding_media_to_the_backend(backend: str, shape: str):
window: Final = litellm.get_model_info(backend)["max_input_tokens"]
assert window is not None
router: Final = _embedding_context_router(backend, window)
pdf: Final = f"data:application/pdf;base64,{base64.b64encode(bytes(window * 8)).decode()}"
inputs: Final = {
"pdf": pdf,
"pdf_file_block": [{"type": "file", "file": {"file_data": pdf}}],
"two_pdfs": [pdf, pdf],
"image": _EMBEDDING_IMAGE_INPUT,
"two_images": [_EMBEDDING_IMAGE_INPUT, _EMBEDDING_IMAGE_INPUT],
}
kept: Final = router._pre_call_checks(
model="embedder", healthy_deployments=router.registry().deployments("embedder"), input=inputs[shape]
)
assert len(kept) == 1
def test_context_window_pre_call_check_counts_an_embedding_image_as_text_on_a_text_only_backend():
router: Final = _embedding_context_router("text-embedding-3-small")
with pytest.raises(litellm.ContextWindowExceededError):
router._pre_call_checks(
model="embedder", healthy_deployments=router.registry().deployments("embedder"), input=_EMBEDDING_IMAGE_INPUT
)
async def test_acount_pre_call_check_tokens_counts_a_group_of_one_openai_model_with_that_models_tokenizer():
router: Final = _context_limited_router("openai/gpt-6.1-sol")

2
tests/unit/test_utils.py
View file

@ -1026,6 +1026,8 @@ _COST_MAP_ROW_SCHEMA: Final = {
"cache_read_input_image_token_cost": {"type": "number"},
"audio_transcription_config": {"type": "string"},
"deprecation_date": {"type": "string"},
"successor_model": {"type": "string"},
"deprecation_source": {"type": "string"},
"input_cost_per_audio_per_second": {"type": "number"},
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
"google_maps_grounding_cost_per_query": {"type": "number"},

475
ui/litellm-dashboard/package-lock.json generated
View file

@ -23,6 +23,7 @@
"@types/papaparse": "5.5.2",
"class-variance-authority": "0.7.1",
"clsx": "^2.1.1",
"cmdk": "1.1.1",
"date-fns": "^4.4.0",
"dayjs": "1.11.19",
"i18next": "26.5.11",
@ -2656,6 +2657,323 @@
"dev": true,
"license": "MIT"
},
"node_modules/@radix-ui/primitive": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.2.0.tgz",
"integrity": "sha512-KABY+VMIRPJL+Vyzv/335FEhXXD/xDv8BMb1kl48/aXJZMH0LtHCTx5ga5RnxaEP3nZHtmTLf1P9vanmFX46bw==",
"license": "MIT"
},
"node_modules/@radix-ui/react-compose-refs": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz",
"integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-context": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.3.0.tgz",
"integrity": "sha512-v5AYEocMISkbJ+1fTXzZpiFv7NRr3D2WiYbLIKf+FkmJ8VkP6d503eV2n34MFussHXY1pcoq3yfJiQH77dc4Mg==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-dialog": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.3.1.tgz",
"integrity": "sha512-+0E4qvVTwjAWra9sEhJ+9BTsvHE00zkUuNw1VGh7S+Gc13dfYk+S1PVQdchWDj+XYHhAai7Z7IpzXVDCGsDplg==",
"license": "MIT",
"dependencies": {
"@radix-ui/primitive": "1.2.0",
"@radix-ui/react-compose-refs": "1.1.5",
"@radix-ui/react-context": "1.3.0",
"@radix-ui/react-dismissable-layer": "1.2.1",
"@radix-ui/react-focus-guards": "1.2.0",
"@radix-ui/react-focus-scope": "1.3.1",
"@radix-ui/react-id": "1.1.4",
"@radix-ui/react-portal": "1.2.0",
"@radix-ui/react-presence": "1.1.11",
"@radix-ui/react-primitive": "2.2.0",
"@radix-ui/react-slot": "1.5.0",
"@radix-ui/react-use-controllable-state": "1.3.0",
"@radix-ui/react-use-layout-effect": "1.1.4",
"aria-hidden": "^1.2.4",
"react-remove-scroll": "^2.7.2"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-dismissable-layer": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.2.1.tgz",
"integrity": "sha512-KIfDcAsx+oVylFZ4xryVKusJUrLLhTQmfnJW4kwk4Km5myvhHCrj04hnWWUC166+xD4DHdH8tQGylwYX1h6+Pg==",
"license": "MIT",
"dependencies": {
"@radix-ui/primitive": "1.2.0",
"@radix-ui/react-compose-refs": "1.1.5",
"@radix-ui/react-primitive": "2.2.0",
"@radix-ui/react-use-callback-ref": "1.1.5",
"@radix-ui/react-use-effect-event": "0.0.5"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-focus-guards": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.2.0.tgz",
"integrity": "sha512-MPm5R0iRGUDrDnPNxmLBkv+X7pN4XKzt+MbWJEuJQsH3NFH4omrbpXpucyYCCKTSt+na/ZxGhEWQrWTZ0285cA==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-focus-scope": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.3.1.tgz",
"integrity": "sha512-ugeMhbXt4y5S2T1GnIqbWDKEN962PDwnX1fN0eKOOgf5f63Dh2mn8F1FUySlgW0iF6GiVsRF32DzuDIWTUyQJg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-compose-refs": "1.1.5",
"@radix-ui/react-primitive": "2.2.0",
"@radix-ui/react-use-callback-ref": "1.1.5"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-id": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz",
"integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-portal": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.2.0.tgz",
"integrity": "sha512-StX7xXnaRU41VvFtMm9zlhVGz8XF6GmWWTXftkQaY+tqVkbs/R9UmAp5mLzHHapk5hM9RMZQvO/3usyIP/g42g==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-primitive": "2.2.0",
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-presence": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.11.tgz",
"integrity": "sha512-Z7/atlkab36Z8hm+V8zYc79r0W+tiOB1o0tun1By94JyadkekLQmtEAnBSP9XKXBNQsStoStS4+gX10JpS5akA==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-primitive": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.2.0.tgz",
"integrity": "sha512-HCTSPMNqv55Qk0IX4geJbpyXazmqulcimwuyjxTkKi29v+QZbG2Zho9BnbBq4YS2PF+KlNb2CXoLz66uUmblkg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-slot": "1.5.0"
},
"peerDependencies": {
"@types/react": "*",
"@types/react-dom": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
},
"@types/react-dom": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-slot": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.5.0.tgz",
"integrity": "sha512-tyYd9ddPXAysAbdbD9eYxgwpCSv6f3PncqVSNKQFSR5lcJVLz3cb/pANDoYrLiHBpXv7skortyUgGUWsYuPPFQ==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-compose-refs": "1.1.5"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-use-callback-ref": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.5.tgz",
"integrity": "sha512-Y5GBf4NKeADoXI1geiMlMuMoURT3k9MizzzKhGb/m55H94aGigZ01PIpkvsO6lML13BA6XYAXSJ6gza7vTn5nw==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-use-controllable-state": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.3.0.tgz",
"integrity": "sha512-TAcBI7bw2ljYCbv4+zuTSd6zICu8pRt6SDN2vJ/Z06K2MXh6I9WYFe/A9ovlFDtR5HVHGm5iDxDoABm/++dbKg==",
"license": "MIT",
"dependencies": {
"@radix-ui/primitive": "1.2.0",
"@radix-ui/react-use-effect-event": "0.0.5",
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-use-effect-event": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz",
"integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-use-layout-effect": "1.1.4"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@radix-ui/react-use-layout-effect": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz",
"integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==",
"license": "MIT",
"peerDependencies": {
"@types/react": "*",
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/@redocly/ajv": {
"version": "8.11.2",
"resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.2.tgz",
@ -3939,7 +4257,7 @@
"version": "19.2.4",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.4.tgz",
"integrity": "sha512-Bsc+QHgp+P/F02XDzNCY9jnZNCUuLki36KT7VKrTXXLdHf+vHMNZnW1rVu5DNW/rCK+fya3DATySbLM4yhtKUw==",
"dev": true,
"devOptional": true,
"license": "MIT",
"peerDependencies": {
"@types/react": "^19.2.0"
@ -4757,6 +5075,18 @@
"dev": true,
"license": "Python-2.0"
},
"node_modules/aria-hidden": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz",
"integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==",
"license": "MIT",
"dependencies": {
"tslib": "^2.0.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/aria-query": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
@ -5308,6 +5638,22 @@
"node": ">=6"
}
},
"node_modules/cmdk": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz",
"integrity": "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==",
"license": "MIT",
"dependencies": {
"@radix-ui/react-compose-refs": "^1.1.1",
"@radix-ui/react-dialog": "^1.1.6",
"@radix-ui/react-id": "^1.1.0",
"@radix-ui/react-primitive": "^2.0.2"
},
"peerDependencies": {
"react": "^18 || ^19 || ^19.0.0-rc",
"react-dom": "^18 || ^19 || ^19.0.0-rc"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@ -5746,6 +6092,12 @@
"node": ">=8"
}
},
"node_modules/detect-node-es": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
"integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==",
"license": "MIT"
},
"node_modules/devlop": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz",
@ -6959,6 +7311,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-nonce": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz",
"integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
@ -10769,6 +11130,53 @@
}
}
},
"node_modules/react-remove-scroll": {
"version": "2.7.2",
"resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz",
"integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==",
"license": "MIT",
"dependencies": {
"react-remove-scroll-bar": "^2.3.7",
"react-style-singleton": "^2.2.3",
"tslib": "^2.1.0",
"use-callback-ref": "^1.3.3",
"use-sidecar": "^1.1.3"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/react-remove-scroll-bar": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz",
"integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==",
"license": "MIT",
"dependencies": {
"react-style-singleton": "^2.2.2",
"tslib": "^2.0.0"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/react-resizable-panels": {
"version": "4.14.1",
"resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-4.14.1.tgz",
@ -10779,6 +11187,28 @@
"react-dom": "^18.0.0 || ^19.0.0"
}
},
"node_modules/react-style-singleton": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz",
"integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==",
"license": "MIT",
"dependencies": {
"get-nonce": "^1.0.0",
"tslib": "^2.0.0"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/react-syntax-highlighter": {
"version": "15.6.6",
"resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz",
@ -12408,6 +12838,49 @@
"dev": true,
"license": "MIT"
},
"node_modules/use-callback-ref": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz",
"integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==",
"license": "MIT",
"dependencies": {
"tslib": "^2.0.0"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/use-sidecar": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz",
"integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==",
"license": "MIT",
"dependencies": {
"detect-node-es": "^1.1.0",
"tslib": "^2.0.0"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"@types/react": "*",
"react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc"
},
"peerDependenciesMeta": {
"@types/react": {
"optional": true
}
}
},
"node_modules/use-sync-external-store": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz",

1
ui/litellm-dashboard/package.json
View file

@ -40,6 +40,7 @@
"@types/papaparse": "5.5.2",
"class-variance-authority": "0.7.1",
"clsx": "^2.1.1",
"cmdk": "1.1.1",
"date-fns": "^4.4.0",
"dayjs": "1.11.19",
"i18next": "26.5.11",

63
ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx
View file

@ -1,5 +1,7 @@
import { useAccessGroupDetails } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails";
import { AccessGroupResponse } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups";
import { useUserLookup } from "@/app/(dashboard)/hooks/users/useUsers";
import { DEFAULT_PROXY_ADMIN_USER_ID } from "@/utils/sentinels";
import { screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { beforeEach, describe, expect, it, vi } from "vitest";
@ -7,6 +9,7 @@ import { renderWithProviders } from "../../../../../tests/test-utils";
import { AccessGroupDetail } from "./AccessGroupsDetailsPage";
vi.mock("@/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails");
vi.mock("@/app/(dashboard)/hooks/users/useUsers");
vi.mock("next/navigation", () => ({ useRouter: () => ({ push: vi.fn() }) }));
vi.mock("./AccessGroupsModal/AccessGroupEditModal", () => ({
AccessGroupEditModal: ({ visible, onCancel }: { visible: boolean; onCancel: () => void }) =>
@ -18,6 +21,7 @@ vi.mock("./AccessGroupsModal/AccessGroupEditModal", () => ({
}));
const mockUseAccessGroupDetails = vi.mocked(useAccessGroupDetails);
const mockUseUserLookup = vi.mocked(useUserLookup);
const baseMockReturnValue = {
data: undefined,
@ -89,6 +93,7 @@ describe("AccessGroupDetail", () => {
...baseMockReturnValue,
data: createMockAccessGroup(),
} as ReturnType<typeof useAccessGroupDetails>);
mockUseUserLookup.mockReturnValue({ data: null } as ReturnType<typeof useUserLookup>);
});
it("should render the component", () => {
@ -144,6 +149,64 @@ describe("AccessGroupDetail", () => {
expect(screen.getByText("A test access group")).toBeInTheDocument();
});
describe("user references", () => {
it("shows a user's alias as a linked name and reveals the id in a tooltip", async () => {
const user = userEvent.setup();
mockUseUserLookup.mockReturnValue({
data: { user_alias: "Alice", user_email: "alice@example.com" },
} as ReturnType<typeof useUserLookup>);
renderWith({ created_by: "user-123" });
const link = screen.getByRole("link", { name: "Alice" });
expect(link).toHaveAttribute("href", expect.stringContaining("/users?user=user-123"));
expect(screen.queryByText("user-123")).not.toBeInTheDocument();
await user.hover(link);
expect(await screen.findByText("user-123")).toBeInTheDocument();
});
it("shows the email when the user has no alias", () => {
mockUseUserLookup.mockReturnValue({
data: { user_alias: null, user_email: "alice@example.com" },
} as ReturnType<typeof useUserLookup>);
renderWith({ updated_by: "user-456" });
expect(screen.getByRole("link", { name: "alice@example.com" })).toHaveAttribute(
"href",
expect.stringContaining("/users?user=user-456"),
);
});
it("falls back to the email when the alias contains only whitespace", () => {
mockUseUserLookup.mockReturnValue({
data: { user_alias: " ", user_email: "alice@example.com" },
} as ReturnType<typeof useUserLookup>);
renderWith({ created_by: "user-789" });
expect(screen.getByRole("link", { name: "alice@example.com" })).toHaveAttribute(
"href",
expect.stringContaining("/users?user=user-789"),
);
});
it("links the user id when lookup has no user details", () => {
renderWith({ updated_by: "missing-user" });
expect(screen.getByRole("link", { name: "missing-user" })).toHaveAttribute(
"href",
expect.stringContaining("/users?user=missing-user"),
);
});
it("shows the default admin badge without looking up its user id", () => {
renderWith({ created_by: DEFAULT_PROXY_ADMIN_USER_ID });
expect(screen.getByText("Default Proxy Admin")).toBeInTheDocument();
expect(mockUseUserLookup).toHaveBeenCalledWith(null);
});
});
it("should display em dash when description is empty", () => {
renderWith({ description: null });

6
ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx
View file

@ -12,9 +12,9 @@ import {
} from "lucide-react";
import { useState } from "react";
import { useTranslation } from "react-i18next";
import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag";
import { BadgeLink } from "@/components/shared/BadgeLink";
import CopyButton from "@/components/shared/CopyButton";
import { UserReference } from "@/components/shared/UserReference";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardAction, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
@ -190,7 +190,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
{accessGroup.created_by && (
<>
<span>{t("accessGroups.detail.by")}</span>
<DefaultProxyAdminTag userId={accessGroup.created_by} />
<UserReference userId={accessGroup.created_by} />
</>
)}
</dd>
@ -200,7 +200,7 @@ export function AccessGroupDetail({ accessGroupId, onBack }: AccessGroupDetailPr
{accessGroup.updated_by && (
<>
<span>{t("accessGroups.detail.by")}</span>
<DefaultProxyAdminTag userId={accessGroup.updated_by} />
<UserReference userId={accessGroup.updated_by} />
</>
)}
</dd>

62
ui/litellm-dashboard/src/app/(dashboard)/hooks/users/useUsers.test.ts
View file

@ -3,10 +3,12 @@ import { renderHook, waitFor } from "@testing-library/react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import React, { ReactNode } from "react";
import { useInfiniteUsers, useUserEmailLookup, useUserLookup } from "./useUsers";
import { userListCall } from "@/components/networking";
import type { UserListResponse } from "@/components/networking";
import { userGetInfoV2, userListCall } from "@/components/networking";
import type { UserInfoV2Response, UserListResponse } from "@/components/networking";
import { ApiError } from "@/lib/http/client";
vi.mock("@/components/networking", () => ({
userGetInfoV2: vi.fn(),
userListCall: vi.fn(),
}));
@ -56,6 +58,23 @@ const buildUserListResponse = (page: number, totalPages: number, userCount = 2):
})),
});
const buildUserInfoV2Response = (userId: string): UserInfoV2Response => ({
user_id: userId,
user_email: "alice@example.com",
user_alias: "Alice",
user_role: "Internal User",
spend: 0,
max_budget: null,
models: [],
budget_duration: null,
budget_reset_at: null,
metadata: null,
created_at: null,
updated_at: null,
sso_user_id: null,
teams: [],
});
describe("useInfiniteUsers", () => {
let queryClient: QueryClient;
@ -299,22 +318,47 @@ describe("useUserLookup", () => {
React.createElement(QueryClientProvider, { client: queryClient }, children);
it("fetches exactly the requested user by id", async () => {
vi.mocked(userListCall).mockResolvedValue(buildUserListResponse(1, 1, 1));
const exactUser = buildUserInfoV2Response("user-1-0");
vi.mocked(userGetInfoV2).mockResolvedValue(exactUser);
const { result } = renderHook(() => useUserLookup("user-1-0"), { wrapper });
await waitFor(() => expect(result.current.isSuccess).toBe(true));
expect(userListCall).toHaveBeenCalledWith("test-access-token", ["user-1-0"], 1, 1);
expect(userGetInfoV2).toHaveBeenCalledWith("test-access-token", "user-1-0");
expect(userListCall).not.toHaveBeenCalled();
expect(result.current.data).toMatchObject(exactUser);
expect(result.current.data?.user_id).toBe("user-1-0");
});
it("resolves to null when the proxy returns a different user than requested", async () => {
vi.mocked(userListCall).mockResolvedValue(buildUserListResponse(1, 1, 1));
it("resolves to no user when user info returns a different user id", async () => {
vi.mocked(userGetInfoV2).mockResolvedValue(buildUserInfoV2Response("a+b"));
const { result } = renderHook(() => useUserLookup("someone-else"), { wrapper });
const { result } = renderHook(() => useUserLookup("a b"), { wrapper });
await waitFor(() => expect(result.current.isSuccess).toBe(true));
expect(result.current.data).toBeNull();
expect(userListCall).not.toHaveBeenCalled();
});
it("resolves to no user when the user info endpoint returns a 404", async () => {
vi.mocked(userGetInfoV2).mockRejectedValue(new ApiError("Not Found", 404, {}));
const { result } = renderHook(() => useUserLookup("bob"), { wrapper });
await waitFor(() => expect(result.current.isSuccess).toBe(true));
expect(result.current.data).toBeNull();
expect(userListCall).not.toHaveBeenCalled();
});
it("rethrows errors other than 404", async () => {
const serverError = new ApiError("Internal Server Error", 500, {});
vi.mocked(userGetInfoV2).mockRejectedValue(serverError);
const { result } = renderHook(() => useUserLookup("user-1-0"), { wrapper });
await waitFor(() => expect(result.current.isError).toBe(true));
expect(result.current.error).toBe(serverError);
expect(userListCall).not.toHaveBeenCalled();
});
it("does not query without a user id", async () => {
@ -322,7 +366,7 @@ describe("useUserLookup", () => {
await new Promise((resolve) => setTimeout(resolve, 20));
expect(result.current.fetchStatus).toBe("idle");
expect(userListCall).not.toHaveBeenCalled();
expect(userGetInfoV2).not.toHaveBeenCalled();
});
it("does not query for a non-admin role", async () => {
@ -332,7 +376,7 @@ describe("useUserLookup", () => {
await new Promise((resolve) => setTimeout(resolve, 20));
expect(result.current.fetchStatus).toBe("idle");
expect(userListCall).not.toHaveBeenCalled();
expect(userGetInfoV2).not.toHaveBeenCalled();
});
});

Some files were not shown because too many files have changed in this diff Show more