fix(model-cost): read model info and capabilities from the live map through the Rust resolver

get_model_info, get_model_cost_key and supports_* resolve through one Rust
walk over the bound dict, with one name index rebuilt when (id, len)
changes. The two lru_caches, the lowercase map and the Fireworks index go,
so direct writes are seen by the next read, and a miss prints no banner.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-28 00:18:59 -07:00
parent 944d1648b9
commit 1b2c40f25f
20 changed files with 13462 additions and 748 deletions

View file

@ -1,194 +1,356 @@
use indexmap::IndexMap;
use serde::Serialize;
use serde_json::Value;
/// The fields main's `ModelInfoBase(...)` copies (`litellm/utils.py` at c7c895db45), one typed field each,
/// read from the raw row without lowering it, so building one never fails
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ModelInfoFields {
pub key: String,
pub max_tokens: Option<i64>,
pub max_input_tokens: Option<i64>,
pub max_output_tokens: Option<i64>,
pub input_cost_per_token: Option<f64>,
pub input_cost_per_token_flex: Option<f64>,
pub input_cost_per_token_priority: Option<f64>,
pub input_cost_per_token_balanced: Option<f64>,
pub input_cost_per_token_ultrafast: Option<f64>,
pub cache_creation_input_token_cost: Option<f64>,
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
pub cache_creation_input_token_cost_flex: Option<f64>,
pub cache_creation_input_token_cost_priority: Option<f64>,
pub cache_creation_input_token_cost_ultrafast: Option<f64>,
pub cache_read_input_token_cost: Option<f64>,
pub cache_read_input_audio_token_cost: Option<f64>,
pub cache_read_input_audio_token_cost_flex: Option<f64>,
pub cache_read_input_audio_token_cost_priority: Option<f64>,
pub cache_read_input_audio_token_cost_batches: Option<f64>,
pub explicit_cache_read_input_token_cost: Option<f64>,
pub prompt_cache_min_tokens: Option<i64>,
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
pub cache_read_input_token_cost_flex: Option<f64>,
pub cache_read_input_token_cost_priority: Option<f64>,
pub cache_read_input_token_cost_balanced: Option<f64>,
pub cache_read_input_token_cost_ultrafast: Option<f64>,
pub cache_read_input_token_cost_batches: Option<f64>,
pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_batches: Option<f64>,
pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
pub cache_creation_input_token_cost_above_12hr: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_flex: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_priority: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_batches: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_ultrafast: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option<f64>,
pub off_peak_pricing: Option<Value>,
pub input_cost_per_character: Option<f64>,
pub input_cost_per_token_above_128k_tokens: Option<f64>,
pub input_cost_per_token_above_200k_tokens: Option<f64>,
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
pub input_cost_per_token_above_272k_tokens: Option<f64>,
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
pub input_cost_per_token_above_512k_tokens: Option<f64>,
pub input_cost_per_query: Option<f64>,
pub cost_per_second: Option<f64>,
pub input_cost_per_second: Option<f64>,
pub input_cost_per_text_message: Option<f64>,
pub input_cost_per_audio_token: Option<f64>,
pub input_cost_per_audio_token_flex: Option<f64>,
pub input_cost_per_audio_token_priority: Option<f64>,
pub input_cost_per_image_token: Option<f64>,
pub input_cost_per_video_token: Option<f64>,
pub input_cost_per_audio_token_batches: Option<f64>,
pub input_cost_per_image_token_batches: Option<f64>,
pub input_cost_per_image: Option<f64>,
pub input_cost_per_audio_per_second: Option<f64>,
pub input_cost_per_video_per_second: Option<f64>,
pub input_cost_per_token_batches: Option<f64>,
pub input_cost_per_video_token_batches: Option<f64>,
pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
pub output_cost_per_token_batches: Option<f64>,
pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
pub output_cost_per_token: Option<f64>,
pub output_cost_per_token_flex: Option<f64>,
pub output_cost_per_token_priority: Option<f64>,
pub output_cost_per_token_balanced: Option<f64>,
pub output_cost_per_token_ultrafast: Option<f64>,
pub regional_processing_uplift_multiplier_eu: Option<f64>,
pub regional_processing_uplift_multiplier_us: Option<f64>,
pub regional_endpoint_uplift_multiplier: Option<f64>,
pub output_cost_per_audio_token: Option<f64>,
pub output_cost_per_character: Option<f64>,
pub output_cost_per_reasoning_token: Option<f64>,
pub output_cost_per_reasoning_token_flex: Option<f64>,
pub output_cost_per_reasoning_token_priority: Option<f64>,
pub output_cost_per_token_above_128k_tokens: Option<f64>,
pub output_cost_per_character_above_128k_tokens: Option<f64>,
pub output_cost_per_token_above_200k_tokens: Option<f64>,
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
pub output_cost_per_token_above_272k_tokens: Option<f64>,
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
pub output_cost_per_token_above_512k_tokens: Option<f64>,
pub output_cost_per_second: Option<f64>,
pub output_cost_per_second_1080p: Option<f64>,
pub output_cost_per_second_480p: Option<f64>,
pub output_cost_per_second_720p: Option<f64>,
pub output_cost_per_second_768p: Option<f64>,
pub output_cost_per_second_2k: Option<f64>,
pub output_cost_per_second_4k: Option<f64>,
pub output_cost_per_video_per_second: Option<f64>,
pub output_cost_per_image: Option<f64>,
pub output_cost_per_pixel: Option<f64>,
pub output_cost_per_image_token: Option<f64>,
pub output_cost_per_image_token_batches: Option<f64>,
pub output_cost_per_video_token: Option<f64>,
pub output_vector_size: Option<i64>,
pub citation_cost_per_token: Option<f64>,
pub tiered_pricing: Option<Value>,
pub litellm_provider: Option<String>,
pub mode: Option<Value>,
pub supported_endpoints: Option<Vec<String>>,
pub supports_system_messages: Option<bool>,
pub supports_response_schema: Option<bool>,
pub supports_vision: Option<bool>,
pub supports_function_calling: Option<bool>,
pub supports_parallel_function_calling: Option<bool>,
pub supports_tool_choice: Option<bool>,
pub supports_assistant_prefill: Option<bool>,
pub supports_prompt_caching: Option<bool>,
pub supports_prompt_cache_breakpoint: Option<bool>,
pub supports_thinking_cache_preservation: Option<bool>,
pub supports_audio_input: Option<bool>,
pub supports_video_input: Option<bool>,
pub supports_box_questions: Option<bool>,
pub supports_audio_output: Option<bool>,
pub supports_pdf_input: Option<bool>,
pub supports_embedding_image_input: Option<bool>,
pub supports_native_streaming: Option<bool>,
pub supports_native_structured_output: Option<bool>,
pub supports_bedrock_native_structured_output: Option<bool>,
pub supports_web_search: Option<bool>,
pub supports_url_context: Option<bool>,
pub supports_reasoning: Option<bool>,
pub supports_adaptive_thinking: Option<bool>,
pub supports_legacy_thinking: Option<bool>,
pub thinking_always_on: Option<bool>,
pub supports_tool_search: Option<bool>,
pub supports_mid_conversation_system: Option<bool>,
pub supports_anthropic_thinking_payload: Option<bool>,
pub supports_anthropic_compaction: Option<bool>,
pub supports_none_reasoning_effort: Option<bool>,
pub supports_minimal_reasoning_effort: Option<bool>,
pub supports_low_reasoning_effort: Option<bool>,
pub supports_xhigh_reasoning_effort: Option<bool>,
pub supports_max_reasoning_effort: Option<bool>,
pub reasoning_effort_levels: Option<Value>,
pub default_reasoning_effort: Option<Value>,
pub bedrock_output_config_effort_ceiling: Option<Value>,
pub bedrock_converse_supports_strict_tools: Option<bool>,
pub supports_regex_lookaround: Option<bool>,
pub supports_computer_use: Option<bool>,
pub search_context_cost_per_query: Option<Value>,
pub web_search_billing_unit: Option<Value>,
pub search_free_requests: Option<Value>,
pub google_maps_grounding_cost_per_query: Option<f64>,
pub tpm: Option<i64>,
pub rpm: Option<i64>,
pub ocr_cost_per_page: Option<f64>,
pub ocr_cost_per_page_batches: Option<f64>,
pub ocr_cost_per_credit: Option<f64>,
pub annotation_cost_per_page: Option<f64>,
pub annotation_cost_per_page_batches: Option<f64>,
pub provider_specific_entry: Option<Value>,
pub uses_embed_content: Option<bool>,
pub supports_image_size: Option<bool>,
pub supported_audio_formats: Option<Value>,
pub token_count_framing: Option<i64>,
pub token_count_model: Option<Value>,
pub token_count_tokenizer: Option<Value>,
pub vertex_ai_audio_api: Option<Value>,
/// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN`, copied as written
pub above_threshold_costs: IndexMap<String, Value>,
use crate::wire::Fields;
macro_rules! model_info_fields {
($($name:ident),* $(,)?) => {
/// The fields main's `ModelInfoBase(...)` copies in `get_model_info`, one field each, read from the raw row
/// without lowering it, so building one never fails and a bad value comes back as written
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub struct ModelInfoFields {
$(pub $name: Value,)*
/// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN` the fields above do not name, copied as written
#[serde(flatten)]
pub above_threshold_costs: IndexMap<String, Value>,
}
impl ModelInfoFields {
/// The copied field names in main's order
pub const FIELDS: &[&str] = &[$(stringify!($name)),*];
fn copied(key: &str, fields: &Fields, provider: Option<&str>) -> Self {
Self {
$($name: copied_field(stringify!($name), key, fields, provider),)*
above_threshold_costs: IndexMap::new(),
}
}
}
};
}
model_info_fields! {
key,
max_tokens,
max_input_tokens,
max_output_tokens,
input_cost_per_token,
input_cost_per_token_flex,
input_cost_per_token_priority,
input_cost_per_token_balanced,
input_cost_per_token_ultrafast,
cache_creation_input_token_cost,
cache_creation_input_token_cost_above_200k_tokens,
cache_creation_input_token_cost_above_272k_tokens,
cache_creation_input_token_cost_above_272k_tokens_priority,
cache_creation_input_token_cost_above_272k_tokens_flex,
cache_creation_input_token_cost_above_272k_tokens_ultrafast,
cache_creation_input_token_cost_flex,
cache_creation_input_token_cost_priority,
cache_creation_input_token_cost_ultrafast,
cache_read_input_token_cost,
cache_read_input_audio_token_cost,
cache_read_input_audio_token_cost_flex,
cache_read_input_audio_token_cost_priority,
cache_read_input_audio_token_cost_batches,
explicit_cache_read_input_token_cost,
prompt_cache_min_tokens,
cache_read_input_token_cost_above_200k_tokens,
cache_read_input_token_cost_above_200k_tokens_priority,
cache_read_input_token_cost_above_272k_tokens,
cache_read_input_token_cost_above_272k_tokens_priority,
cache_read_input_token_cost_above_272k_tokens_flex,
cache_read_input_token_cost_above_272k_tokens_ultrafast,
cache_read_input_token_cost_above_512k_tokens,
cache_read_input_token_cost_flex,
cache_read_input_token_cost_priority,
cache_read_input_token_cost_balanced,
cache_read_input_token_cost_ultrafast,
cache_read_input_token_cost_batches,
cache_read_input_token_cost_above_200k_tokens_batches,
cache_read_input_token_cost_above_272k_tokens_batches,
cache_creation_input_token_cost_batches,
cache_creation_input_token_cost_above_200k_tokens_batches,
cache_creation_input_token_cost_above_272k_tokens_batches,
cache_creation_input_token_cost_above_1hr,
cache_creation_input_token_cost_above_12hr,
cache_creation_input_token_cost_above_12hr_flex,
cache_creation_input_token_cost_above_12hr_priority,
cache_creation_input_token_cost_above_12hr_above_272k_tokens,
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex,
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority,
cache_creation_input_token_cost_above_12hr_batches,
cache_creation_input_token_cost_above_12hr_ultrafast,
cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches,
cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast,
off_peak_pricing,
input_cost_per_character,
input_cost_per_token_above_128k_tokens,
input_cost_per_token_above_200k_tokens,
input_cost_per_token_above_200k_tokens_priority,
input_cost_per_token_above_272k_tokens,
input_cost_per_token_above_272k_tokens_priority,
input_cost_per_token_above_272k_tokens_flex,
input_cost_per_token_above_272k_tokens_ultrafast,
input_cost_per_token_above_512k_tokens,
input_cost_per_query,
cost_per_second,
input_cost_per_second,
input_cost_per_text_message,
input_cost_per_audio_token,
input_cost_per_audio_token_flex,
input_cost_per_audio_token_priority,
input_cost_per_image_token,
input_cost_per_video_token,
input_cost_per_audio_token_batches,
input_cost_per_image_token_batches,
input_cost_per_image,
input_cost_per_audio_per_second,
input_cost_per_video_per_second,
input_cost_per_token_batches,
input_cost_per_video_token_batches,
input_cost_per_token_above_200k_tokens_batches,
input_cost_per_token_above_272k_tokens_batches,
output_cost_per_token_batches,
output_cost_per_token_above_200k_tokens_batches,
output_cost_per_token_above_272k_tokens_batches,
output_cost_per_token,
output_cost_per_token_flex,
output_cost_per_token_priority,
output_cost_per_token_balanced,
output_cost_per_token_ultrafast,
regional_processing_uplift_multiplier_eu,
regional_processing_uplift_multiplier_us,
regional_endpoint_uplift_multiplier,
output_cost_per_audio_token,
output_cost_per_character,
output_cost_per_reasoning_token,
output_cost_per_reasoning_token_flex,
output_cost_per_reasoning_token_priority,
output_cost_per_token_above_128k_tokens,
output_cost_per_character_above_128k_tokens,
output_cost_per_token_above_200k_tokens,
output_cost_per_token_above_200k_tokens_priority,
output_cost_per_token_above_272k_tokens,
output_cost_per_token_above_272k_tokens_priority,
output_cost_per_token_above_272k_tokens_flex,
output_cost_per_token_above_272k_tokens_ultrafast,
output_cost_per_token_above_512k_tokens,
output_cost_per_second,
output_cost_per_second_1080p,
output_cost_per_second_480p,
output_cost_per_second_720p,
output_cost_per_second_768p,
output_cost_per_second_2k,
output_cost_per_second_4k,
output_cost_per_video_per_second,
output_cost_per_image,
output_cost_per_pixel,
output_cost_per_image_token,
output_cost_per_image_token_batches,
output_cost_per_video_token,
output_vector_size,
citation_cost_per_token,
tiered_pricing,
litellm_provider,
mode,
supported_endpoints,
supports_system_messages,
supports_response_schema,
supports_vision,
supports_function_calling,
supports_parallel_function_calling,
supports_tool_choice,
supports_assistant_prefill,
supports_prompt_caching,
supports_prompt_cache_breakpoint,
supports_thinking_cache_preservation,
supports_audio_input,
supports_video_input,
supports_box_questions,
supports_audio_output,
supports_pdf_input,
supports_embedding_image_input,
supports_native_streaming,
supports_native_structured_output,
supports_bedrock_native_structured_output,
supports_web_search,
supports_url_context,
supports_reasoning,
supports_adaptive_thinking,
supports_legacy_thinking,
thinking_always_on,
supports_tool_search,
supports_mid_conversation_system,
supports_anthropic_thinking_payload,
supports_anthropic_compaction,
supports_none_reasoning_effort,
supports_minimal_reasoning_effort,
supports_low_reasoning_effort,
supports_xhigh_reasoning_effort,
supports_max_reasoning_effort,
reasoning_effort_levels,
default_reasoning_effort,
bedrock_output_config_effort_ceiling,
bedrock_converse_supports_strict_tools,
supports_regex_lookaround,
supports_computer_use,
search_context_cost_per_query,
browser_open_cost_per_request,
cost_per_additional_result,
web_search_billing_unit,
search_free_requests,
google_maps_grounding_cost_per_query,
tpm,
rpm,
ocr_cost_per_page,
ocr_cost_per_page_batches,
ocr_cost_per_credit,
annotation_cost_per_page,
annotation_cost_per_page_batches,
provider_specific_entry,
uses_embed_content,
supports_image_size,
supported_audio_formats,
token_count_framing,
token_count_model,
token_count_tokenizer,
vertex_ai_audio_api,
}
/// main's `get_model_info_helper` copy of one matched row. `provider` is the requested provider, which a row with no
/// `litellm_provider` field reports as its own
pub fn model_info(key: &str, fields: &Fields, provider: Option<&str>) -> ModelInfoFields {
let copied = ModelInfoFields::copied(key, fields, provider);
ModelInfoFields {
above_threshold_costs: fields
.iter()
.filter(|(name, _)| {
!ModelInfoFields::FIELDS.contains(&name.as_str())
&& is_above_threshold_cost_key(name)
})
.map(|(name, value)| (name.clone(), value.clone()))
.collect(),
..copied
}
}
/// One field as main's copy writes it. `max_tokens` is derived, a missing token price defaults to `0`
pub fn copied_field(name: &str, key: &str, fields: &Fields, provider: Option<&str>) -> Value {
let field = |name: &str| fields.get(name).cloned().unwrap_or(Value::Null);
match name {
"key" => Value::String(key.to_owned()),
"max_tokens" => {
let output = field("max_output_tokens");
if truthy(&output) {
output
} else {
field("max_input_tokens")
}
}
"input_cost_per_token" | "output_cost_per_token" => match field(name) {
Value::Null => Value::from(0),
value => value,
},
"litellm_provider" => fields.get(name).cloned().unwrap_or_else(|| {
provider.map_or(Value::Null, |provider| Value::String(provider.to_owned()))
}),
name => field(name),
}
}
/// `ABOVE_THRESHOLD_COST_KEY_PATTERN`, `_above_\d+k?_tokens(?:_batches)?$`
pub fn is_above_threshold_cost_key(name: &str) -> bool {
let name = name.strip_suffix("_batches").unwrap_or(name);
let Some(rest) = name.strip_suffix("_tokens") else {
return false;
};
let rest = rest.strip_suffix('k').unwrap_or(rest);
let Some((_, digits)) = rest.rsplit_once("_above_") else {
return false;
};
!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
}
/// Python truthiness of a JSON value, for main's `a or b`
fn truthy(value: &Value) -> bool {
match value {
Value::Null => false,
Value::Bool(flag) => *flag,
Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0),
Value::String(text) => !text.is_empty(),
Value::Array(items) => !items.is_empty(),
Value::Object(map) => !map.is_empty(),
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn fields(row: Value) -> Fields {
let Value::Object(fields) = row else {
unreachable!()
};
fields
}
#[rstest]
#[case::output_first(json!({"max_output_tokens": 10, "max_input_tokens": 20}), json!(10))]
#[case::zero_output_falls_to_input(json!({"max_output_tokens": 0, "max_input_tokens": 20}), json!(20))]
#[case::neither(json!({}), Value::Null)]
fn max_tokens_is_output_or_input(#[case] row: Value, #[case] expected: Value) {
assert_eq!(model_info("m", &fields(row), None).max_tokens, expected);
}
#[rstest]
fn a_missing_token_price_is_zero_and_a_bad_one_comes_back() {
let info = model_info("m", &fields(json!({"input_cost_per_token": "abc"})), None);
assert_eq!(info.input_cost_per_token, json!("abc"));
assert_eq!(info.output_cost_per_token, json!(0));
assert_eq!(info.key, json!("m"));
}
#[rstest]
fn above_threshold_keys_the_copy_does_not_name_are_kept() {
let info = model_info(
"m",
&fields(json!({"input_cost_per_token_above_999k_tokens": 1.0, "unrelated_field": 2.0})),
None,
);
assert_eq!(
info.above_threshold_costs
.get("input_cost_per_token_above_999k_tokens"),
Some(&json!(1.0))
);
assert!(!info.above_threshold_costs.contains_key("unrelated_field"));
}
#[rstest]
#[case::absent_takes_the_request(json!({}), json!("azure"))]
#[case::null_stays_null(json!({"litellm_provider": null}), Value::Null)]
#[case::written_wins(json!({"litellm_provider": "openai"}), json!("openai"))]
fn a_row_without_a_provider_reports_the_requested_one(
#[case] row: Value,
#[case] expected: Value,
) {
assert_eq!(
model_info("m", &fields(row), Some("azure")).litellm_provider,
expected
);
}
#[rstest]
#[case::k("input_cost_per_token_above_200k_tokens", true)]
#[case::batches("input_cost_per_token_above_200k_tokens_batches", true)]
#[case::plain_count("input_cost_per_token_above_128_tokens", true)]
#[case::priority_suffix("input_cost_per_token_above_200k_tokens_priority", false)]
#[case::one_hour("cache_creation_input_token_cost_above_1hr", false)]
fn the_threshold_key_pattern_matches_mains(#[case] name: &str, #[case] expected: bool) {
assert_eq!(is_above_threshold_cost_key(name), expected);
}
}

View file

@ -1,8 +1,11 @@
mod image;
mod provider_match;
use std::borrow::Cow;
pub use image::*;
pub use provider_match::*;
use serde_json::Value;
use crate::wire::Fields;
@ -59,3 +62,69 @@ pub enum Resolution {
tried: Vec<String>,
},
}
impl Candidates {
/// The names in main's lookup order, each with the slot it fills
pub fn in_order(&self) -> impl Iterator<Item = (CandidateSlot, &str)> {
[
(
CandidateSlot::CombinedModelName,
self.combined_model_name.as_deref(),
),
(CandidateSlot::Model, Some(self.model.as_str())),
(
CandidateSlot::RegionFreeCombinedModelName,
self.region_free_combined_model_name.as_deref(),
),
(CandidateSlot::SplitModel, self.split_model.as_deref()),
(
CandidateSlot::CombinedStrippedModelName,
self.combined_stripped_model_name.as_deref(),
),
(
CandidateSlot::StrippedModelName,
self.stripped_model_name.as_deref(),
),
(
CandidateSlot::ProviderPrefixedModelName,
self.provider_prefixed_model_name.as_deref(),
),
]
.into_iter()
.filter_map(|(slot, name)| Some((slot, name?)))
}
}
/// main's candidate walk. Each name is tried as an exact key, then through the case-insensitive index, and the first
/// whose row passes `provider_matches` wins
pub fn resolve(keys: &impl Keys, candidates: &Candidates, provider: Option<&str>) -> Resolution {
let found = candidates.in_order().find_map(|(slot, name)| {
let (key, case_insensitive) = if keys.contains(name) {
(name.to_owned(), false)
} else {
(keys.by_normalized(name)?, true)
};
let row = keys.row(&key)?;
row_provider_matches(&row, provider).then_some(Resolution::Found {
key,
via: Via {
slot,
case_insensitive,
},
})
});
found.unwrap_or_else(|| Resolution::Missing {
tried: candidates
.in_order()
.map(|(_, name)| name.to_owned())
.collect(),
})
}
fn row_provider_matches(row: &Fields, provider: Option<&str>) -> bool {
match row.get("litellm_provider") {
None | Some(Value::Null) => provider_matches(None, provider),
Some(Value::String(row_provider)) => provider_matches(Some(row_provider), provider),
Some(_) => provider_matches(Some(""), provider),
}
}

View file

@ -0,0 +1,50 @@
/// main's `_check_provider_match`. A row with no provider matches any request, and a few requested providers reuse
/// another provider's rows
pub fn provider_matches(row_provider: Option<&str>, requested: Option<&str>) -> bool {
let (Some(row), Some(requested)) = (
row_provider,
requested.filter(|requested| !requested.is_empty()),
) else {
return true;
};
if row == requested {
return true;
}
let same_family = |family: &str| requested == family && row.starts_with(family);
same_family("vertex_ai")
|| same_family("fireworks_ai")
|| (requested.starts_with("bedrock") && row.starts_with("bedrock"))
|| matches!(requested, "litellm_proxy" | "github" | "nadir")
|| (requested == "azure_ai" && matches!(row, "azure" | "openai"))
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::no_row_provider(None, Some("openai"), true)]
#[case::no_request(Some("openai"), None, true)]
#[case::empty_request(Some("openai"), Some(""), true)]
#[case::same(Some("openai"), Some("openai"), true)]
#[case::other(Some("anthropic"), Some("openai"), false)]
#[case::vertex_family(Some("vertex_ai-language-models"), Some("vertex_ai"), true)]
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
#[case::bedrock_family(Some("bedrock"), Some("bedrock_converse"), true)]
#[case::proxy_reuses_anything(Some("anthropic"), Some("litellm_proxy"), true)]
#[case::github_reuses_anything(Some("anthropic"), Some("github"), true)]
#[case::nadir_reuses_anything(Some("anthropic"), Some("nadir"), true)]
#[case::azure_ai_reuses_azure(Some("azure"), Some("azure_ai"), true)]
#[case::azure_ai_reuses_openai(Some("openai"), Some("azure_ai"), true)]
#[case::azure_ai_not_anthropic(Some("anthropic"), Some("azure_ai"), false)]
fn follows_mains_table(
#[case] row: Option<&str>,
#[case] requested: Option<&str>,
#[case] expected: bool,
) {
assert_eq!(provider_matches(row, requested), expected);
}
}

View file

@ -0,0 +1,107 @@
use std::borrow::Cow;
use std::collections::BTreeMap;
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{CandidateSlot, Candidates, Keys, Resolution, RowRef, Via, resolve};
use rstest::rstest;
use serde_json::{Value, json};
struct Map(BTreeMap<String, Fields>);
impl Keys for Map {
fn contains(&self, key: &str) -> bool {
self.0.contains_key(key)
}
fn row(&self, key: &str) -> Option<RowRef<'_>> {
self.0.get(key).map(Cow::Borrowed)
}
fn by_normalized(&self, name: &str) -> Option<String> {
let lower = name.to_lowercase();
self.0
.keys()
.find(|key| key.to_lowercase() == lower)
.cloned()
}
}
fn map(rows: Value) -> Map {
let Value::Object(rows) = rows else {
unreachable!()
};
Map(rows
.into_iter()
.map(|(key, row)| {
let Value::Object(fields) = row else {
unreachable!()
};
(key, fields)
})
.collect())
}
fn candidates(model: &str, combined: &str, split: &str) -> Candidates {
Candidates {
combined_model_name: Some(combined.to_owned()),
model: model.to_owned(),
region_free_combined_model_name: Some(combined.to_owned()),
split_model: Some(split.to_owned()),
combined_stripped_model_name: Some(combined.to_owned()),
stripped_model_name: Some(split.to_owned()),
provider_prefixed_model_name: Some(combined.to_owned()),
}
}
fn found(key: &str, slot: CandidateSlot, case_insensitive: bool) -> Resolution {
Resolution::Found {
key: key.to_owned(),
via: Via {
slot,
case_insensitive,
},
}
}
#[rstest]
#[case::provider_prefixed_row_first(
json!({"groq/llama": {"litellm_provider": "groq"}, "llama": {"litellm_provider": "groq"}}),
found("groq/llama", CandidateSlot::CombinedModelName, false)
)]
#[case::bare_row_when_no_prefixed_one(
json!({"llama": {"litellm_provider": "groq"}}),
found("llama", CandidateSlot::Model, false)
)]
#[case::a_mismatched_provider_is_skipped(
json!({"groq/llama": {"litellm_provider": "openai"}, "llama": {"litellm_provider": "groq"}}),
found("llama", CandidateSlot::Model, false)
)]
#[case::case_insensitive_through_the_index(
json!({"Groq/Llama": {"litellm_provider": "groq"}}),
found("Groq/Llama", CandidateSlot::CombinedModelName, true)
)]
fn the_first_matching_candidate_wins(#[case] rows: Value, #[case] expected: Resolution) {
assert_eq!(
resolve(
&map(rows),
&candidates("llama", "groq/llama", "llama"),
Some("groq")
),
expected
);
}
#[rstest]
fn a_miss_lists_every_candidate_tried() {
let Resolution::Missing { tried } = resolve(
&map(json!({})),
&candidates("x", "acme/x", "x"),
Some("acme"),
) else {
panic!("expected a miss");
};
assert_eq!(
tried,
["acme/x", "x", "acme/x", "x", "acme/x", "x", "acme/x"]
);
}

View file

@ -35,7 +35,8 @@ mod _native {
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::model_registry::{
Bundled, CatalogHandle, Fetched, File, Installed, Provenance, Remote, RowError, Unavailable,
Bundled, CatalogHandle, Fetched, File, Installed, Invalid, Missing, Provenance, Remote,
Resolved, RowError, Unavailable,
};
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
@ -163,6 +164,9 @@ mod tests {
"File",
"Bundled",
"Provenance",
"Resolved",
"Missing",
"Invalid",
"RowError",
"RowErrorKind",
"NativeClickHouseSpendConfig",

View file

@ -1,5 +1,7 @@
mod fetch;
mod install;
mod name_index;
mod resolve;
mod row_error;
use std::collections::HashMap;
@ -8,7 +10,7 @@ use std::time::SystemTime;
use litellm_host_python::{release_gil, to_py};
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Layers, Located};
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Keys, Layers, Located};
use litellm_model_catalog_refresh::{Source, git_blob_id};
use pyo3::exceptions::PyTypeError;
use pyo3::prelude::*;
@ -17,6 +19,7 @@ use serde::Serialize;
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
use install::{Record, write_into};
pub(crate) use resolve::{Invalid, Missing, Resolved};
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
@ -90,6 +93,7 @@ pub(crate) struct CatalogHandle {
module: Py<PyModule>,
attr: String,
state: Mutex<State>,
index: Mutex<Option<name_index::NameIndex>>,
}
#[pymethods]
@ -151,6 +155,7 @@ impl CatalogHandle {
record: None,
provenance: None,
}),
index: Mutex::new(None),
}
}
@ -185,6 +190,70 @@ impl CatalogHandle {
.unbind())
}
/// The bound dict's row for `model`, through main's candidate names (decisions 18 and 20)
#[pyo3(signature = (model, provider = None))]
fn resolve(&self, py: Python<'_>, model: &str, provider: Option<&str>) -> PyResult<Py<PyAny>> {
let dict = self.bound_dict(py)?;
let keys = resolve::LiveKeys {
dict: &dict,
index: &self.index,
};
resolve::resolve(py, &keys, model, provider)
}
/// The actual key for `name`, exact first, then through the case-insensitive index
fn lookup_key(&self, py: Python<'_>, name: &str) -> PyResult<Option<String>> {
let dict = self.bound_dict(py)?;
let keys = resolve::LiveKeys {
dict: &dict,
index: &self.index,
};
Ok(if keys.contains(name) {
Some(name.to_owned())
} else {
keys.by_normalized(name)
})
}
/// The keys whose row names this `litellm_provider`, from the index
fn provider_keys(&self, py: Python<'_>, provider: &str) -> PyResult<Vec<String>> {
let dict = self.bound_dict(py)?;
let keys = resolve::LiveKeys {
dict: &dict,
index: &self.index,
};
Ok(keys.provider_keys(provider))
}
/// main's 133-field copy of a row Python built, a generalization match or a fill-missing merge
#[staticmethod]
#[pyo3(signature = (key, row, provider = None))]
fn model_info_of(
py: Python<'_>,
key: &str,
row: &Bound<'_, PyDict>,
provider: Option<&str>,
) -> PyResult<Py<PyAny>> {
let fields: Fields = row
.iter()
.filter_map(|(name, value)| {
Some((
name.extract::<String>().ok()?,
litellm_host_python::from_py::<serde_json::Value>(&value).ok()?,
))
})
.collect();
to_py(
py,
&litellm_model_catalog::model_info(key, &fields, provider),
)
}
/// Rebuild the index on the next lookup, for a caller that knows keys changed with the length unchanged
fn mark_index_stale(&self) {
*self.index.lock().unwrap_or_else(PoisonError::into_inner) = None;
}
/// Where the installed map came from, `None` before the first install
fn provenance(&self) -> Option<Provenance> {
self.lock().provenance.clone()
@ -235,6 +304,13 @@ impl CatalogHandle {
}
impl CatalogHandle {
fn bound_dict<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
bound
.cast_into::<PyDict>()
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))
}
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}

View file

@ -0,0 +1,53 @@
use std::collections::HashMap;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
/// The one name index of decision 18. A normalized name to the actual key, and the keys each raw `litellm_provider`
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale
#[derive(Debug, Default)]
pub(crate) struct NameIndex {
dict_id: usize,
len: usize,
by_lower: HashMap<String, String>,
by_provider: HashMap<String, Vec<String>>,
}
impl NameIndex {
pub(crate) fn build(dict: &Bound<'_, PyDict>) -> Self {
let mut by_lower = HashMap::with_capacity(dict.len());
let mut by_provider: HashMap<String, Vec<String>> = HashMap::new();
for (key, row) in dict.iter() {
let Ok(key) = key.cast::<PyString>().map(|key| key.to_string()) else {
continue;
};
let provider = row
.cast::<PyDict>()
.ok()
.and_then(|row| row.get_item("litellm_provider").ok().flatten())
.and_then(|provider| provider.extract::<String>().ok());
if let Some(provider) = provider {
by_provider.entry(provider).or_default().push(key.clone());
}
by_lower.insert(key.to_lowercase(), key);
}
Self {
dict_id: dict.as_ptr() as usize,
len: dict.len(),
by_lower,
by_provider,
}
}
pub(crate) fn is_current(&self, dict: &Bound<'_, PyDict>) -> bool {
self.dict_id == dict.as_ptr() as usize && self.len == dict.len()
}
pub(crate) fn by_lower(&self, name: &str) -> Option<&str> {
self.by_lower.get(&name.to_lowercase()).map(String::as_str)
}
pub(crate) fn provider_keys(&self, provider: &str) -> &[String] {
self.by_provider.get(provider).map_or(&[], Vec::as_slice)
}
}

View file

@ -0,0 +1,201 @@
use std::borrow::Cow;
use std::sync::{Arc, Mutex, PoisonError};
use litellm_host_python::{from_py, to_py};
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{
Candidates, Catalog, Keys, Resolution, RowRef, model_info, rejects_row,
};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use serde_json::Value;
use super::name_index::NameIndex;
const CANDIDATES_MODULE: &str = "litellm.rust_bridge.host.model_names";
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Resolved {
#[pyo3(get)]
key: String,
/// The row as it was when `resolve` ran
fields: Arc<Fields>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Missing {
#[pyo3(get)]
tried: Vec<String>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Invalid {
#[pyo3(get)]
key: String,
#[pyo3(get)]
errors: Vec<String>,
fields: Arc<Fields>,
}
#[pymethods]
impl Resolved {
#[pyo3(signature = (provider = None))]
fn model_info(&self, py: Python<'_>, provider: Option<&str>) -> PyResult<Py<PyAny>> {
to_py(py, &model_info(&self.key, &self.fields, provider))
}
fn supports(&self, capability: &str) -> bool {
self.fields.get(capability) == Some(&Value::Bool(true))
}
/// The row as `resolve` read it, for the Python steps that still merge into it
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_py(py, self.fields.as_ref())
}
}
#[pymethods]
impl Invalid {
#[pyo3(signature = (provider = None))]
fn model_info(&self, py: Python<'_>, provider: Option<&str>) -> PyResult<Py<PyAny>> {
to_py(py, &model_info(&self.key, &self.fields, provider))
}
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_py(py, self.fields.as_ref())
}
}
/// The live map as `resolve` sees it, the dict bound when the call started (decision 1)
pub(crate) struct LiveKeys<'a, 'py> {
pub(crate) dict: &'a Bound<'py, PyDict>,
pub(crate) index: &'a Mutex<Option<NameIndex>>,
}
impl LiveKeys<'_, '_> {
fn indexed<T>(&self, read: impl Fn(&NameIndex) -> T) -> T {
let mut index = self.index.lock().unwrap_or_else(PoisonError::into_inner);
if !index
.as_ref()
.is_some_and(|index| index.is_current(self.dict))
{
*index = Some(NameIndex::build(self.dict));
}
read(index.as_ref().expect("the index was just built"))
}
fn rebuild(&self) {
*self.index.lock().unwrap_or_else(PoisonError::into_inner) =
Some(NameIndex::build(self.dict));
}
pub(crate) fn provider_keys(&self, provider: &str) -> Vec<String> {
self.indexed(|index| index.provider_keys(provider).to_vec())
}
/// Every field that converts to JSON. A value only Python can hold is left out, and reported by `full_row`
fn fields(&self, key: &str) -> Option<(Fields, bool)> {
let row = self.dict.get_item(key).ok().flatten()?;
let row = row.cast::<PyDict>().ok()?;
if let Ok(fields) = from_py::<Fields>(row) {
return Some((fields, true));
}
let fields = row
.iter()
.filter_map(|(name, value)| {
Some((
name.extract::<String>().ok()?,
from_py::<Value>(&value).ok()?,
))
})
.collect();
Some((fields, false))
}
}
impl Keys for LiveKeys<'_, '_> {
fn contains(&self, key: &str) -> bool {
self.dict.contains(key).unwrap_or(false)
}
fn row(&self, key: &str) -> Option<RowRef<'_>> {
self.fields(key).map(|(fields, _)| Cow::Owned(fields))
}
fn by_normalized(&self, name: &str) -> Option<String> {
let hit = self.indexed(|index| index.by_lower(name).map(str::to_owned))?;
if self.contains(&hit) {
return Some(hit);
}
self.rebuild();
self.indexed(|index| index.by_lower(name).map(str::to_owned))
.filter(|key| self.contains(key))
}
}
/// `get_model_info`'s lookup. Python builds the candidate names, Rust walks them over the live dict
pub(crate) fn resolve(
py: Python<'_>,
keys: &LiveKeys<'_, '_>,
model: &str,
provider: Option<&str>,
) -> PyResult<Py<PyAny>> {
let (candidates, provider) = candidates(py, model, provider)?;
match litellm_model_catalog::resolve(keys, &candidates, provider.as_deref()) {
Resolution::Missing { tried } => {
Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind())
}
Resolution::Found { key, .. } => {
let (fields, converted) = keys.fields(&key).unwrap_or_default();
let errors: Vec<String> = if converted {
Catalog::row_errors(&key, &fields)
.iter()
.filter(|error| rejects_row(&error.error))
.map(ToString::to_string)
.collect()
} else {
vec![format!("{key} must contain JSON-compatible values")]
};
let fields = Arc::new(fields);
if errors.is_empty() {
Ok(Resolved { key, fields }
.into_pyobject(py)?
.into_any()
.unbind())
} else {
Ok(Invalid {
key,
errors,
fields,
}
.into_pyobject(py)?
.into_any()
.unbind())
}
}
}
}
fn candidates(
py: Python<'_>,
model: &str,
provider: Option<&str>,
) -> PyResult<(Candidates, Option<String>)> {
let names = py
.import(CANDIDATES_MODULE)?
.call_method1("candidates", (model, provider))?;
let names = names.cast::<PyTuple>()?;
let name = |index: usize| -> PyResult<Option<String>> { names.get_item(index)?.extract() };
Ok((
Candidates {
combined_model_name: name(0)?,
model: model.to_owned(),
region_free_combined_model_name: name(1)?,
split_model: name(2)?,
combined_stripped_model_name: name(3)?,
stripped_model_name: name(4)?,
provider_prefixed_model_name: name(5)?,
},
name(6)?,
))
}

View file

@ -11,8 +11,11 @@ from litellm.rust_bridge._native import (
Fetched,
File,
Installed,
Invalid,
Missing,
Provenance,
Remote,
Resolved,
RowError,
Unavailable,
)
@ -38,8 +41,11 @@ __all__ = [
"Fetched",
"File",
"Installed",
"Invalid",
"Missing",
"Provenance",
"Remote",
"Resolved",
"RowError",
"Source",
"Unavailable",

View file

@ -1,3 +1,6 @@
import contextlib
import contextvars
from collections.abc import Generator
from typing import Final, cast
from urllib.parse import urlparse
@ -17,6 +20,18 @@ from litellm.secret_managers.main import get_secret, get_secret_str
from ..types.router import GenericLiteLLMParams, LiteLLM_Params
_PROVIDER_BANNER: Final = contextvars.ContextVar("litellm_provider_banner", default=True)
@contextlib.contextmanager
def provider_banner_suppressed() -> Generator[None]:
"""A model-info lookup tries names that need not route anywhere, so a miss there prints nothing (#5982)"""
token: Final = _PROVIDER_BANNER.set(False)
try:
yield
finally:
_PROVIDER_BANNER.reset(token)
def invalid_provider_routing_hints_message(error: ValidationError) -> str:
return "; ".join(
@ -553,7 +568,7 @@ def get_llm_provider(
custom_llm_provider = match_routing_generalization(model)
if not custom_llm_provider:
if litellm.suppress_debug_info is False:
if litellm.suppress_debug_info is False and _PROVIDER_BANNER.get():
print() # noqa: T201
print( # noqa: T201
"\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m"

View file

@ -34,7 +34,6 @@ from litellm.types.utils import (
ProviderSpecificModelInfo,
)
from litellm.utils import (
get_model_cost_mutation_generation,
supports_function_calling,
supports_reasoning,
supports_tool_choice,
@ -528,36 +527,22 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return [strip_litellm_response_only_fields(message) for message in messages]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of
# entries), so we memoize it. The cache key is (id(model_cost),
# mutation_generation): the generation counter is bumped on every
# register_model / reload path, so add+remove or in-place value
# replacement (which can leave id and len unchanged) still invalidates.
_fireworks_index_cache: tuple[int, int, list[tuple[str, dict]]] | None = None
@classmethod
def _get_fireworks_index(cls) -> list[tuple[str, dict]]:
"""The ``fireworks_ai/*`` rows of the live map, short name first. The keys come from the catalog's name index,
which follows keys added or removed by direct writes, and the rows are read live"""
from litellm._catalog import CATALOG
model_cost: Final = litellm.model_cost
signature: Final = (id(model_cost), get_model_cost_mutation_generation())
cached: Final = cls._fireworks_index_cache
if cached is not None and cached[0] == signature[0] and cached[1] == signature[1]:
return cached[2]
index: Final[list[tuple[str, dict]]] = []
for key, model_info in (*model_cost.items(),):
if not key.startswith("fireworks_ai/"):
continue
if not isinstance(model_info, dict):
continue
key_short = key[len("fireworks_ai/") :]
key_short = key_short.removeprefix("accounts/fireworks/models/")
if not key_short:
continue
index.append((key_short, model_info))
cls._fireworks_index_cache = (signature[0], signature[1], index)
return index
keys: Final = (*CATALOG.provider_keys("fireworks_ai"), *CATALOG.provider_keys("fireworks_ai-embedding-models"))
rows: Final = ((key, model_cost.get(key)) for key in keys)
return [
(key_short, model_info)
for key, model_info in rows
if key.startswith("fireworks_ai/")
and isinstance(model_info, dict)
and (key_short := key[len("fireworks_ai/") :].removeprefix("accounts/fireworks/models/"))
]
@staticmethod
def _matches_on_hyphen_boundary(short_name: str, key_short: str) -> bool:

View file

@ -100,6 +100,24 @@ class Provenance:
@property
def loaded_at(self) -> datetime: ...
@final
class Resolved:
key: str
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
def supports(self, capability: str) -> bool: ...
def row(self) -> dict[str, JsonValue]: ...
@final
class Missing:
tried: list[str]
@final
class Invalid:
key: str
errors: list[str]
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
def row(self) -> dict[str, JsonValue]: ...
@final
class Catalog:
@staticmethod
@ -108,6 +126,12 @@ class Catalog:
def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ...
def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ...
def provenance(self) -> Provenance | None: ...
def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ...
def lookup_key(self, name: str) -> str | None: ...
def provider_keys(self, provider: str) -> list[str]: ...
def mark_index_stale(self) -> None: ...
@staticmethod
def model_info_of(key: str, row: Mapping[str, object], provider: str | None = None) -> dict[str, JsonValue]: ...
def install(
self,
fetched: Fetched,

View file

@ -2,7 +2,7 @@
Python state Rust needs but does not own, behind one module per question. Rust imports a module here by path and calls a function by name
- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds)
- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds), `model_names` (the names `get_model_info` tries for a model)
- A module is the only place Rust reads that Python state. Rust never does `py.import("litellm").getattr(...)` on a global; it asks here
- Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers
- The module path and function signature are a contract. The Rust side pins the path in a `const MODULE` in the crate that asks (`python-bridge/src/routes/messages/host.rs`, `python-bridge/src/secrets/`, `callbacks-legacy-python/src/python.rs`); change both sides together and keep `tests/unit/rust_bridge/host/` green

View file

@ -0,0 +1,29 @@
"""The candidate names ``get_model_info`` tries for a model, for the Rust resolver"""
from typing import Final
from litellm.litellm_core_utils.get_llm_provider_logic import provider_banner_suppressed
def candidates(
model: str, custom_llm_provider: str | None
) -> tuple[str | None, str | None, str | None, str | None, str | None, str | None, str | None]:
"""Main's ``get_potential_model_names`` with the provider banner suppressed. The names come back in the order
``combined_model_name``, ``region_free_combined_model_name``, ``split_model``, ``combined_stripped_model_name``,
``stripped_model_name``, ``provider_prefixed_model_name``, then the provider rows must match"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.utils import get_potential_model_names
with provider_banner_suppressed():
names: Final = get_potential_model_names(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
return (
names["combined_model_name"],
names["region_free_combined_model_name"],
names["split_model"],
names["combined_stripped_model_name"],
names["stripped_model_name"],
names["provider_prefixed_model_name"],
names["custom_llm_provider"],
)

View file

@ -231,7 +231,6 @@ from litellm.types.llms.openai import (
OpenAIWebSearchOptions,
)
from litellm.types.utils import (
ABOVE_THRESHOLD_COST_KEY_PATTERN,
API_KEY_UPSTREAM_PARAMS,
OPENAI_RESPONSE_HEADERS,
AdminApiKeySecretReference,
@ -277,7 +276,7 @@ from litellm.types.utils import (
)
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
_BACKFILL_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
# +-----------------------------------------------+
# | |
@ -3095,7 +3094,10 @@ def supports_factory(model: str, custom_llm_provider: str | None, key: str) -> b
Raises:
Exception: If the given model is not found or there's an error in retrieval.
"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
provider_banner_suppressed,
)
try:
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
@ -3103,9 +3105,10 @@ def supports_factory(model: str, custom_llm_provider: str | None, key: str) -> b
model = model.removeprefix(f"{declared}/")
custom_llm_provider = declared # rebind-ok: same
else:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider
)
with provider_banner_suppressed():
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider
)
model_info: Final = get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
@ -6179,131 +6182,24 @@ def _strip_model_name(model: str, custom_llm_provider: str | None) -> str:
return _strip_dated_snapshot_suffix(model_name=model)
# Global case-insensitive lookup map for model_cost (built eagerly at module import)
_model_cost_lowercase_map: dict[str, str] | None = None
# Monotonic counter bumped on every model_cost mutation. Consumers that
# memoize derived state (e.g. provider-specific indices) can include this
# value in their cache key so they invalidate even when key add+remove or
# in-place value replacement leaves len/id unchanged.
_model_cost_mutation_generation: int = 0
def get_model_cost_mutation_generation() -> int:
return _model_cost_mutation_generation
def _invalidate_model_cost_lowercase_map() -> None:
"""Invalidate the case-insensitive lookup map for model_cost.
"""Kept for callers on main. Reads go to the live dict, so only a key swap that leaves the length unchanged needs
the name index rebuilt, and this marks it stale"""
from litellm._catalog import CATALOG
Call this whenever litellm.model_cost is modified to ensure the map is rebuilt.
Also clears related LRU caches that depend on model_cost data.
"""
global _model_cost_lowercase_map, _model_cost_mutation_generation
_model_cost_lowercase_map = None
_model_cost_mutation_generation += 1
# Clear LRU caches that depend on model_cost data
_cached_get_model_info.cache_clear()
cached_get_model_info_helper.cache_clear()
def _rebuild_model_cost_lowercase_map() -> dict[str, str]:
"""Rebuild the case-insensitive lookup map from the current model_cost.
Returns:
The rebuilt map (guaranteed to be not None).
"""
global _model_cost_lowercase_map
_model_cost_lowercase_map = {k.lower(): k for k in tuple(litellm.model_cost)}
return _model_cost_lowercase_map
def _handle_stale_map_entry_rebuild(
potential_key_lower: str,
) -> str | None:
"""
Handle stale _model_cost_lowercase_map entry (key was popped).
Rebuilds the map and retries the lookup.
Returns:
The matched key if found after rebuild, None otherwise.
"""
matched_key: Final = _rebuild_model_cost_lowercase_map().get(potential_key_lower)
if matched_key is not None and matched_key in litellm.model_cost:
return matched_key
return None
def _handle_new_key_with_scan(
potential_key_lower: str,
) -> str | None:
"""
Handle new key added to model_cost without invalidating _model_cost_lowercase_map.
Scans model_cost for case-insensitive match and rebuilds the map if found.
Returns:
The matched key if found, None otherwise.
"""
global _model_cost_lowercase_map
for key in tuple(_builtin_cost_map()):
if key.lower() == potential_key_lower:
_model_cost_lowercase_map = _rebuild_model_cost_lowercase_map()
return key
return None
CATALOG.mark_index_stale()
def get_model_cost_key(potential_key: str) -> str | None:
"""
Get the actual key from model_cost, with case-insensitive fallback.
"""The actual key in ``litellm.model_cost`` for ``potential_key``, exact first, then case-insensitive"""
from litellm._catalog import CATALOG
WARNING: Only O(1) lookup operations are acceptable. O(n) lookups will cause severe
CPU overhead. This function is called frequently during router operations.
ALLOWED HELPER FUNCTIONS (conditionally called, O(n) operations are acceptable):
- _rebuild_model_cost_lowercase_map: Rebuilds the lookup map (only when map is None)
- _handle_stale_map_entry_rebuild: Rebuilds map when stale entry detected (rare case)
If you need to add a new helper function with O(n) operations that is conditionally
called and confirmed not to cause performance issues, add it to the allowed_helpers
list in: tests/code_coverage_tests/check_get_model_cost_key_performance.py
"""
# Exact match (O(1))
if potential_key in litellm.model_cost:
return potential_key
# Case-insensitive lookup via map (O(1)); bound locally because another thread's
# registration can invalidate the module-level map at any point
cached_lowercase_map: Final = _model_cost_lowercase_map
lowercase_map: Final = (
cached_lowercase_map if cached_lowercase_map is not None else _rebuild_model_cost_lowercase_map()
)
potential_key_lower: Final = potential_key.lower()
matched_key = lowercase_map.get(potential_key_lower)
# Verify key exists (O(1) - handles model_cost.pop() case)
if matched_key is not None and matched_key in litellm.model_cost:
return matched_key
# Rebuild map if stale entry detected (O(n) rebuild, but only when stale entry found)
if matched_key is not None:
matched_key = _handle_stale_map_entry_rebuild(potential_key_lower)
if matched_key is not None:
return matched_key
return None
return CATALOG.lookup_key(potential_key)
_get_model_cost_key = get_model_cost_key
def _get_model_info_from_model_cost(key: str) -> dict[str, object]:
return litellm.model_cost[key]
def _check_provider_match(model_info: dict, custom_llm_provider: str | None) -> bool:
"""
Check if the model info provider matches the custom provider.
@ -6370,19 +6266,6 @@ class PotentialModelNamesAndCustomLLMProvider(TypedDict):
custom_llm_provider: str
def _first_registered_match(
candidates: Sequence[str], custom_llm_provider: str | None
) -> tuple[str | None, dict[str, Any] | None]:
registered_keys: Final = (key for key in map(get_model_cost_key, candidates) if key is not None)
entries: Final = ((key, _get_model_info_from_model_cost(key=key)) for key in registered_keys)
matches: Final = (
(key, info)
for key, info in entries
if _check_provider_match(model_info=info, custom_llm_provider=custom_llm_provider)
)
return next(matches, (None, None))
def _get_model_info_from_generalization(
model: str,
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
@ -6515,6 +6398,7 @@ def get_potential_model_names(model: str, custom_llm_provider: str | None) -> Po
_get_potential_model_names = get_potential_model_names
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _get_max_position_embeddings(model_name: str) -> int | None:
# Construct the URL for the config.json file
config_url: Final = f"https://huggingface.co/{model_name}/raw/main/config.json"
@ -6538,17 +6422,12 @@ def _get_max_position_embeddings(model_name: str) -> int | None:
return None
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def cached_get_model_info_helper(
model: str,
custom_llm_provider: str | None,
api_base: str | None = None,
) -> ModelInfoBase:
"""
_get_model_info_helper wrapped with lru_cache
Speed Optimization to hit high RPS
"""
"""Kept for callers on main. Reads go to the live dict, so there is nothing to cache"""
return get_model_info_helper(
model=model,
custom_llm_provider=custom_llm_provider,
@ -6579,19 +6458,7 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp
return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None
def _is_potential_model_name_in_model_cost(
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
) -> bool:
"""
Check if the potential model name is in the model cost (case-insensitive).
"""
return any(
get_model_cost_key(str(potential_model_name)) is not None
for potential_model_name in potential_model_names.values()
)
_ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
@ -6610,7 +6477,10 @@ def get_model_info_helper(
"""
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
"""
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
provider_banner_suppressed,
)
try:
azure_llms: Final = {**litellm.azure_llms, **litellm.azure_embedding_models}
@ -6626,19 +6496,15 @@ def get_model_info_helper(
):
model = model + "@latest"
##########################
potential_model_names: Final = get_potential_model_names(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
requested_provider: Final = custom_llm_provider
with provider_banner_suppressed():
potential_model_names: Final = get_potential_model_names(
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
)
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
combined_model_name: Final = potential_model_names["combined_model_name"]
stripped_model_name: Final = potential_model_names["stripped_model_name"]
combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"]
provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"]
split_model: Final = potential_model_names["split_model"]
custom_llm_provider = potential_model_names["custom_llm_provider"]
model_cost_custom_llm_provider: Final = custom_llm_provider
#########################
provider_config: BaseLLMModelInfo | None = None
if custom_llm_provider and custom_llm_provider in LlmProvidersSet:
@ -6703,20 +6569,18 @@ def get_model_info_helper(
and strip it. Tried last so no model that already resolves through 1-6 can change.
"""
lookup_order: Final = (
combined_model_name,
model,
potential_model_names["region_free_combined_model_name"],
split_model,
combined_stripped_model_name,
stripped_model_name,
provider_prefixed_model_name,
)
lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider)
key: str | None = lookup[0]
_model_info: dict[str, Any] | None = lookup[1]
from litellm._catalog import CATALOG, Invalid, Missing
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
resolution: Final = CATALOG.resolve(model, requested_provider)
key: str | None = None if isinstance(resolution, Missing) else resolution.key
if isinstance(resolution, Invalid) and resolution.key not in _REPORTED_INVALID_ROWS:
_REPORTED_INVALID_ROWS.add(resolution.key)
verbose_logger.warning(
"litellm.model_cost[%r] fails validation: %s", resolution.key, "; ".join(resolution.errors)
)
_model_info: dict[str, Any] | None = None if isinstance(resolution, Missing) else resolution.row()
if _model_info is not None and key is not None and (_model_info.get("mode") or "chat") in _BACKFILL_MODES:
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
if fill_missing is not None:
_model_info = {
@ -6735,321 +6599,11 @@ def get_model_info_helper(
if _model_info is None or key is None:
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
if _input_cost_per_token is None:
# default value to 0, be noisy about this
verbose_logger.debug(
"model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.",
model,
custom_llm_provider,
)
_input_cost_per_token = 0
from litellm._catalog import Catalog
_output_cost_per_token: float | None = _model_info.get("output_cost_per_token")
if _output_cost_per_token is None:
# default value to 0, be noisy about this
verbose_logger.debug(
"model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.",
model,
custom_llm_provider,
)
_output_cost_per_token = 0
returned_model_info: Final = ModelInfoBase(
key=key,
max_tokens=_model_info.get("max_output_tokens") or _model_info.get("max_input_tokens"),
max_input_tokens=_model_info.get("max_input_tokens", None),
max_output_tokens=_model_info.get("max_output_tokens", None),
input_cost_per_token=_input_cost_per_token,
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
input_cost_per_token_balanced=_model_info.get("input_cost_per_token_balanced", None),
input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None),
cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None),
cache_creation_input_token_cost_above_200k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_200k_tokens", None
),
cache_creation_input_token_cost_above_272k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens", None
),
cache_creation_input_token_cost_above_272k_tokens_priority=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_priority", None
),
cache_creation_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_ultrafast", None
),
cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None),
cache_creation_input_token_cost_priority=_model_info.get(
"cache_creation_input_token_cost_priority", None
),
cache_creation_input_token_cost_ultrafast=_model_info.get(
"cache_creation_input_token_cost_ultrafast", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
cache_read_input_audio_token_cost_priority=_model_info.get(
"cache_read_input_audio_token_cost_priority", None
),
cache_read_input_audio_token_cost_batches=_model_info.get(
"cache_read_input_audio_token_cost_batches", None
),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None
),
cache_read_input_token_cost_above_200k_tokens_priority=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens_priority", None
),
cache_read_input_token_cost_above_272k_tokens=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens", None
),
cache_read_input_token_cost_above_272k_tokens_priority=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_priority", None
),
cache_read_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_flex", None
),
cache_read_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_ultrafast", None
),
cache_read_input_token_cost_above_512k_tokens=_model_info.get(
"cache_read_input_token_cost_above_512k_tokens", None
),
cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None),
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
cache_read_input_token_cost_balanced=_model_info.get("cache_read_input_token_cost_balanced", None),
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"),
cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens_batches"
),
cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_batches"
),
cache_creation_input_token_cost_batches=_model_info.get("cache_creation_input_token_cost_batches"),
cache_creation_input_token_cost_above_200k_tokens_batches=_model_info.get(
"cache_creation_input_token_cost_above_200k_tokens_batches"
),
cache_creation_input_token_cost_above_272k_tokens_batches=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_batches"
),
cache_creation_input_token_cost_above_1hr=_model_info.get(
"cache_creation_input_token_cost_above_1hr", None
),
cache_creation_input_token_cost_above_12hr=_model_info.get(
"cache_creation_input_token_cost_above_12hr", None
),
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_flex", None
),
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_priority", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
),
cache_creation_input_token_cost_above_12hr_batches=_model_info.get(
"cache_creation_input_token_cost_above_12hr_batches", None
),
cache_creation_input_token_cost_above_12hr_ultrafast=_model_info.get(
"cache_creation_input_token_cost_above_12hr_ultrafast", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None
),
off_peak_pricing=_model_info.get("off_peak_pricing", None),
input_cost_per_character=_model_info.get("input_cost_per_character", None),
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
input_cost_per_token_above_200k_tokens=_model_info.get("input_cost_per_token_above_200k_tokens", None),
input_cost_per_token_above_200k_tokens_priority=_model_info.get(
"input_cost_per_token_above_200k_tokens_priority", None
),
input_cost_per_token_above_272k_tokens=_model_info.get("input_cost_per_token_above_272k_tokens", None),
input_cost_per_token_above_272k_tokens_priority=_model_info.get(
"input_cost_per_token_above_272k_tokens_priority", None
),
input_cost_per_token_above_272k_tokens_flex=_model_info.get(
"input_cost_per_token_above_272k_tokens_flex", None
),
input_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
"input_cost_per_token_above_272k_tokens_ultrafast", None
),
input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None),
input_cost_per_query=_model_info.get("input_cost_per_query", None),
cost_per_second=_model_info.get("cost_per_second", None),
input_cost_per_second=_model_info.get("input_cost_per_second", None),
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
input_cost_per_image_token_batches=_model_info.get("input_cost_per_image_token_batches", None),
input_cost_per_image=_model_info.get("input_cost_per_image", None),
input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None),
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None),
input_cost_per_token_above_200k_tokens_batches=_model_info.get(
"input_cost_per_token_above_200k_tokens_batches"
),
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
"input_cost_per_token_above_272k_tokens_batches"
),
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
output_cost_per_token_above_200k_tokens_batches=_model_info.get(
"output_cost_per_token_above_200k_tokens_batches"
),
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
"output_cost_per_token_above_272k_tokens_batches"
),
output_cost_per_token=_output_cost_per_token,
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
output_cost_per_token_balanced=_model_info.get("output_cost_per_token_balanced", None),
output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None),
regional_processing_uplift_multiplier_eu=_model_info.get(
"regional_processing_uplift_multiplier_eu", None
),
regional_processing_uplift_multiplier_us=_model_info.get(
"regional_processing_uplift_multiplier_us", None
),
regional_endpoint_uplift_multiplier=_model_info.get("regional_endpoint_uplift_multiplier", None),
output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None),
output_cost_per_character=_model_info.get("output_cost_per_character", None),
output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None),
output_cost_per_reasoning_token_flex=_model_info.get("output_cost_per_reasoning_token_flex", None),
output_cost_per_reasoning_token_priority=_model_info.get(
"output_cost_per_reasoning_token_priority", None
),
output_cost_per_token_above_128k_tokens=_model_info.get(
"output_cost_per_token_above_128k_tokens", None
),
output_cost_per_character_above_128k_tokens=_model_info.get(
"output_cost_per_character_above_128k_tokens", None
),
output_cost_per_token_above_200k_tokens=_model_info.get(
"output_cost_per_token_above_200k_tokens", None
),
output_cost_per_token_above_200k_tokens_priority=_model_info.get(
"output_cost_per_token_above_200k_tokens_priority", None
),
output_cost_per_token_above_272k_tokens=_model_info.get(
"output_cost_per_token_above_272k_tokens", None
),
output_cost_per_token_above_272k_tokens_priority=_model_info.get(
"output_cost_per_token_above_272k_tokens_priority", None
),
output_cost_per_token_above_272k_tokens_flex=_model_info.get(
"output_cost_per_token_above_272k_tokens_flex", None
),
output_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
"output_cost_per_token_above_272k_tokens_ultrafast", None
),
output_cost_per_token_above_512k_tokens=_model_info.get(
"output_cost_per_token_above_512k_tokens", None
),
output_cost_per_second=_model_info.get("output_cost_per_second", None),
output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None),
output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None),
output_cost_per_second_720p=_model_info.get("output_cost_per_second_720p", None),
output_cost_per_second_768p=_model_info.get("output_cost_per_second_768p", None),
output_cost_per_second_2k=_model_info.get("output_cost_per_second_2k", None),
output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None),
output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None),
output_cost_per_image=_model_info.get("output_cost_per_image", None),
output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None),
output_vector_size=_model_info.get("output_vector_size", None),
citation_cost_per_token=_model_info.get("citation_cost_per_token", None),
tiered_pricing=_model_info.get("tiered_pricing", None),
litellm_provider=_model_info.get("litellm_provider", custom_llm_provider),
mode=_model_info.get("mode"),
supported_endpoints=_model_info.get("supported_endpoints", None),
supports_system_messages=_model_info.get("supports_system_messages", None),
supports_response_schema=_model_info.get("supports_response_schema", None),
supports_vision=_model_info.get("supports_vision", None),
supports_function_calling=_model_info.get("supports_function_calling", None),
supports_parallel_function_calling=_model_info.get("supports_parallel_function_calling", None),
supports_tool_choice=_model_info.get("supports_tool_choice", None),
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None),
supports_audio_input=_model_info.get("supports_audio_input", None),
supports_video_input=_model_info.get("supports_video_input", None),
supports_box_questions=_model_info.get("supports_box_questions", None),
supports_audio_output=_model_info.get("supports_audio_output", None),
supports_pdf_input=_model_info.get("supports_pdf_input", None),
supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None),
supports_native_streaming=_model_info.get("supports_native_streaming", None),
supports_native_structured_output=_model_info.get("supports_native_structured_output", None),
supports_bedrock_native_structured_output=_model_info.get(
"supports_bedrock_native_structured_output", None
),
supports_web_search=_model_info.get("supports_web_search", None),
supports_url_context=_model_info.get("supports_url_context", None),
supports_reasoning=_model_info.get("supports_reasoning", None),
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
thinking_always_on=_model_info.get("thinking_always_on", None),
supports_tool_search=_model_info.get("supports_tool_search", None),
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),
supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None),
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None),
default_reasoning_effort=_model_info.get("default_reasoning_effort", None),
bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
supports_regex_lookaround=_model_info.get("supports_regex_lookaround", None),
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
browser_open_cost_per_request=_model_info.get("browser_open_cost_per_request", None),
cost_per_additional_result=_model_info.get("cost_per_additional_result", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),
ocr_cost_per_page=_model_info.get("ocr_cost_per_page", None),
ocr_cost_per_page_batches=_model_info.get("ocr_cost_per_page_batches", None),
ocr_cost_per_credit=_model_info.get("ocr_cost_per_credit", None),
annotation_cost_per_page=_model_info.get("annotation_cost_per_page", None),
annotation_cost_per_page_batches=_model_info.get("annotation_cost_per_page_batches", None),
provider_specific_entry=_model_info.get("provider_specific_entry", None),
uses_embed_content=_model_info.get("uses_embed_content", None),
supports_image_size=_model_info.get("supports_image_size", None),
supported_audio_formats=_model_info.get("supported_audio_formats", None),
token_count_framing=_model_info.get("token_count_framing", None),
token_count_model=_model_info.get("token_count_model", None),
token_count_tokenizer=_model_info.get("token_count_tokenizer", None),
vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
returned_model_info: Final = cast( # cast-ok: Rust builds the dict from the ModelInfoBase field list
ModelInfoBase, Catalog.model_info_of(key, _model_info, custom_llm_provider)
)
for cost_key, cost_value in _model_info.items():
if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None:
returned_model_info[cost_key] = cost_value
return returned_model_info
except ModelNotMappedError:
raise
@ -7067,7 +6621,12 @@ def _build_model_info(
api_base: str | None = None,
api_key: str | None = None,
) -> ModelInfo:
supported_openai_params = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider)
from litellm.litellm_core_utils.get_llm_provider_logic import provider_banner_suppressed
with provider_banner_suppressed():
supported_openai_params = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
_model_info: Final = get_model_info_helper(
model=model,
@ -7088,15 +6647,6 @@ def _build_model_info(
return ModelInfo(**_model_info, supported_openai_params=supported_openai_params)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
def _cached_get_model_info(
model: str,
custom_llm_provider: str | None = None,
api_base: str | None = None,
) -> ModelInfo:
return _build_model_info(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base)
def get_model_info(
model: str,
custom_llm_provider: str | None = None,
@ -7176,15 +6726,12 @@ def get_model_info(
"supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
}
"""
# api_key is a per-caller credential, not part of the model identity, so it is
# kept out of the cache key; explicit keys are resolved without the cache.
if api_key is not None:
return _build_model_info(model, custom_llm_provider, api_base, api_key)
return _cached_get_model_info(model, custom_llm_provider, api_base)
return _build_model_info(model, custom_llm_provider, api_base, api_key)
get_model_info.cache_clear = _cached_get_model_info.cache_clear
get_model_info.cache_info = _cached_get_model_info.cache_info
get_model_info.cache_clear = _get_max_position_embeddings.cache_clear # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
get_model_info.cache_info = _get_max_position_embeddings.cache_info # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
cached_get_model_info_helper.cache_clear = _get_max_position_embeddings.cache_clear # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
def json_schema_type(python_type_name: str):

View file

@ -152,11 +152,7 @@ def check_get_model_cost_key_performance():
# Check for for loops
if re.search(r"\bfor\s+\w+\s+in\s+", line):
# Allow helper function calls (they're conditional)
if not re.search(
r"(_rebuild_model_cost_lowercase_map|_handle_stale_map_entry_rebuild|_handle_new_key_with_scan)",
line,
):
problematic_lines.append((i, "for loop", line_stripped))
problematic_lines.append((i, "for loop", line_stripped))
# Check for while loops
if re.search(r"\bwhile\s+", line):
@ -187,11 +183,7 @@ def check_get_model_cost_key_performance():
# Check for calls to functions that might have O(n) operations
# Allow known helper functions that are conditional
allowed_helpers = [
"_rebuild_model_cost_lowercase_map",
"_handle_stale_map_entry_rebuild",
"_handle_new_key_with_scan",
]
allowed_helpers = []
# Check for function calls (pattern: function_name(...), but not function definitions)
# Skip function definitions (def function_name(...))

View file

@ -2358,3 +2358,17 @@ def test_global_disable_flag_with_transform_messages_helper(
)
assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}]
def test_a_fireworks_row_added_by_a_direct_write_is_matched(monkeypatch):
import litellm
monkeypatch.setitem(
litellm.model_cost,
"fireworks_ai/accounts/fireworks/models/acme-direct-9b",
{"litellm_provider": "fireworks_ai", "mode": "chat", "supports_vision": True},
)
info = FireworksAIConfig().get_provider_info("accounts/myorg/models/custom-acme-direct-9b")
assert info.get("supports_vision") is True

View file

@ -396,8 +396,6 @@ class TestOllamaGetModelInfo:
key must still reach the backend rather than be served from a result cached
with a different key.
"""
from litellm.utils import _cached_get_model_info
captured_headers = []
def mock_post(url, json, headers=None):
@ -420,7 +418,6 @@ class TestOllamaGetModelInfo:
api_key=api_key,
)
assert _cached_get_model_info.cache_info().currsize <= 1
assert captured_headers == [
{"Authorization": "Bearer key-one"},
{"Authorization": "Bearer key-two"},

File diff suppressed because it is too large Load diff

View file

@ -5100,7 +5100,7 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey
def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.utils import _invalidate_model_cost_lowercase_map, _rebuild_model_cost_lowercase_map
from litellm.utils import _invalidate_model_cost_lowercase_map, get_model_cost_key
class _KeyRegisteredMidWalk(str):
def lower(self) -> str:
@ -5115,9 +5115,8 @@ def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: p
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = _EntryRegisteredMidWalk(mode="chat")
try:
assert _rebuild_model_cost_lowercase_map()["lit-5853-walked-last"] == "LIT-5853-WALKED-LAST"
assert get_model_cost_key("lit-5853-walked-last") == "LIT-5853-WALKED-LAST"
litellm.add_known_models(model_cost_map=litellm.model_cost)
assert "lit-5853-registered-during-the-rebuild" in litellm.model_cost
assert "lit-5853-registered-during-add-known-models" in litellm.model_cost
finally:
_invalidate_model_cost_lowercase_map()
@ -5169,22 +5168,19 @@ class _EntryReadDuringARegistration(dict[str, object]):
return super().__contains__(key)
@pytest.mark.parametrize("walk", ["fal_ai", "fireworks_ai", "ollama", "gpt-5", "case-insensitive scan"])
@pytest.mark.parametrize("walk", ["fal_ai", "ollama", "gpt-5"])
def test_request_path_cost_map_walks_survive_a_registration_mid_walk(
monkeypatch: pytest.MonkeyPatch, walk: str
) -> None:
from litellm.llms.fal_ai.cost_calculator import _keyed_rows
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
from litellm.llms.ollama.common_utils import OllamaModelInfo
from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort
from litellm.utils import _handle_new_key_with_scan, _invalidate_model_cost_lowercase_map
from litellm.utils import _invalidate_model_cost_lowercase_map
walks: Final[Mapping[str, Callable[[], object]]] = {
"fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"),
"fireworks_ai": FireworksAIConfig._get_fireworks_index,
"ollama": lambda: OllamaModelInfo._is_static_ollama_model("ollama/lit-5853-model"),
"gpt-5": _catalogue_declares_default_effort,
"case-insensitive scan": lambda: _handle_new_key_with_scan("lit-5853-model"),
}
monkeypatch.setattr(
litellm,
@ -10900,3 +10896,67 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_
)
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}
def test_get_model_info_sees_a_direct_write_without_an_invalidator(monkeypatch):
from litellm.utils import cached_get_model_info_helper
litellm.get_model_info("gpt-4o")
monkeypatch.setitem(litellm.model_cost, "gpt-4o", {**litellm.model_cost["gpt-4o"], "input_cost_per_token": 1.0})
assert litellm.get_model_info("gpt-4o")["input_cost_per_token"] == 1.0
assert cached_get_model_info_helper("gpt-4o", "openai")["input_cost_per_token"] == 1.0
def test_get_model_info_sees_a_nested_write_without_an_invalidator(monkeypatch):
litellm.get_model_info("gpt-4o")
litellm.supports_vision("gpt-4o")
monkeypatch.setitem(litellm.model_cost, "gpt-4o", dict(litellm.model_cost["gpt-4o"]))
litellm.model_cost["gpt-4o"]["input_cost_per_token"] = 1.0
litellm.model_cost["gpt-4o"]["supports_vision"] = False
assert litellm.get_model_info("gpt-4o")["input_cost_per_token"] == 1.0
assert litellm.supports_vision("gpt-4o") is False
@pytest.mark.parametrize(
"lookup",
[
pytest.param(lambda: litellm.get_model_info("acme-unknown-model-5982"), id="get_model_info"),
pytest.param(lambda: litellm.supports_function_calling("acme-unknown-model-5982"), id="supports"),
],
)
def test_an_unknown_model_prints_no_provider_banner(monkeypatch, capsys, lookup):
monkeypatch.setattr(litellm, "suppress_debug_info", False)
with contextlib.suppress(Exception):
lookup()
assert capsys.readouterr().out == ""
def test_a_key_added_by_a_direct_write_is_found_case_insensitively(monkeypatch):
from litellm.utils import get_model_cost_key
assert get_model_cost_key("mixed-case-direct") is None
monkeypatch.setitem(litellm.model_cost, "Mixed-Case-Direct", {"litellm_provider": "openai"})
assert get_model_cost_key("mixed-case-direct") == "Mixed-Case-Direct"
def test_resolution_matches_mains_for_every_bundled_key_and_variant():
reference = json.loads((Path(__file__).parent / "resolution_reference.json").read_text())
def resolved(model, provider):
try:
return litellm.get_model_info(model, provider)["key"]
except Exception:
return None
mismatches = [
(model, provider, expected, actual)
for model, provider, expected in reference["cases"]
if (actual := resolved(model, provider)) != expected
]
assert mismatches == []