mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(model-cost): read model info and capabilities from the live map through the Rust resolver
get_model_info, get_model_cost_key and supports_* resolve through one Rust walk over the bound dict, with one name index rebuilt when (id, len) changes. The two lru_caches, the lowercase map and the Fireworks index go, so direct writes are seen by the next read, and a miss prints no banner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
944d1648b9
commit
1b2c40f25f
20 changed files with 13462 additions and 748 deletions
|
|
@ -1,194 +1,356 @@
|
|||
use indexmap::IndexMap;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
/// The fields main's `ModelInfoBase(...)` copies (`litellm/utils.py` at c7c895db45), one typed field each,
|
||||
/// read from the raw row without lowering it, so building one never fails
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct ModelInfoFields {
|
||||
pub key: String,
|
||||
pub max_tokens: Option<i64>,
|
||||
pub max_input_tokens: Option<i64>,
|
||||
pub max_output_tokens: Option<i64>,
|
||||
pub input_cost_per_token: Option<f64>,
|
||||
pub input_cost_per_token_flex: Option<f64>,
|
||||
pub input_cost_per_token_priority: Option<f64>,
|
||||
pub input_cost_per_token_balanced: Option<f64>,
|
||||
pub input_cost_per_token_ultrafast: Option<f64>,
|
||||
pub cache_creation_input_token_cost: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub cache_creation_input_token_cost_flex: Option<f64>,
|
||||
pub cache_creation_input_token_cost_priority: Option<f64>,
|
||||
pub cache_creation_input_token_cost_ultrafast: Option<f64>,
|
||||
pub cache_read_input_token_cost: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_flex: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_priority: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_batches: Option<f64>,
|
||||
pub explicit_cache_read_input_token_cost: Option<f64>,
|
||||
pub prompt_cache_min_tokens: Option<i64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
|
||||
pub cache_read_input_token_cost_flex: Option<f64>,
|
||||
pub cache_read_input_token_cost_priority: Option<f64>,
|
||||
pub cache_read_input_token_cost_balanced: Option<f64>,
|
||||
pub cache_read_input_token_cost_ultrafast: Option<f64>,
|
||||
pub cache_read_input_token_cost_batches: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_flex: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_priority: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_ultrafast: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub off_peak_pricing: Option<Value>,
|
||||
pub input_cost_per_character: Option<f64>,
|
||||
pub input_cost_per_token_above_128k_tokens: Option<f64>,
|
||||
pub input_cost_per_token_above_200k_tokens: Option<f64>,
|
||||
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
|
||||
pub input_cost_per_token_above_272k_tokens: Option<f64>,
|
||||
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
|
||||
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
|
||||
pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub input_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
pub input_cost_per_query: Option<f64>,
|
||||
pub cost_per_second: Option<f64>,
|
||||
pub input_cost_per_second: Option<f64>,
|
||||
pub input_cost_per_text_message: Option<f64>,
|
||||
pub input_cost_per_audio_token: Option<f64>,
|
||||
pub input_cost_per_audio_token_flex: Option<f64>,
|
||||
pub input_cost_per_audio_token_priority: Option<f64>,
|
||||
pub input_cost_per_image_token: Option<f64>,
|
||||
pub input_cost_per_video_token: Option<f64>,
|
||||
pub input_cost_per_audio_token_batches: Option<f64>,
|
||||
pub input_cost_per_image_token_batches: Option<f64>,
|
||||
pub input_cost_per_image: Option<f64>,
|
||||
pub input_cost_per_audio_per_second: Option<f64>,
|
||||
pub input_cost_per_video_per_second: Option<f64>,
|
||||
pub input_cost_per_token_batches: Option<f64>,
|
||||
pub input_cost_per_video_token_batches: Option<f64>,
|
||||
pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
|
||||
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
|
||||
pub output_cost_per_token_batches: Option<f64>,
|
||||
pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
|
||||
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
|
||||
pub output_cost_per_token: Option<f64>,
|
||||
pub output_cost_per_token_flex: Option<f64>,
|
||||
pub output_cost_per_token_priority: Option<f64>,
|
||||
pub output_cost_per_token_balanced: Option<f64>,
|
||||
pub output_cost_per_token_ultrafast: Option<f64>,
|
||||
pub regional_processing_uplift_multiplier_eu: Option<f64>,
|
||||
pub regional_processing_uplift_multiplier_us: Option<f64>,
|
||||
pub regional_endpoint_uplift_multiplier: Option<f64>,
|
||||
pub output_cost_per_audio_token: Option<f64>,
|
||||
pub output_cost_per_character: Option<f64>,
|
||||
pub output_cost_per_reasoning_token: Option<f64>,
|
||||
pub output_cost_per_reasoning_token_flex: Option<f64>,
|
||||
pub output_cost_per_reasoning_token_priority: Option<f64>,
|
||||
pub output_cost_per_token_above_128k_tokens: Option<f64>,
|
||||
pub output_cost_per_character_above_128k_tokens: Option<f64>,
|
||||
pub output_cost_per_token_above_200k_tokens: Option<f64>,
|
||||
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
|
||||
pub output_cost_per_token_above_272k_tokens: Option<f64>,
|
||||
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
|
||||
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
|
||||
pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub output_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
pub output_cost_per_second: Option<f64>,
|
||||
pub output_cost_per_second_1080p: Option<f64>,
|
||||
pub output_cost_per_second_480p: Option<f64>,
|
||||
pub output_cost_per_second_720p: Option<f64>,
|
||||
pub output_cost_per_second_768p: Option<f64>,
|
||||
pub output_cost_per_second_2k: Option<f64>,
|
||||
pub output_cost_per_second_4k: Option<f64>,
|
||||
pub output_cost_per_video_per_second: Option<f64>,
|
||||
pub output_cost_per_image: Option<f64>,
|
||||
pub output_cost_per_pixel: Option<f64>,
|
||||
pub output_cost_per_image_token: Option<f64>,
|
||||
pub output_cost_per_image_token_batches: Option<f64>,
|
||||
pub output_cost_per_video_token: Option<f64>,
|
||||
pub output_vector_size: Option<i64>,
|
||||
pub citation_cost_per_token: Option<f64>,
|
||||
pub tiered_pricing: Option<Value>,
|
||||
pub litellm_provider: Option<String>,
|
||||
pub mode: Option<Value>,
|
||||
pub supported_endpoints: Option<Vec<String>>,
|
||||
pub supports_system_messages: Option<bool>,
|
||||
pub supports_response_schema: Option<bool>,
|
||||
pub supports_vision: Option<bool>,
|
||||
pub supports_function_calling: Option<bool>,
|
||||
pub supports_parallel_function_calling: Option<bool>,
|
||||
pub supports_tool_choice: Option<bool>,
|
||||
pub supports_assistant_prefill: Option<bool>,
|
||||
pub supports_prompt_caching: Option<bool>,
|
||||
pub supports_prompt_cache_breakpoint: Option<bool>,
|
||||
pub supports_thinking_cache_preservation: Option<bool>,
|
||||
pub supports_audio_input: Option<bool>,
|
||||
pub supports_video_input: Option<bool>,
|
||||
pub supports_box_questions: Option<bool>,
|
||||
pub supports_audio_output: Option<bool>,
|
||||
pub supports_pdf_input: Option<bool>,
|
||||
pub supports_embedding_image_input: Option<bool>,
|
||||
pub supports_native_streaming: Option<bool>,
|
||||
pub supports_native_structured_output: Option<bool>,
|
||||
pub supports_bedrock_native_structured_output: Option<bool>,
|
||||
pub supports_web_search: Option<bool>,
|
||||
pub supports_url_context: Option<bool>,
|
||||
pub supports_reasoning: Option<bool>,
|
||||
pub supports_adaptive_thinking: Option<bool>,
|
||||
pub supports_legacy_thinking: Option<bool>,
|
||||
pub thinking_always_on: Option<bool>,
|
||||
pub supports_tool_search: Option<bool>,
|
||||
pub supports_mid_conversation_system: Option<bool>,
|
||||
pub supports_anthropic_thinking_payload: Option<bool>,
|
||||
pub supports_anthropic_compaction: Option<bool>,
|
||||
pub supports_none_reasoning_effort: Option<bool>,
|
||||
pub supports_minimal_reasoning_effort: Option<bool>,
|
||||
pub supports_low_reasoning_effort: Option<bool>,
|
||||
pub supports_xhigh_reasoning_effort: Option<bool>,
|
||||
pub supports_max_reasoning_effort: Option<bool>,
|
||||
pub reasoning_effort_levels: Option<Value>,
|
||||
pub default_reasoning_effort: Option<Value>,
|
||||
pub bedrock_output_config_effort_ceiling: Option<Value>,
|
||||
pub bedrock_converse_supports_strict_tools: Option<bool>,
|
||||
pub supports_regex_lookaround: Option<bool>,
|
||||
pub supports_computer_use: Option<bool>,
|
||||
pub search_context_cost_per_query: Option<Value>,
|
||||
pub web_search_billing_unit: Option<Value>,
|
||||
pub search_free_requests: Option<Value>,
|
||||
pub google_maps_grounding_cost_per_query: Option<f64>,
|
||||
pub tpm: Option<i64>,
|
||||
pub rpm: Option<i64>,
|
||||
pub ocr_cost_per_page: Option<f64>,
|
||||
pub ocr_cost_per_page_batches: Option<f64>,
|
||||
pub ocr_cost_per_credit: Option<f64>,
|
||||
pub annotation_cost_per_page: Option<f64>,
|
||||
pub annotation_cost_per_page_batches: Option<f64>,
|
||||
pub provider_specific_entry: Option<Value>,
|
||||
pub uses_embed_content: Option<bool>,
|
||||
pub supports_image_size: Option<bool>,
|
||||
pub supported_audio_formats: Option<Value>,
|
||||
pub token_count_framing: Option<i64>,
|
||||
pub token_count_model: Option<Value>,
|
||||
pub token_count_tokenizer: Option<Value>,
|
||||
pub vertex_ai_audio_api: Option<Value>,
|
||||
/// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN`, copied as written
|
||||
pub above_threshold_costs: IndexMap<String, Value>,
|
||||
use crate::wire::Fields;
|
||||
|
||||
macro_rules! model_info_fields {
|
||||
($($name:ident),* $(,)?) => {
|
||||
/// The fields main's `ModelInfoBase(...)` copies in `get_model_info`, one field each, read from the raw row
|
||||
/// without lowering it, so building one never fails and a bad value comes back as written
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
|
||||
pub struct ModelInfoFields {
|
||||
$(pub $name: Value,)*
|
||||
/// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN` the fields above do not name, copied as written
|
||||
#[serde(flatten)]
|
||||
pub above_threshold_costs: IndexMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ModelInfoFields {
|
||||
/// The copied field names in main's order
|
||||
pub const FIELDS: &[&str] = &[$(stringify!($name)),*];
|
||||
|
||||
fn copied(key: &str, fields: &Fields, provider: Option<&str>) -> Self {
|
||||
Self {
|
||||
$($name: copied_field(stringify!($name), key, fields, provider),)*
|
||||
above_threshold_costs: IndexMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
model_info_fields! {
|
||||
key,
|
||||
max_tokens,
|
||||
max_input_tokens,
|
||||
max_output_tokens,
|
||||
input_cost_per_token,
|
||||
input_cost_per_token_flex,
|
||||
input_cost_per_token_priority,
|
||||
input_cost_per_token_balanced,
|
||||
input_cost_per_token_ultrafast,
|
||||
cache_creation_input_token_cost,
|
||||
cache_creation_input_token_cost_above_200k_tokens,
|
||||
cache_creation_input_token_cost_above_272k_tokens,
|
||||
cache_creation_input_token_cost_above_272k_tokens_priority,
|
||||
cache_creation_input_token_cost_above_272k_tokens_flex,
|
||||
cache_creation_input_token_cost_above_272k_tokens_ultrafast,
|
||||
cache_creation_input_token_cost_flex,
|
||||
cache_creation_input_token_cost_priority,
|
||||
cache_creation_input_token_cost_ultrafast,
|
||||
cache_read_input_token_cost,
|
||||
cache_read_input_audio_token_cost,
|
||||
cache_read_input_audio_token_cost_flex,
|
||||
cache_read_input_audio_token_cost_priority,
|
||||
cache_read_input_audio_token_cost_batches,
|
||||
explicit_cache_read_input_token_cost,
|
||||
prompt_cache_min_tokens,
|
||||
cache_read_input_token_cost_above_200k_tokens,
|
||||
cache_read_input_token_cost_above_200k_tokens_priority,
|
||||
cache_read_input_token_cost_above_272k_tokens,
|
||||
cache_read_input_token_cost_above_272k_tokens_priority,
|
||||
cache_read_input_token_cost_above_272k_tokens_flex,
|
||||
cache_read_input_token_cost_above_272k_tokens_ultrafast,
|
||||
cache_read_input_token_cost_above_512k_tokens,
|
||||
cache_read_input_token_cost_flex,
|
||||
cache_read_input_token_cost_priority,
|
||||
cache_read_input_token_cost_balanced,
|
||||
cache_read_input_token_cost_ultrafast,
|
||||
cache_read_input_token_cost_batches,
|
||||
cache_read_input_token_cost_above_200k_tokens_batches,
|
||||
cache_read_input_token_cost_above_272k_tokens_batches,
|
||||
cache_creation_input_token_cost_batches,
|
||||
cache_creation_input_token_cost_above_200k_tokens_batches,
|
||||
cache_creation_input_token_cost_above_272k_tokens_batches,
|
||||
cache_creation_input_token_cost_above_1hr,
|
||||
cache_creation_input_token_cost_above_12hr,
|
||||
cache_creation_input_token_cost_above_12hr_flex,
|
||||
cache_creation_input_token_cost_above_12hr_priority,
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens,
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex,
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority,
|
||||
cache_creation_input_token_cost_above_12hr_batches,
|
||||
cache_creation_input_token_cost_above_12hr_ultrafast,
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches,
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast,
|
||||
off_peak_pricing,
|
||||
input_cost_per_character,
|
||||
input_cost_per_token_above_128k_tokens,
|
||||
input_cost_per_token_above_200k_tokens,
|
||||
input_cost_per_token_above_200k_tokens_priority,
|
||||
input_cost_per_token_above_272k_tokens,
|
||||
input_cost_per_token_above_272k_tokens_priority,
|
||||
input_cost_per_token_above_272k_tokens_flex,
|
||||
input_cost_per_token_above_272k_tokens_ultrafast,
|
||||
input_cost_per_token_above_512k_tokens,
|
||||
input_cost_per_query,
|
||||
cost_per_second,
|
||||
input_cost_per_second,
|
||||
input_cost_per_text_message,
|
||||
input_cost_per_audio_token,
|
||||
input_cost_per_audio_token_flex,
|
||||
input_cost_per_audio_token_priority,
|
||||
input_cost_per_image_token,
|
||||
input_cost_per_video_token,
|
||||
input_cost_per_audio_token_batches,
|
||||
input_cost_per_image_token_batches,
|
||||
input_cost_per_image,
|
||||
input_cost_per_audio_per_second,
|
||||
input_cost_per_video_per_second,
|
||||
input_cost_per_token_batches,
|
||||
input_cost_per_video_token_batches,
|
||||
input_cost_per_token_above_200k_tokens_batches,
|
||||
input_cost_per_token_above_272k_tokens_batches,
|
||||
output_cost_per_token_batches,
|
||||
output_cost_per_token_above_200k_tokens_batches,
|
||||
output_cost_per_token_above_272k_tokens_batches,
|
||||
output_cost_per_token,
|
||||
output_cost_per_token_flex,
|
||||
output_cost_per_token_priority,
|
||||
output_cost_per_token_balanced,
|
||||
output_cost_per_token_ultrafast,
|
||||
regional_processing_uplift_multiplier_eu,
|
||||
regional_processing_uplift_multiplier_us,
|
||||
regional_endpoint_uplift_multiplier,
|
||||
output_cost_per_audio_token,
|
||||
output_cost_per_character,
|
||||
output_cost_per_reasoning_token,
|
||||
output_cost_per_reasoning_token_flex,
|
||||
output_cost_per_reasoning_token_priority,
|
||||
output_cost_per_token_above_128k_tokens,
|
||||
output_cost_per_character_above_128k_tokens,
|
||||
output_cost_per_token_above_200k_tokens,
|
||||
output_cost_per_token_above_200k_tokens_priority,
|
||||
output_cost_per_token_above_272k_tokens,
|
||||
output_cost_per_token_above_272k_tokens_priority,
|
||||
output_cost_per_token_above_272k_tokens_flex,
|
||||
output_cost_per_token_above_272k_tokens_ultrafast,
|
||||
output_cost_per_token_above_512k_tokens,
|
||||
output_cost_per_second,
|
||||
output_cost_per_second_1080p,
|
||||
output_cost_per_second_480p,
|
||||
output_cost_per_second_720p,
|
||||
output_cost_per_second_768p,
|
||||
output_cost_per_second_2k,
|
||||
output_cost_per_second_4k,
|
||||
output_cost_per_video_per_second,
|
||||
output_cost_per_image,
|
||||
output_cost_per_pixel,
|
||||
output_cost_per_image_token,
|
||||
output_cost_per_image_token_batches,
|
||||
output_cost_per_video_token,
|
||||
output_vector_size,
|
||||
citation_cost_per_token,
|
||||
tiered_pricing,
|
||||
litellm_provider,
|
||||
mode,
|
||||
supported_endpoints,
|
||||
supports_system_messages,
|
||||
supports_response_schema,
|
||||
supports_vision,
|
||||
supports_function_calling,
|
||||
supports_parallel_function_calling,
|
||||
supports_tool_choice,
|
||||
supports_assistant_prefill,
|
||||
supports_prompt_caching,
|
||||
supports_prompt_cache_breakpoint,
|
||||
supports_thinking_cache_preservation,
|
||||
supports_audio_input,
|
||||
supports_video_input,
|
||||
supports_box_questions,
|
||||
supports_audio_output,
|
||||
supports_pdf_input,
|
||||
supports_embedding_image_input,
|
||||
supports_native_streaming,
|
||||
supports_native_structured_output,
|
||||
supports_bedrock_native_structured_output,
|
||||
supports_web_search,
|
||||
supports_url_context,
|
||||
supports_reasoning,
|
||||
supports_adaptive_thinking,
|
||||
supports_legacy_thinking,
|
||||
thinking_always_on,
|
||||
supports_tool_search,
|
||||
supports_mid_conversation_system,
|
||||
supports_anthropic_thinking_payload,
|
||||
supports_anthropic_compaction,
|
||||
supports_none_reasoning_effort,
|
||||
supports_minimal_reasoning_effort,
|
||||
supports_low_reasoning_effort,
|
||||
supports_xhigh_reasoning_effort,
|
||||
supports_max_reasoning_effort,
|
||||
reasoning_effort_levels,
|
||||
default_reasoning_effort,
|
||||
bedrock_output_config_effort_ceiling,
|
||||
bedrock_converse_supports_strict_tools,
|
||||
supports_regex_lookaround,
|
||||
supports_computer_use,
|
||||
search_context_cost_per_query,
|
||||
browser_open_cost_per_request,
|
||||
cost_per_additional_result,
|
||||
web_search_billing_unit,
|
||||
search_free_requests,
|
||||
google_maps_grounding_cost_per_query,
|
||||
tpm,
|
||||
rpm,
|
||||
ocr_cost_per_page,
|
||||
ocr_cost_per_page_batches,
|
||||
ocr_cost_per_credit,
|
||||
annotation_cost_per_page,
|
||||
annotation_cost_per_page_batches,
|
||||
provider_specific_entry,
|
||||
uses_embed_content,
|
||||
supports_image_size,
|
||||
supported_audio_formats,
|
||||
token_count_framing,
|
||||
token_count_model,
|
||||
token_count_tokenizer,
|
||||
vertex_ai_audio_api,
|
||||
}
|
||||
|
||||
/// main's `get_model_info_helper` copy of one matched row. `provider` is the requested provider, which a row with no
|
||||
/// `litellm_provider` field reports as its own
|
||||
pub fn model_info(key: &str, fields: &Fields, provider: Option<&str>) -> ModelInfoFields {
|
||||
let copied = ModelInfoFields::copied(key, fields, provider);
|
||||
ModelInfoFields {
|
||||
above_threshold_costs: fields
|
||||
.iter()
|
||||
.filter(|(name, _)| {
|
||||
!ModelInfoFields::FIELDS.contains(&name.as_str())
|
||||
&& is_above_threshold_cost_key(name)
|
||||
})
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect(),
|
||||
..copied
|
||||
}
|
||||
}
|
||||
|
||||
/// One field as main's copy writes it. `max_tokens` is derived, a missing token price defaults to `0`
|
||||
pub fn copied_field(name: &str, key: &str, fields: &Fields, provider: Option<&str>) -> Value {
|
||||
let field = |name: &str| fields.get(name).cloned().unwrap_or(Value::Null);
|
||||
match name {
|
||||
"key" => Value::String(key.to_owned()),
|
||||
"max_tokens" => {
|
||||
let output = field("max_output_tokens");
|
||||
if truthy(&output) {
|
||||
output
|
||||
} else {
|
||||
field("max_input_tokens")
|
||||
}
|
||||
}
|
||||
"input_cost_per_token" | "output_cost_per_token" => match field(name) {
|
||||
Value::Null => Value::from(0),
|
||||
value => value,
|
||||
},
|
||||
"litellm_provider" => fields.get(name).cloned().unwrap_or_else(|| {
|
||||
provider.map_or(Value::Null, |provider| Value::String(provider.to_owned()))
|
||||
}),
|
||||
name => field(name),
|
||||
}
|
||||
}
|
||||
|
||||
/// `ABOVE_THRESHOLD_COST_KEY_PATTERN`, `_above_\d+k?_tokens(?:_batches)?$`
|
||||
pub fn is_above_threshold_cost_key(name: &str) -> bool {
|
||||
let name = name.strip_suffix("_batches").unwrap_or(name);
|
||||
let Some(rest) = name.strip_suffix("_tokens") else {
|
||||
return false;
|
||||
};
|
||||
let rest = rest.strip_suffix('k').unwrap_or(rest);
|
||||
let Some((_, digits)) = rest.rsplit_once("_above_") else {
|
||||
return false;
|
||||
};
|
||||
!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
/// Python truthiness of a JSON value, for main's `a or b`
|
||||
fn truthy(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::Bool(flag) => *flag,
|
||||
Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0),
|
||||
Value::String(text) => !text.is_empty(),
|
||||
Value::Array(items) => !items.is_empty(),
|
||||
Value::Object(map) => !map.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn fields(row: Value) -> Fields {
|
||||
let Value::Object(fields) = row else {
|
||||
unreachable!()
|
||||
};
|
||||
fields
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::output_first(json!({"max_output_tokens": 10, "max_input_tokens": 20}), json!(10))]
|
||||
#[case::zero_output_falls_to_input(json!({"max_output_tokens": 0, "max_input_tokens": 20}), json!(20))]
|
||||
#[case::neither(json!({}), Value::Null)]
|
||||
fn max_tokens_is_output_or_input(#[case] row: Value, #[case] expected: Value) {
|
||||
assert_eq!(model_info("m", &fields(row), None).max_tokens, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_missing_token_price_is_zero_and_a_bad_one_comes_back() {
|
||||
let info = model_info("m", &fields(json!({"input_cost_per_token": "abc"})), None);
|
||||
assert_eq!(info.input_cost_per_token, json!("abc"));
|
||||
assert_eq!(info.output_cost_per_token, json!(0));
|
||||
assert_eq!(info.key, json!("m"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn above_threshold_keys_the_copy_does_not_name_are_kept() {
|
||||
let info = model_info(
|
||||
"m",
|
||||
&fields(json!({"input_cost_per_token_above_999k_tokens": 1.0, "unrelated_field": 2.0})),
|
||||
None,
|
||||
);
|
||||
assert_eq!(
|
||||
info.above_threshold_costs
|
||||
.get("input_cost_per_token_above_999k_tokens"),
|
||||
Some(&json!(1.0))
|
||||
);
|
||||
assert!(!info.above_threshold_costs.contains_key("unrelated_field"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::absent_takes_the_request(json!({}), json!("azure"))]
|
||||
#[case::null_stays_null(json!({"litellm_provider": null}), Value::Null)]
|
||||
#[case::written_wins(json!({"litellm_provider": "openai"}), json!("openai"))]
|
||||
fn a_row_without_a_provider_reports_the_requested_one(
|
||||
#[case] row: Value,
|
||||
#[case] expected: Value,
|
||||
) {
|
||||
assert_eq!(
|
||||
model_info("m", &fields(row), Some("azure")).litellm_provider,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::k("input_cost_per_token_above_200k_tokens", true)]
|
||||
#[case::batches("input_cost_per_token_above_200k_tokens_batches", true)]
|
||||
#[case::plain_count("input_cost_per_token_above_128_tokens", true)]
|
||||
#[case::priority_suffix("input_cost_per_token_above_200k_tokens_priority", false)]
|
||||
#[case::one_hour("cache_creation_input_token_cost_above_1hr", false)]
|
||||
fn the_threshold_key_pattern_matches_mains(#[case] name: &str, #[case] expected: bool) {
|
||||
assert_eq!(is_above_threshold_cost_key(name), expected);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
mod image;
|
||||
mod provider_match;
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
pub use image::*;
|
||||
pub use provider_match::*;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::wire::Fields;
|
||||
|
||||
|
|
@ -59,3 +62,69 @@ pub enum Resolution {
|
|||
tried: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Candidates {
|
||||
/// The names in main's lookup order, each with the slot it fills
|
||||
pub fn in_order(&self) -> impl Iterator<Item = (CandidateSlot, &str)> {
|
||||
[
|
||||
(
|
||||
CandidateSlot::CombinedModelName,
|
||||
self.combined_model_name.as_deref(),
|
||||
),
|
||||
(CandidateSlot::Model, Some(self.model.as_str())),
|
||||
(
|
||||
CandidateSlot::RegionFreeCombinedModelName,
|
||||
self.region_free_combined_model_name.as_deref(),
|
||||
),
|
||||
(CandidateSlot::SplitModel, self.split_model.as_deref()),
|
||||
(
|
||||
CandidateSlot::CombinedStrippedModelName,
|
||||
self.combined_stripped_model_name.as_deref(),
|
||||
),
|
||||
(
|
||||
CandidateSlot::StrippedModelName,
|
||||
self.stripped_model_name.as_deref(),
|
||||
),
|
||||
(
|
||||
CandidateSlot::ProviderPrefixedModelName,
|
||||
self.provider_prefixed_model_name.as_deref(),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(slot, name)| Some((slot, name?)))
|
||||
}
|
||||
}
|
||||
|
||||
/// main's candidate walk. Each name is tried as an exact key, then through the case-insensitive index, and the first
|
||||
/// whose row passes `provider_matches` wins
|
||||
pub fn resolve(keys: &impl Keys, candidates: &Candidates, provider: Option<&str>) -> Resolution {
|
||||
let found = candidates.in_order().find_map(|(slot, name)| {
|
||||
let (key, case_insensitive) = if keys.contains(name) {
|
||||
(name.to_owned(), false)
|
||||
} else {
|
||||
(keys.by_normalized(name)?, true)
|
||||
};
|
||||
let row = keys.row(&key)?;
|
||||
row_provider_matches(&row, provider).then_some(Resolution::Found {
|
||||
key,
|
||||
via: Via {
|
||||
slot,
|
||||
case_insensitive,
|
||||
},
|
||||
})
|
||||
});
|
||||
found.unwrap_or_else(|| Resolution::Missing {
|
||||
tried: candidates
|
||||
.in_order()
|
||||
.map(|(_, name)| name.to_owned())
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn row_provider_matches(row: &Fields, provider: Option<&str>) -> bool {
|
||||
match row.get("litellm_provider") {
|
||||
None | Some(Value::Null) => provider_matches(None, provider),
|
||||
Some(Value::String(row_provider)) => provider_matches(Some(row_provider), provider),
|
||||
Some(_) => provider_matches(Some(""), provider),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/// main's `_check_provider_match`. A row with no provider matches any request, and a few requested providers reuse
|
||||
/// another provider's rows
|
||||
pub fn provider_matches(row_provider: Option<&str>, requested: Option<&str>) -> bool {
|
||||
let (Some(row), Some(requested)) = (
|
||||
row_provider,
|
||||
requested.filter(|requested| !requested.is_empty()),
|
||||
) else {
|
||||
return true;
|
||||
};
|
||||
if row == requested {
|
||||
return true;
|
||||
}
|
||||
let same_family = |family: &str| requested == family && row.starts_with(family);
|
||||
same_family("vertex_ai")
|
||||
|| same_family("fireworks_ai")
|
||||
|| (requested.starts_with("bedrock") && row.starts_with("bedrock"))
|
||||
|| matches!(requested, "litellm_proxy" | "github" | "nadir")
|
||||
|| (requested == "azure_ai" && matches!(row, "azure" | "openai"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::no_row_provider(None, Some("openai"), true)]
|
||||
#[case::no_request(Some("openai"), None, true)]
|
||||
#[case::empty_request(Some("openai"), Some(""), true)]
|
||||
#[case::same(Some("openai"), Some("openai"), true)]
|
||||
#[case::other(Some("anthropic"), Some("openai"), false)]
|
||||
#[case::vertex_family(Some("vertex_ai-language-models"), Some("vertex_ai"), true)]
|
||||
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
|
||||
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
|
||||
#[case::bedrock_family(Some("bedrock"), Some("bedrock_converse"), true)]
|
||||
#[case::proxy_reuses_anything(Some("anthropic"), Some("litellm_proxy"), true)]
|
||||
#[case::github_reuses_anything(Some("anthropic"), Some("github"), true)]
|
||||
#[case::nadir_reuses_anything(Some("anthropic"), Some("nadir"), true)]
|
||||
#[case::azure_ai_reuses_azure(Some("azure"), Some("azure_ai"), true)]
|
||||
#[case::azure_ai_reuses_openai(Some("openai"), Some("azure_ai"), true)]
|
||||
#[case::azure_ai_not_anthropic(Some("anthropic"), Some("azure_ai"), false)]
|
||||
fn follows_mains_table(
|
||||
#[case] row: Option<&str>,
|
||||
#[case] requested: Option<&str>,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(provider_matches(row, requested), expected);
|
||||
}
|
||||
}
|
||||
107
litellm-rust/crates/model-catalog/tests/resolve.rs
Normal file
107
litellm-rust/crates/model-catalog/tests/resolve.rs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{CandidateSlot, Candidates, Keys, Resolution, RowRef, Via, resolve};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Map(BTreeMap<String, Fields>);
|
||||
|
||||
impl Keys for Map {
|
||||
fn contains(&self, key: &str) -> bool {
|
||||
self.0.contains_key(key)
|
||||
}
|
||||
|
||||
fn row(&self, key: &str) -> Option<RowRef<'_>> {
|
||||
self.0.get(key).map(Cow::Borrowed)
|
||||
}
|
||||
|
||||
fn by_normalized(&self, name: &str) -> Option<String> {
|
||||
let lower = name.to_lowercase();
|
||||
self.0
|
||||
.keys()
|
||||
.find(|key| key.to_lowercase() == lower)
|
||||
.cloned()
|
||||
}
|
||||
}
|
||||
|
||||
fn map(rows: Value) -> Map {
|
||||
let Value::Object(rows) = rows else {
|
||||
unreachable!()
|
||||
};
|
||||
Map(rows
|
||||
.into_iter()
|
||||
.map(|(key, row)| {
|
||||
let Value::Object(fields) = row else {
|
||||
unreachable!()
|
||||
};
|
||||
(key, fields)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn candidates(model: &str, combined: &str, split: &str) -> Candidates {
|
||||
Candidates {
|
||||
combined_model_name: Some(combined.to_owned()),
|
||||
model: model.to_owned(),
|
||||
region_free_combined_model_name: Some(combined.to_owned()),
|
||||
split_model: Some(split.to_owned()),
|
||||
combined_stripped_model_name: Some(combined.to_owned()),
|
||||
stripped_model_name: Some(split.to_owned()),
|
||||
provider_prefixed_model_name: Some(combined.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn found(key: &str, slot: CandidateSlot, case_insensitive: bool) -> Resolution {
|
||||
Resolution::Found {
|
||||
key: key.to_owned(),
|
||||
via: Via {
|
||||
slot,
|
||||
case_insensitive,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::provider_prefixed_row_first(
|
||||
json!({"groq/llama": {"litellm_provider": "groq"}, "llama": {"litellm_provider": "groq"}}),
|
||||
found("groq/llama", CandidateSlot::CombinedModelName, false)
|
||||
)]
|
||||
#[case::bare_row_when_no_prefixed_one(
|
||||
json!({"llama": {"litellm_provider": "groq"}}),
|
||||
found("llama", CandidateSlot::Model, false)
|
||||
)]
|
||||
#[case::a_mismatched_provider_is_skipped(
|
||||
json!({"groq/llama": {"litellm_provider": "openai"}, "llama": {"litellm_provider": "groq"}}),
|
||||
found("llama", CandidateSlot::Model, false)
|
||||
)]
|
||||
#[case::case_insensitive_through_the_index(
|
||||
json!({"Groq/Llama": {"litellm_provider": "groq"}}),
|
||||
found("Groq/Llama", CandidateSlot::CombinedModelName, true)
|
||||
)]
|
||||
fn the_first_matching_candidate_wins(#[case] rows: Value, #[case] expected: Resolution) {
|
||||
assert_eq!(
|
||||
resolve(
|
||||
&map(rows),
|
||||
&candidates("llama", "groq/llama", "llama"),
|
||||
Some("groq")
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_miss_lists_every_candidate_tried() {
|
||||
let Resolution::Missing { tried } = resolve(
|
||||
&map(json!({})),
|
||||
&candidates("x", "acme/x", "x"),
|
||||
Some("acme"),
|
||||
) else {
|
||||
panic!("expected a miss");
|
||||
};
|
||||
assert_eq!(
|
||||
tried,
|
||||
["acme/x", "x", "acme/x", "x", "acme/x", "x", "acme/x"]
|
||||
);
|
||||
}
|
||||
|
|
@ -35,7 +35,8 @@ mod _native {
|
|||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{
|
||||
Bundled, CatalogHandle, Fetched, File, Installed, Provenance, Remote, RowError, Unavailable,
|
||||
Bundled, CatalogHandle, Fetched, File, Installed, Invalid, Missing, Provenance, Remote,
|
||||
Resolved, RowError, Unavailable,
|
||||
};
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
|
|
@ -163,6 +164,9 @@ mod tests {
|
|||
"File",
|
||||
"Bundled",
|
||||
"Provenance",
|
||||
"Resolved",
|
||||
"Missing",
|
||||
"Invalid",
|
||||
"RowError",
|
||||
"RowErrorKind",
|
||||
"NativeClickHouseSpendConfig",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
mod fetch;
|
||||
mod install;
|
||||
mod name_index;
|
||||
mod resolve;
|
||||
mod row_error;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
|
@ -8,7 +10,7 @@ use std::time::SystemTime;
|
|||
|
||||
use litellm_host_python::{release_gil, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Layers, Located};
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Keys, Layers, Located};
|
||||
use litellm_model_catalog_refresh::{Source, git_blob_id};
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
use pyo3::prelude::*;
|
||||
|
|
@ -17,6 +19,7 @@ use serde::Serialize;
|
|||
|
||||
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
|
||||
use install::{Record, write_into};
|
||||
pub(crate) use resolve::{Invalid, Missing, Resolved};
|
||||
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
|
||||
|
||||
static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
|
||||
|
|
@ -90,6 +93,7 @@ pub(crate) struct CatalogHandle {
|
|||
module: Py<PyModule>,
|
||||
attr: String,
|
||||
state: Mutex<State>,
|
||||
index: Mutex<Option<name_index::NameIndex>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
|
|
@ -151,6 +155,7 @@ impl CatalogHandle {
|
|||
record: None,
|
||||
provenance: None,
|
||||
}),
|
||||
index: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -185,6 +190,70 @@ impl CatalogHandle {
|
|||
.unbind())
|
||||
}
|
||||
|
||||
/// The bound dict's row for `model`, through main's candidate names (decisions 18 and 20)
|
||||
#[pyo3(signature = (model, provider = None))]
|
||||
fn resolve(&self, py: Python<'_>, model: &str, provider: Option<&str>) -> PyResult<Py<PyAny>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
resolve::resolve(py, &keys, model, provider)
|
||||
}
|
||||
|
||||
/// The actual key for `name`, exact first, then through the case-insensitive index
|
||||
fn lookup_key(&self, py: Python<'_>, name: &str) -> PyResult<Option<String>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
Ok(if keys.contains(name) {
|
||||
Some(name.to_owned())
|
||||
} else {
|
||||
keys.by_normalized(name)
|
||||
})
|
||||
}
|
||||
|
||||
/// The keys whose row names this `litellm_provider`, from the index
|
||||
fn provider_keys(&self, py: Python<'_>, provider: &str) -> PyResult<Vec<String>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
Ok(keys.provider_keys(provider))
|
||||
}
|
||||
|
||||
/// main's 133-field copy of a row Python built, a generalization match or a fill-missing merge
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (key, row, provider = None))]
|
||||
fn model_info_of(
|
||||
py: Python<'_>,
|
||||
key: &str,
|
||||
row: &Bound<'_, PyDict>,
|
||||
provider: Option<&str>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let fields: Fields = row
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
Some((
|
||||
name.extract::<String>().ok()?,
|
||||
litellm_host_python::from_py::<serde_json::Value>(&value).ok()?,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
to_py(
|
||||
py,
|
||||
&litellm_model_catalog::model_info(key, &fields, provider),
|
||||
)
|
||||
}
|
||||
|
||||
/// Rebuild the index on the next lookup, for a caller that knows keys changed with the length unchanged
|
||||
fn mark_index_stale(&self) {
|
||||
*self.index.lock().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
|
||||
/// Where the installed map came from, `None` before the first install
|
||||
fn provenance(&self) -> Option<Provenance> {
|
||||
self.lock().provenance.clone()
|
||||
|
|
@ -235,6 +304,13 @@ impl CatalogHandle {
|
|||
}
|
||||
|
||||
impl CatalogHandle {
|
||||
fn bound_dict<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
|
||||
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
|
||||
bound
|
||||
.cast_into::<PyDict>()
|
||||
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))
|
||||
}
|
||||
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyString};
|
||||
|
||||
/// The one name index of decision 18. A normalized name to the actual key, and the keys each raw `litellm_provider`
|
||||
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct NameIndex {
|
||||
dict_id: usize,
|
||||
len: usize,
|
||||
by_lower: HashMap<String, String>,
|
||||
by_provider: HashMap<String, Vec<String>>,
|
||||
}
|
||||
|
||||
impl NameIndex {
|
||||
pub(crate) fn build(dict: &Bound<'_, PyDict>) -> Self {
|
||||
let mut by_lower = HashMap::with_capacity(dict.len());
|
||||
let mut by_provider: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for (key, row) in dict.iter() {
|
||||
let Ok(key) = key.cast::<PyString>().map(|key| key.to_string()) else {
|
||||
continue;
|
||||
};
|
||||
let provider = row
|
||||
.cast::<PyDict>()
|
||||
.ok()
|
||||
.and_then(|row| row.get_item("litellm_provider").ok().flatten())
|
||||
.and_then(|provider| provider.extract::<String>().ok());
|
||||
if let Some(provider) = provider {
|
||||
by_provider.entry(provider).or_default().push(key.clone());
|
||||
}
|
||||
by_lower.insert(key.to_lowercase(), key);
|
||||
}
|
||||
Self {
|
||||
dict_id: dict.as_ptr() as usize,
|
||||
len: dict.len(),
|
||||
by_lower,
|
||||
by_provider,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_current(&self, dict: &Bound<'_, PyDict>) -> bool {
|
||||
self.dict_id == dict.as_ptr() as usize && self.len == dict.len()
|
||||
}
|
||||
|
||||
pub(crate) fn by_lower(&self, name: &str) -> Option<&str> {
|
||||
self.by_lower.get(&name.to_lowercase()).map(String::as_str)
|
||||
}
|
||||
|
||||
pub(crate) fn provider_keys(&self, provider: &str) -> &[String] {
|
||||
self.by_provider.get(provider).map_or(&[], Vec::as_slice)
|
||||
}
|
||||
}
|
||||
201
litellm-rust/crates/python-bridge/src/model_registry/resolve.rs
Normal file
201
litellm-rust/crates/python-bridge/src/model_registry/resolve.rs
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
use std::borrow::Cow;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use litellm_host_python::{from_py, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{
|
||||
Candidates, Catalog, Keys, Resolution, RowRef, model_info, rejects_row,
|
||||
};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::name_index::NameIndex;
|
||||
|
||||
const CANDIDATES_MODULE: &str = "litellm.rust_bridge.host.model_names";
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Resolved {
|
||||
#[pyo3(get)]
|
||||
key: String,
|
||||
/// The row as it was when `resolve` ran
|
||||
fields: Arc<Fields>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Missing {
|
||||
#[pyo3(get)]
|
||||
tried: Vec<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Invalid {
|
||||
#[pyo3(get)]
|
||||
key: String,
|
||||
#[pyo3(get)]
|
||||
errors: Vec<String>,
|
||||
fields: Arc<Fields>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Resolved {
|
||||
#[pyo3(signature = (provider = None))]
|
||||
fn model_info(&self, py: Python<'_>, provider: Option<&str>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, &model_info(&self.key, &self.fields, provider))
|
||||
}
|
||||
|
||||
fn supports(&self, capability: &str) -> bool {
|
||||
self.fields.get(capability) == Some(&Value::Bool(true))
|
||||
}
|
||||
|
||||
/// The row as `resolve` read it, for the Python steps that still merge into it
|
||||
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, self.fields.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Invalid {
|
||||
#[pyo3(signature = (provider = None))]
|
||||
fn model_info(&self, py: Python<'_>, provider: Option<&str>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, &model_info(&self.key, &self.fields, provider))
|
||||
}
|
||||
|
||||
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, self.fields.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
/// The live map as `resolve` sees it, the dict bound when the call started (decision 1)
|
||||
pub(crate) struct LiveKeys<'a, 'py> {
|
||||
pub(crate) dict: &'a Bound<'py, PyDict>,
|
||||
pub(crate) index: &'a Mutex<Option<NameIndex>>,
|
||||
}
|
||||
|
||||
impl LiveKeys<'_, '_> {
|
||||
fn indexed<T>(&self, read: impl Fn(&NameIndex) -> T) -> T {
|
||||
let mut index = self.index.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if !index
|
||||
.as_ref()
|
||||
.is_some_and(|index| index.is_current(self.dict))
|
||||
{
|
||||
*index = Some(NameIndex::build(self.dict));
|
||||
}
|
||||
read(index.as_ref().expect("the index was just built"))
|
||||
}
|
||||
|
||||
fn rebuild(&self) {
|
||||
*self.index.lock().unwrap_or_else(PoisonError::into_inner) =
|
||||
Some(NameIndex::build(self.dict));
|
||||
}
|
||||
|
||||
pub(crate) fn provider_keys(&self, provider: &str) -> Vec<String> {
|
||||
self.indexed(|index| index.provider_keys(provider).to_vec())
|
||||
}
|
||||
|
||||
/// Every field that converts to JSON. A value only Python can hold is left out, and reported by `full_row`
|
||||
fn fields(&self, key: &str) -> Option<(Fields, bool)> {
|
||||
let row = self.dict.get_item(key).ok().flatten()?;
|
||||
let row = row.cast::<PyDict>().ok()?;
|
||||
if let Ok(fields) = from_py::<Fields>(row) {
|
||||
return Some((fields, true));
|
||||
}
|
||||
let fields = row
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
Some((
|
||||
name.extract::<String>().ok()?,
|
||||
from_py::<Value>(&value).ok()?,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
Some((fields, false))
|
||||
}
|
||||
}
|
||||
|
||||
impl Keys for LiveKeys<'_, '_> {
|
||||
fn contains(&self, key: &str) -> bool {
|
||||
self.dict.contains(key).unwrap_or(false)
|
||||
}
|
||||
|
||||
fn row(&self, key: &str) -> Option<RowRef<'_>> {
|
||||
self.fields(key).map(|(fields, _)| Cow::Owned(fields))
|
||||
}
|
||||
|
||||
fn by_normalized(&self, name: &str) -> Option<String> {
|
||||
let hit = self.indexed(|index| index.by_lower(name).map(str::to_owned))?;
|
||||
if self.contains(&hit) {
|
||||
return Some(hit);
|
||||
}
|
||||
self.rebuild();
|
||||
self.indexed(|index| index.by_lower(name).map(str::to_owned))
|
||||
.filter(|key| self.contains(key))
|
||||
}
|
||||
}
|
||||
|
||||
/// `get_model_info`'s lookup. Python builds the candidate names, Rust walks them over the live dict
|
||||
pub(crate) fn resolve(
|
||||
py: Python<'_>,
|
||||
keys: &LiveKeys<'_, '_>,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let (candidates, provider) = candidates(py, model, provider)?;
|
||||
match litellm_model_catalog::resolve(keys, &candidates, provider.as_deref()) {
|
||||
Resolution::Missing { tried } => {
|
||||
Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind())
|
||||
}
|
||||
Resolution::Found { key, .. } => {
|
||||
let (fields, converted) = keys.fields(&key).unwrap_or_default();
|
||||
let errors: Vec<String> = if converted {
|
||||
Catalog::row_errors(&key, &fields)
|
||||
.iter()
|
||||
.filter(|error| rejects_row(&error.error))
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
} else {
|
||||
vec![format!("{key} must contain JSON-compatible values")]
|
||||
};
|
||||
let fields = Arc::new(fields);
|
||||
if errors.is_empty() {
|
||||
Ok(Resolved { key, fields }
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
} else {
|
||||
Ok(Invalid {
|
||||
key,
|
||||
errors,
|
||||
fields,
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidates(
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
) -> PyResult<(Candidates, Option<String>)> {
|
||||
let names = py
|
||||
.import(CANDIDATES_MODULE)?
|
||||
.call_method1("candidates", (model, provider))?;
|
||||
let names = names.cast::<PyTuple>()?;
|
||||
let name = |index: usize| -> PyResult<Option<String>> { names.get_item(index)?.extract() };
|
||||
Ok((
|
||||
Candidates {
|
||||
combined_model_name: name(0)?,
|
||||
model: model.to_owned(),
|
||||
region_free_combined_model_name: name(1)?,
|
||||
split_model: name(2)?,
|
||||
combined_stripped_model_name: name(3)?,
|
||||
stripped_model_name: name(4)?,
|
||||
provider_prefixed_model_name: name(5)?,
|
||||
},
|
||||
name(6)?,
|
||||
))
|
||||
}
|
||||
|
|
@ -11,8 +11,11 @@ from litellm.rust_bridge._native import (
|
|||
Fetched,
|
||||
File,
|
||||
Installed,
|
||||
Invalid,
|
||||
Missing,
|
||||
Provenance,
|
||||
Remote,
|
||||
Resolved,
|
||||
RowError,
|
||||
Unavailable,
|
||||
)
|
||||
|
|
@ -38,8 +41,11 @@ __all__ = [
|
|||
"Fetched",
|
||||
"File",
|
||||
"Installed",
|
||||
"Invalid",
|
||||
"Missing",
|
||||
"Provenance",
|
||||
"Remote",
|
||||
"Resolved",
|
||||
"RowError",
|
||||
"Source",
|
||||
"Unavailable",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
import contextlib
|
||||
import contextvars
|
||||
from collections.abc import Generator
|
||||
from typing import Final, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -17,6 +20,18 @@ from litellm.secret_managers.main import get_secret, get_secret_str
|
|||
|
||||
from ..types.router import GenericLiteLLMParams, LiteLLM_Params
|
||||
|
||||
_PROVIDER_BANNER: Final = contextvars.ContextVar("litellm_provider_banner", default=True)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def provider_banner_suppressed() -> Generator[None]:
|
||||
"""A model-info lookup tries names that need not route anywhere, so a miss there prints nothing (#5982)"""
|
||||
token: Final = _PROVIDER_BANNER.set(False)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_PROVIDER_BANNER.reset(token)
|
||||
|
||||
|
||||
def invalid_provider_routing_hints_message(error: ValidationError) -> str:
|
||||
return "; ".join(
|
||||
|
|
@ -553,7 +568,7 @@ def get_llm_provider(
|
|||
custom_llm_provider = match_routing_generalization(model)
|
||||
|
||||
if not custom_llm_provider:
|
||||
if litellm.suppress_debug_info is False:
|
||||
if litellm.suppress_debug_info is False and _PROVIDER_BANNER.get():
|
||||
print() # noqa: T201
|
||||
print( # noqa: T201
|
||||
"\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m"
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ from litellm.types.utils import (
|
|||
ProviderSpecificModelInfo,
|
||||
)
|
||||
from litellm.utils import (
|
||||
get_model_cost_mutation_generation,
|
||||
supports_function_calling,
|
||||
supports_reasoning,
|
||||
supports_tool_choice,
|
||||
|
|
@ -528,36 +527,22 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
# entries), so we memoize it. The cache key is (id(model_cost),
|
||||
# mutation_generation): the generation counter is bumped on every
|
||||
# register_model / reload path, so add+remove or in-place value
|
||||
# replacement (which can leave id and len unchanged) still invalidates.
|
||||
_fireworks_index_cache: tuple[int, int, list[tuple[str, dict]]] | None = None
|
||||
|
||||
@classmethod
|
||||
def _get_fireworks_index(cls) -> list[tuple[str, dict]]:
|
||||
"""The ``fireworks_ai/*`` rows of the live map, short name first. The keys come from the catalog's name index,
|
||||
which follows keys added or removed by direct writes, and the rows are read live"""
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
model_cost: Final = litellm.model_cost
|
||||
signature: Final = (id(model_cost), get_model_cost_mutation_generation())
|
||||
cached: Final = cls._fireworks_index_cache
|
||||
if cached is not None and cached[0] == signature[0] and cached[1] == signature[1]:
|
||||
return cached[2]
|
||||
|
||||
index: Final[list[tuple[str, dict]]] = []
|
||||
for key, model_info in (*model_cost.items(),):
|
||||
if not key.startswith("fireworks_ai/"):
|
||||
continue
|
||||
if not isinstance(model_info, dict):
|
||||
continue
|
||||
key_short = key[len("fireworks_ai/") :]
|
||||
key_short = key_short.removeprefix("accounts/fireworks/models/")
|
||||
if not key_short:
|
||||
continue
|
||||
index.append((key_short, model_info))
|
||||
|
||||
cls._fireworks_index_cache = (signature[0], signature[1], index)
|
||||
return index
|
||||
keys: Final = (*CATALOG.provider_keys("fireworks_ai"), *CATALOG.provider_keys("fireworks_ai-embedding-models"))
|
||||
rows: Final = ((key, model_cost.get(key)) for key in keys)
|
||||
return [
|
||||
(key_short, model_info)
|
||||
for key, model_info in rows
|
||||
if key.startswith("fireworks_ai/")
|
||||
and isinstance(model_info, dict)
|
||||
and (key_short := key[len("fireworks_ai/") :].removeprefix("accounts/fireworks/models/"))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _matches_on_hyphen_boundary(short_name: str, key_short: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -100,6 +100,24 @@ class Provenance:
|
|||
@property
|
||||
def loaded_at(self) -> datetime: ...
|
||||
|
||||
@final
|
||||
class Resolved:
|
||||
key: str
|
||||
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
|
||||
def supports(self, capability: str) -> bool: ...
|
||||
def row(self) -> dict[str, JsonValue]: ...
|
||||
|
||||
@final
|
||||
class Missing:
|
||||
tried: list[str]
|
||||
|
||||
@final
|
||||
class Invalid:
|
||||
key: str
|
||||
errors: list[str]
|
||||
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
|
||||
def row(self) -> dict[str, JsonValue]: ...
|
||||
|
||||
@final
|
||||
class Catalog:
|
||||
@staticmethod
|
||||
|
|
@ -108,6 +126,12 @@ class Catalog:
|
|||
def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ...
|
||||
def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ...
|
||||
def provenance(self) -> Provenance | None: ...
|
||||
def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ...
|
||||
def lookup_key(self, name: str) -> str | None: ...
|
||||
def provider_keys(self, provider: str) -> list[str]: ...
|
||||
def mark_index_stale(self) -> None: ...
|
||||
@staticmethod
|
||||
def model_info_of(key: str, row: Mapping[str, object], provider: str | None = None) -> dict[str, JsonValue]: ...
|
||||
def install(
|
||||
self,
|
||||
fetched: Fetched,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Python state Rust needs but does not own, behind one module per question. Rust imports a module here by path and calls a function by name
|
||||
|
||||
- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds)
|
||||
- One module answers one question: `model_capabilities` (what a model can do), `callbacks_legacy_python` (the legacy callback registries), `settings` (configured snapshots), `cache` (the live configured cache), `diagnostics` (legacy failure logging), `secret_manager` (the configured secret manager and native config capture), `http_transport` (which async transport is in use), `timeouts` (timeout to seconds), `model_names` (the names `get_model_info` tries for a model)
|
||||
- A module is the only place Rust reads that Python state. Rust never does `py.import("litellm").getattr(...)` on a global; it asks here
|
||||
- Each module reuses the Python implementation the Python route runs (`AnthropicModelInfo`, `Logging`). Do not re-derive those rules here or in Rust; one resolver, two callers
|
||||
- The module path and function signature are a contract. The Rust side pins the path in a `const MODULE` in the crate that asks (`python-bridge/src/routes/messages/host.rs`, `python-bridge/src/secrets/`, `callbacks-legacy-python/src/python.rs`); change both sides together and keep `tests/unit/rust_bridge/host/` green
|
||||
|
|
|
|||
29
litellm/rust_bridge/host/model_names.py
Normal file
29
litellm/rust_bridge/host/model_names.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
"""The candidate names ``get_model_info`` tries for a model, for the Rust resolver"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import provider_banner_suppressed
|
||||
|
||||
|
||||
def candidates(
|
||||
model: str, custom_llm_provider: str | None
|
||||
) -> tuple[str | None, str | None, str | None, str | None, str | None, str | None, str | None]:
|
||||
"""Main's ``get_potential_model_names`` with the provider banner suppressed. The names come back in the order
|
||||
``combined_model_name``, ``region_free_combined_model_name``, ``split_model``, ``combined_stripped_model_name``,
|
||||
``stripped_model_name``, ``provider_prefixed_model_name``, then the provider rows must match"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.utils import get_potential_model_names
|
||||
|
||||
with provider_banner_suppressed():
|
||||
names: Final = get_potential_model_names(
|
||||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
return (
|
||||
names["combined_model_name"],
|
||||
names["region_free_combined_model_name"],
|
||||
names["split_model"],
|
||||
names["combined_stripped_model_name"],
|
||||
names["stripped_model_name"],
|
||||
names["provider_prefixed_model_name"],
|
||||
names["custom_llm_provider"],
|
||||
)
|
||||
557
litellm/utils.py
557
litellm/utils.py
|
|
@ -231,7 +231,6 @@ from litellm.types.llms.openai import (
|
|||
OpenAIWebSearchOptions,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ABOVE_THRESHOLD_COST_KEY_PATTERN,
|
||||
API_KEY_UPSTREAM_PARAMS,
|
||||
OPENAI_RESPONSE_HEADERS,
|
||||
AdminApiKeySecretReference,
|
||||
|
|
@ -277,7 +276,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
|
||||
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
|
||||
_BACKFILL_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
|
||||
|
||||
# +-----------------------------------------------+
|
||||
# | |
|
||||
|
|
@ -3095,7 +3094,10 @@ def supports_factory(model: str, custom_llm_provider: str | None, key: str) -> b
|
|||
Raises:
|
||||
Exception: If the given model is not found or there's an error in retrieval.
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
provider_banner_suppressed,
|
||||
)
|
||||
|
||||
try:
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
|
|
@ -3103,9 +3105,10 @@ def supports_factory(model: str, custom_llm_provider: str | None, key: str) -> b
|
|||
model = model.removeprefix(f"{declared}/")
|
||||
custom_llm_provider = declared # rebind-ok: same
|
||||
else:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
with provider_banner_suppressed():
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
model_info: Final = get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
|
@ -6179,131 +6182,24 @@ def _strip_model_name(model: str, custom_llm_provider: str | None) -> str:
|
|||
return _strip_dated_snapshot_suffix(model_name=model)
|
||||
|
||||
|
||||
# Global case-insensitive lookup map for model_cost (built eagerly at module import)
|
||||
_model_cost_lowercase_map: dict[str, str] | None = None
|
||||
|
||||
# Monotonic counter bumped on every model_cost mutation. Consumers that
|
||||
# memoize derived state (e.g. provider-specific indices) can include this
|
||||
# value in their cache key so they invalidate even when key add+remove or
|
||||
# in-place value replacement leaves len/id unchanged.
|
||||
_model_cost_mutation_generation: int = 0
|
||||
|
||||
|
||||
def get_model_cost_mutation_generation() -> int:
|
||||
return _model_cost_mutation_generation
|
||||
|
||||
|
||||
def _invalidate_model_cost_lowercase_map() -> None:
|
||||
"""Invalidate the case-insensitive lookup map for model_cost.
|
||||
"""Kept for callers on main. Reads go to the live dict, so only a key swap that leaves the length unchanged needs
|
||||
the name index rebuilt, and this marks it stale"""
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
Call this whenever litellm.model_cost is modified to ensure the map is rebuilt.
|
||||
Also clears related LRU caches that depend on model_cost data.
|
||||
"""
|
||||
global _model_cost_lowercase_map, _model_cost_mutation_generation
|
||||
_model_cost_lowercase_map = None
|
||||
_model_cost_mutation_generation += 1
|
||||
|
||||
# Clear LRU caches that depend on model_cost data
|
||||
_cached_get_model_info.cache_clear()
|
||||
cached_get_model_info_helper.cache_clear()
|
||||
|
||||
|
||||
def _rebuild_model_cost_lowercase_map() -> dict[str, str]:
|
||||
"""Rebuild the case-insensitive lookup map from the current model_cost.
|
||||
|
||||
Returns:
|
||||
The rebuilt map (guaranteed to be not None).
|
||||
"""
|
||||
global _model_cost_lowercase_map
|
||||
_model_cost_lowercase_map = {k.lower(): k for k in tuple(litellm.model_cost)}
|
||||
return _model_cost_lowercase_map
|
||||
|
||||
|
||||
def _handle_stale_map_entry_rebuild(
|
||||
potential_key_lower: str,
|
||||
) -> str | None:
|
||||
"""
|
||||
Handle stale _model_cost_lowercase_map entry (key was popped).
|
||||
|
||||
Rebuilds the map and retries the lookup.
|
||||
|
||||
Returns:
|
||||
The matched key if found after rebuild, None otherwise.
|
||||
"""
|
||||
matched_key: Final = _rebuild_model_cost_lowercase_map().get(potential_key_lower)
|
||||
if matched_key is not None and matched_key in litellm.model_cost:
|
||||
return matched_key
|
||||
return None
|
||||
|
||||
|
||||
def _handle_new_key_with_scan(
|
||||
potential_key_lower: str,
|
||||
) -> str | None:
|
||||
"""
|
||||
Handle new key added to model_cost without invalidating _model_cost_lowercase_map.
|
||||
|
||||
Scans model_cost for case-insensitive match and rebuilds the map if found.
|
||||
|
||||
Returns:
|
||||
The matched key if found, None otherwise.
|
||||
"""
|
||||
global _model_cost_lowercase_map
|
||||
for key in tuple(_builtin_cost_map()):
|
||||
if key.lower() == potential_key_lower:
|
||||
_model_cost_lowercase_map = _rebuild_model_cost_lowercase_map()
|
||||
return key
|
||||
return None
|
||||
CATALOG.mark_index_stale()
|
||||
|
||||
|
||||
def get_model_cost_key(potential_key: str) -> str | None:
|
||||
"""
|
||||
Get the actual key from model_cost, with case-insensitive fallback.
|
||||
"""The actual key in ``litellm.model_cost`` for ``potential_key``, exact first, then case-insensitive"""
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
WARNING: Only O(1) lookup operations are acceptable. O(n) lookups will cause severe
|
||||
CPU overhead. This function is called frequently during router operations.
|
||||
|
||||
ALLOWED HELPER FUNCTIONS (conditionally called, O(n) operations are acceptable):
|
||||
- _rebuild_model_cost_lowercase_map: Rebuilds the lookup map (only when map is None)
|
||||
- _handle_stale_map_entry_rebuild: Rebuilds map when stale entry detected (rare case)
|
||||
|
||||
If you need to add a new helper function with O(n) operations that is conditionally
|
||||
called and confirmed not to cause performance issues, add it to the allowed_helpers
|
||||
list in: tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
||||
"""
|
||||
# Exact match (O(1))
|
||||
if potential_key in litellm.model_cost:
|
||||
return potential_key
|
||||
|
||||
# Case-insensitive lookup via map (O(1)); bound locally because another thread's
|
||||
# registration can invalidate the module-level map at any point
|
||||
cached_lowercase_map: Final = _model_cost_lowercase_map
|
||||
lowercase_map: Final = (
|
||||
cached_lowercase_map if cached_lowercase_map is not None else _rebuild_model_cost_lowercase_map()
|
||||
)
|
||||
|
||||
potential_key_lower: Final = potential_key.lower()
|
||||
matched_key = lowercase_map.get(potential_key_lower)
|
||||
|
||||
# Verify key exists (O(1) - handles model_cost.pop() case)
|
||||
if matched_key is not None and matched_key in litellm.model_cost:
|
||||
return matched_key
|
||||
|
||||
# Rebuild map if stale entry detected (O(n) rebuild, but only when stale entry found)
|
||||
if matched_key is not None:
|
||||
matched_key = _handle_stale_map_entry_rebuild(potential_key_lower)
|
||||
if matched_key is not None:
|
||||
return matched_key
|
||||
|
||||
return None
|
||||
return CATALOG.lookup_key(potential_key)
|
||||
|
||||
|
||||
_get_model_cost_key = get_model_cost_key
|
||||
|
||||
|
||||
def _get_model_info_from_model_cost(key: str) -> dict[str, object]:
|
||||
return litellm.model_cost[key]
|
||||
|
||||
|
||||
def _check_provider_match(model_info: dict, custom_llm_provider: str | None) -> bool:
|
||||
"""
|
||||
Check if the model info provider matches the custom provider.
|
||||
|
|
@ -6370,19 +6266,6 @@ class PotentialModelNamesAndCustomLLMProvider(TypedDict):
|
|||
custom_llm_provider: str
|
||||
|
||||
|
||||
def _first_registered_match(
|
||||
candidates: Sequence[str], custom_llm_provider: str | None
|
||||
) -> tuple[str | None, dict[str, Any] | None]:
|
||||
registered_keys: Final = (key for key in map(get_model_cost_key, candidates) if key is not None)
|
||||
entries: Final = ((key, _get_model_info_from_model_cost(key=key)) for key in registered_keys)
|
||||
matches: Final = (
|
||||
(key, info)
|
||||
for key, info in entries
|
||||
if _check_provider_match(model_info=info, custom_llm_provider=custom_llm_provider)
|
||||
)
|
||||
return next(matches, (None, None))
|
||||
|
||||
|
||||
def _get_model_info_from_generalization(
|
||||
model: str,
|
||||
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
|
||||
|
|
@ -6515,6 +6398,7 @@ def get_potential_model_names(model: str, custom_llm_provider: str | None) -> Po
|
|||
_get_potential_model_names = get_potential_model_names
|
||||
|
||||
|
||||
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
|
||||
def _get_max_position_embeddings(model_name: str) -> int | None:
|
||||
# Construct the URL for the config.json file
|
||||
config_url: Final = f"https://huggingface.co/{model_name}/raw/main/config.json"
|
||||
|
|
@ -6538,17 +6422,12 @@ def _get_max_position_embeddings(model_name: str) -> int | None:
|
|||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
|
||||
def cached_get_model_info_helper(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: str | None = None,
|
||||
) -> ModelInfoBase:
|
||||
"""
|
||||
_get_model_info_helper wrapped with lru_cache
|
||||
|
||||
Speed Optimization to hit high RPS
|
||||
"""
|
||||
"""Kept for callers on main. Reads go to the live dict, so there is nothing to cache"""
|
||||
return get_model_info_helper(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -6579,19 +6458,7 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp
|
|||
return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None
|
||||
|
||||
|
||||
def _is_potential_model_name_in_model_cost(
|
||||
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the potential model name is in the model cost (case-insensitive).
|
||||
"""
|
||||
return any(
|
||||
get_model_cost_key(str(potential_model_name)) is not None
|
||||
for potential_model_name in potential_model_names.values()
|
||||
)
|
||||
|
||||
|
||||
_ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN
|
||||
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
|
||||
|
||||
|
||||
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
|
||||
|
|
@ -6610,7 +6477,10 @@ def get_model_info_helper(
|
|||
"""
|
||||
Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
provider_banner_suppressed,
|
||||
)
|
||||
|
||||
try:
|
||||
azure_llms: Final = {**litellm.azure_llms, **litellm.azure_embedding_models}
|
||||
|
|
@ -6626,19 +6496,15 @@ def get_model_info_helper(
|
|||
):
|
||||
model = model + "@latest"
|
||||
##########################
|
||||
potential_model_names: Final = get_potential_model_names(
|
||||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
requested_provider: Final = custom_llm_provider
|
||||
with provider_banner_suppressed():
|
||||
potential_model_names: Final = get_potential_model_names(
|
||||
model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model)
|
||||
)
|
||||
|
||||
verbose_logger.debug("checking potential_model_names in litellm.model_cost: %s", potential_model_names)
|
||||
|
||||
combined_model_name: Final = potential_model_names["combined_model_name"]
|
||||
stripped_model_name: Final = potential_model_names["stripped_model_name"]
|
||||
combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"]
|
||||
provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"]
|
||||
split_model: Final = potential_model_names["split_model"]
|
||||
custom_llm_provider = potential_model_names["custom_llm_provider"]
|
||||
model_cost_custom_llm_provider: Final = custom_llm_provider
|
||||
#########################
|
||||
provider_config: BaseLLMModelInfo | None = None
|
||||
if custom_llm_provider and custom_llm_provider in LlmProvidersSet:
|
||||
|
|
@ -6703,20 +6569,18 @@ def get_model_info_helper(
|
|||
and strip it. Tried last so no model that already resolves through 1-6 can change.
|
||||
"""
|
||||
|
||||
lookup_order: Final = (
|
||||
combined_model_name,
|
||||
model,
|
||||
potential_model_names["region_free_combined_model_name"],
|
||||
split_model,
|
||||
combined_stripped_model_name,
|
||||
stripped_model_name,
|
||||
provider_prefixed_model_name,
|
||||
)
|
||||
lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider)
|
||||
key: str | None = lookup[0]
|
||||
_model_info: dict[str, Any] | None = lookup[1]
|
||||
from litellm._catalog import CATALOG, Invalid, Missing
|
||||
|
||||
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
|
||||
resolution: Final = CATALOG.resolve(model, requested_provider)
|
||||
key: str | None = None if isinstance(resolution, Missing) else resolution.key
|
||||
if isinstance(resolution, Invalid) and resolution.key not in _REPORTED_INVALID_ROWS:
|
||||
_REPORTED_INVALID_ROWS.add(resolution.key)
|
||||
verbose_logger.warning(
|
||||
"litellm.model_cost[%r] fails validation: %s", resolution.key, "; ".join(resolution.errors)
|
||||
)
|
||||
_model_info: dict[str, Any] | None = None if isinstance(resolution, Missing) else resolution.row()
|
||||
|
||||
if _model_info is not None and key is not None and (_model_info.get("mode") or "chat") in _BACKFILL_MODES:
|
||||
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
|
||||
if fill_missing is not None:
|
||||
_model_info = {
|
||||
|
|
@ -6735,321 +6599,11 @@ def get_model_info_helper(
|
|||
|
||||
if _model_info is None or key is None:
|
||||
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
|
||||
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
|
||||
if _input_cost_per_token is None:
|
||||
# default value to 0, be noisy about this
|
||||
verbose_logger.debug(
|
||||
"model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
_input_cost_per_token = 0
|
||||
from litellm._catalog import Catalog
|
||||
|
||||
_output_cost_per_token: float | None = _model_info.get("output_cost_per_token")
|
||||
if _output_cost_per_token is None:
|
||||
# default value to 0, be noisy about this
|
||||
verbose_logger.debug(
|
||||
"model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
_output_cost_per_token = 0
|
||||
|
||||
returned_model_info: Final = ModelInfoBase(
|
||||
key=key,
|
||||
max_tokens=_model_info.get("max_output_tokens") or _model_info.get("max_input_tokens"),
|
||||
max_input_tokens=_model_info.get("max_input_tokens", None),
|
||||
max_output_tokens=_model_info.get("max_output_tokens", None),
|
||||
input_cost_per_token=_input_cost_per_token,
|
||||
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
|
||||
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
|
||||
input_cost_per_token_balanced=_model_info.get("input_cost_per_token_balanced", None),
|
||||
input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None),
|
||||
cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None),
|
||||
cache_creation_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_200k_tokens", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_272k_tokens=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_272k_tokens", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_272k_tokens_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_272k_tokens_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None),
|
||||
cache_creation_input_token_cost_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_ultrafast=_model_info.get(
|
||||
"cache_creation_input_token_cost_ultrafast", None
|
||||
),
|
||||
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
|
||||
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
|
||||
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
|
||||
cache_read_input_audio_token_cost_priority=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_priority", None
|
||||
),
|
||||
cache_read_input_audio_token_cost_batches=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_batches", None
|
||||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens", None
|
||||
),
|
||||
cache_read_input_token_cost_above_200k_tokens_priority=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority", None
|
||||
),
|
||||
cache_read_input_token_cost_above_272k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_272k_tokens", None
|
||||
),
|
||||
cache_read_input_token_cost_above_272k_tokens_priority=_model_info.get(
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority", None
|
||||
),
|
||||
cache_read_input_token_cost_above_272k_tokens_flex=_model_info.get(
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex", None
|
||||
),
|
||||
cache_read_input_token_cost_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"cache_read_input_token_cost_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
cache_read_input_token_cost_above_512k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_512k_tokens", None
|
||||
),
|
||||
cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None),
|
||||
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
|
||||
cache_read_input_token_cost_balanced=_model_info.get("cache_read_input_token_cost_balanced", None),
|
||||
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
|
||||
cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"),
|
||||
cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens_batches"
|
||||
),
|
||||
cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get(
|
||||
"cache_read_input_token_cost_above_272k_tokens_batches"
|
||||
),
|
||||
cache_creation_input_token_cost_batches=_model_info.get("cache_creation_input_token_cost_batches"),
|
||||
cache_creation_input_token_cost_above_200k_tokens_batches=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_200k_tokens_batches"
|
||||
),
|
||||
cache_creation_input_token_cost_above_272k_tokens_batches=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches"
|
||||
),
|
||||
cache_creation_input_token_cost_above_1hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_1hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_batches=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_batches", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_ultrafast=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_ultrafast", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
off_peak_pricing=_model_info.get("off_peak_pricing", None),
|
||||
input_cost_per_character=_model_info.get("input_cost_per_character", None),
|
||||
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
|
||||
input_cost_per_token_above_200k_tokens=_model_info.get("input_cost_per_token_above_200k_tokens", None),
|
||||
input_cost_per_token_above_200k_tokens_priority=_model_info.get(
|
||||
"input_cost_per_token_above_200k_tokens_priority", None
|
||||
),
|
||||
input_cost_per_token_above_272k_tokens=_model_info.get("input_cost_per_token_above_272k_tokens", None),
|
||||
input_cost_per_token_above_272k_tokens_priority=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_priority", None
|
||||
),
|
||||
input_cost_per_token_above_272k_tokens_flex=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_flex", None
|
||||
),
|
||||
input_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None),
|
||||
input_cost_per_query=_model_info.get("input_cost_per_query", None),
|
||||
cost_per_second=_model_info.get("cost_per_second", None),
|
||||
input_cost_per_second=_model_info.get("input_cost_per_second", None),
|
||||
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
|
||||
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
|
||||
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
|
||||
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
|
||||
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
|
||||
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
|
||||
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
|
||||
input_cost_per_image_token_batches=_model_info.get("input_cost_per_image_token_batches", None),
|
||||
input_cost_per_image=_model_info.get("input_cost_per_image", None),
|
||||
input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None),
|
||||
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
|
||||
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
|
||||
input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None),
|
||||
input_cost_per_token_above_200k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_token_above_200k_tokens_batches"
|
||||
),
|
||||
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
|
||||
output_cost_per_token_above_200k_tokens_batches=_model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token=_output_cost_per_token,
|
||||
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
|
||||
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
|
||||
output_cost_per_token_balanced=_model_info.get("output_cost_per_token_balanced", None),
|
||||
output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None),
|
||||
regional_processing_uplift_multiplier_eu=_model_info.get(
|
||||
"regional_processing_uplift_multiplier_eu", None
|
||||
),
|
||||
regional_processing_uplift_multiplier_us=_model_info.get(
|
||||
"regional_processing_uplift_multiplier_us", None
|
||||
),
|
||||
regional_endpoint_uplift_multiplier=_model_info.get("regional_endpoint_uplift_multiplier", None),
|
||||
output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None),
|
||||
output_cost_per_character=_model_info.get("output_cost_per_character", None),
|
||||
output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None),
|
||||
output_cost_per_reasoning_token_flex=_model_info.get("output_cost_per_reasoning_token_flex", None),
|
||||
output_cost_per_reasoning_token_priority=_model_info.get(
|
||||
"output_cost_per_reasoning_token_priority", None
|
||||
),
|
||||
output_cost_per_token_above_128k_tokens=_model_info.get(
|
||||
"output_cost_per_token_above_128k_tokens", None
|
||||
),
|
||||
output_cost_per_character_above_128k_tokens=_model_info.get(
|
||||
"output_cost_per_character_above_128k_tokens", None
|
||||
),
|
||||
output_cost_per_token_above_200k_tokens=_model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens", None
|
||||
),
|
||||
output_cost_per_token_above_200k_tokens_priority=_model_info.get(
|
||||
"output_cost_per_token_above_200k_tokens_priority", None
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens", None
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens_priority=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_priority", None
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens_flex=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_flex", None
|
||||
),
|
||||
output_cost_per_token_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
output_cost_per_token_above_512k_tokens=_model_info.get(
|
||||
"output_cost_per_token_above_512k_tokens", None
|
||||
),
|
||||
output_cost_per_second=_model_info.get("output_cost_per_second", None),
|
||||
output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None),
|
||||
output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None),
|
||||
output_cost_per_second_720p=_model_info.get("output_cost_per_second_720p", None),
|
||||
output_cost_per_second_768p=_model_info.get("output_cost_per_second_768p", None),
|
||||
output_cost_per_second_2k=_model_info.get("output_cost_per_second_2k", None),
|
||||
output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None),
|
||||
output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None),
|
||||
output_cost_per_image=_model_info.get("output_cost_per_image", None),
|
||||
output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
|
||||
output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
|
||||
output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
|
||||
output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None),
|
||||
output_vector_size=_model_info.get("output_vector_size", None),
|
||||
citation_cost_per_token=_model_info.get("citation_cost_per_token", None),
|
||||
tiered_pricing=_model_info.get("tiered_pricing", None),
|
||||
litellm_provider=_model_info.get("litellm_provider", custom_llm_provider),
|
||||
mode=_model_info.get("mode"),
|
||||
supported_endpoints=_model_info.get("supported_endpoints", None),
|
||||
supports_system_messages=_model_info.get("supports_system_messages", None),
|
||||
supports_response_schema=_model_info.get("supports_response_schema", None),
|
||||
supports_vision=_model_info.get("supports_vision", None),
|
||||
supports_function_calling=_model_info.get("supports_function_calling", None),
|
||||
supports_parallel_function_calling=_model_info.get("supports_parallel_function_calling", None),
|
||||
supports_tool_choice=_model_info.get("supports_tool_choice", None),
|
||||
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
|
||||
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
|
||||
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
|
||||
supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None),
|
||||
supports_audio_input=_model_info.get("supports_audio_input", None),
|
||||
supports_video_input=_model_info.get("supports_video_input", None),
|
||||
supports_box_questions=_model_info.get("supports_box_questions", None),
|
||||
supports_audio_output=_model_info.get("supports_audio_output", None),
|
||||
supports_pdf_input=_model_info.get("supports_pdf_input", None),
|
||||
supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None),
|
||||
supports_native_streaming=_model_info.get("supports_native_streaming", None),
|
||||
supports_native_structured_output=_model_info.get("supports_native_structured_output", None),
|
||||
supports_bedrock_native_structured_output=_model_info.get(
|
||||
"supports_bedrock_native_structured_output", None
|
||||
),
|
||||
supports_web_search=_model_info.get("supports_web_search", None),
|
||||
supports_url_context=_model_info.get("supports_url_context", None),
|
||||
supports_reasoning=_model_info.get("supports_reasoning", None),
|
||||
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
|
||||
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
|
||||
thinking_always_on=_model_info.get("thinking_always_on", None),
|
||||
supports_tool_search=_model_info.get("supports_tool_search", None),
|
||||
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
|
||||
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),
|
||||
supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None),
|
||||
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
|
||||
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
|
||||
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
|
||||
supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
|
||||
supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
|
||||
reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None),
|
||||
default_reasoning_effort=_model_info.get("default_reasoning_effort", None),
|
||||
bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
|
||||
bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
|
||||
supports_regex_lookaround=_model_info.get("supports_regex_lookaround", None),
|
||||
supports_computer_use=_model_info.get("supports_computer_use", None),
|
||||
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
|
||||
browser_open_cost_per_request=_model_info.get("browser_open_cost_per_request", None),
|
||||
cost_per_additional_result=_model_info.get("cost_per_additional_result", None),
|
||||
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
|
||||
search_free_requests=_model_info.get("search_free_requests", None),
|
||||
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
|
||||
tpm=_model_info.get("tpm", None),
|
||||
rpm=_model_info.get("rpm", None),
|
||||
ocr_cost_per_page=_model_info.get("ocr_cost_per_page", None),
|
||||
ocr_cost_per_page_batches=_model_info.get("ocr_cost_per_page_batches", None),
|
||||
ocr_cost_per_credit=_model_info.get("ocr_cost_per_credit", None),
|
||||
annotation_cost_per_page=_model_info.get("annotation_cost_per_page", None),
|
||||
annotation_cost_per_page_batches=_model_info.get("annotation_cost_per_page_batches", None),
|
||||
provider_specific_entry=_model_info.get("provider_specific_entry", None),
|
||||
uses_embed_content=_model_info.get("uses_embed_content", None),
|
||||
supports_image_size=_model_info.get("supports_image_size", None),
|
||||
supported_audio_formats=_model_info.get("supported_audio_formats", None),
|
||||
token_count_framing=_model_info.get("token_count_framing", None),
|
||||
token_count_model=_model_info.get("token_count_model", None),
|
||||
token_count_tokenizer=_model_info.get("token_count_tokenizer", None),
|
||||
vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None),
|
||||
returned_model_info: Final = cast( # cast-ok: Rust builds the dict from the ModelInfoBase field list
|
||||
ModelInfoBase, Catalog.model_info_of(key, _model_info, custom_llm_provider)
|
||||
)
|
||||
for cost_key, cost_value in _model_info.items():
|
||||
if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None:
|
||||
returned_model_info[cost_key] = cost_value
|
||||
return returned_model_info
|
||||
except ModelNotMappedError:
|
||||
raise
|
||||
|
|
@ -7067,7 +6621,12 @@ def _build_model_info(
|
|||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
) -> ModelInfo:
|
||||
supported_openai_params = litellm.get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import provider_banner_suppressed
|
||||
|
||||
with provider_banner_suppressed():
|
||||
supported_openai_params = litellm.get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
_model_info: Final = get_model_info_helper(
|
||||
model=model,
|
||||
|
|
@ -7088,15 +6647,6 @@ def _build_model_info(
|
|||
return ModelInfo(**_model_info, supported_openai_params=supported_openai_params)
|
||||
|
||||
|
||||
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
|
||||
def _cached_get_model_info(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> ModelInfo:
|
||||
return _build_model_info(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base)
|
||||
|
||||
|
||||
def get_model_info(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -7176,15 +6726,12 @@ def get_model_info(
|
|||
"supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
|
||||
}
|
||||
"""
|
||||
# api_key is a per-caller credential, not part of the model identity, so it is
|
||||
# kept out of the cache key; explicit keys are resolved without the cache.
|
||||
if api_key is not None:
|
||||
return _build_model_info(model, custom_llm_provider, api_base, api_key)
|
||||
return _cached_get_model_info(model, custom_llm_provider, api_base)
|
||||
return _build_model_info(model, custom_llm_provider, api_base, api_key)
|
||||
|
||||
|
||||
get_model_info.cache_clear = _cached_get_model_info.cache_clear
|
||||
get_model_info.cache_info = _cached_get_model_info.cache_info
|
||||
get_model_info.cache_clear = _get_max_position_embeddings.cache_clear # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
|
||||
get_model_info.cache_info = _get_max_position_embeddings.cache_info # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
|
||||
cached_get_model_info_helper.cache_clear = _get_max_position_embeddings.cache_clear # pyright: ignore[reportFunctionMemberAccess] # main callers still call cache_clear, reads are live
|
||||
|
||||
|
||||
def json_schema_type(python_type_name: str):
|
||||
|
|
|
|||
|
|
@ -152,11 +152,7 @@ def check_get_model_cost_key_performance():
|
|||
# Check for for loops
|
||||
if re.search(r"\bfor\s+\w+\s+in\s+", line):
|
||||
# Allow helper function calls (they're conditional)
|
||||
if not re.search(
|
||||
r"(_rebuild_model_cost_lowercase_map|_handle_stale_map_entry_rebuild|_handle_new_key_with_scan)",
|
||||
line,
|
||||
):
|
||||
problematic_lines.append((i, "for loop", line_stripped))
|
||||
problematic_lines.append((i, "for loop", line_stripped))
|
||||
|
||||
# Check for while loops
|
||||
if re.search(r"\bwhile\s+", line):
|
||||
|
|
@ -187,11 +183,7 @@ def check_get_model_cost_key_performance():
|
|||
|
||||
# Check for calls to functions that might have O(n) operations
|
||||
# Allow known helper functions that are conditional
|
||||
allowed_helpers = [
|
||||
"_rebuild_model_cost_lowercase_map",
|
||||
"_handle_stale_map_entry_rebuild",
|
||||
"_handle_new_key_with_scan",
|
||||
]
|
||||
allowed_helpers = []
|
||||
|
||||
# Check for function calls (pattern: function_name(...), but not function definitions)
|
||||
# Skip function definitions (def function_name(...))
|
||||
|
|
|
|||
|
|
@ -2358,3 +2358,17 @@ def test_global_disable_flag_with_transform_messages_helper(
|
|||
)
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}]
|
||||
|
||||
|
||||
def test_a_fireworks_row_added_by_a_direct_write_is_matched(monkeypatch):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"fireworks_ai/accounts/fireworks/models/acme-direct-9b",
|
||||
{"litellm_provider": "fireworks_ai", "mode": "chat", "supports_vision": True},
|
||||
)
|
||||
|
||||
info = FireworksAIConfig().get_provider_info("accounts/myorg/models/custom-acme-direct-9b")
|
||||
|
||||
assert info.get("supports_vision") is True
|
||||
|
|
|
|||
|
|
@ -396,8 +396,6 @@ class TestOllamaGetModelInfo:
|
|||
key must still reach the backend rather than be served from a result cached
|
||||
with a different key.
|
||||
"""
|
||||
from litellm.utils import _cached_get_model_info
|
||||
|
||||
captured_headers = []
|
||||
|
||||
def mock_post(url, json, headers=None):
|
||||
|
|
@ -420,7 +418,6 @@ class TestOllamaGetModelInfo:
|
|||
api_key=api_key,
|
||||
)
|
||||
|
||||
assert _cached_get_model_info.cache_info().currsize <= 1
|
||||
assert captured_headers == [
|
||||
{"Authorization": "Bearer key-one"},
|
||||
{"Authorization": "Bearer key-two"},
|
||||
|
|
|
|||
12323
tests/unit/resolution_reference.json
Normal file
12323
tests/unit/resolution_reference.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -5100,7 +5100,7 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey
|
|||
|
||||
|
||||
def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map, _rebuild_model_cost_lowercase_map
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map, get_model_cost_key
|
||||
|
||||
class _KeyRegisteredMidWalk(str):
|
||||
def lower(self) -> str:
|
||||
|
|
@ -5115,9 +5115,8 @@ def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: p
|
|||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = _EntryRegisteredMidWalk(mode="chat")
|
||||
try:
|
||||
assert _rebuild_model_cost_lowercase_map()["lit-5853-walked-last"] == "LIT-5853-WALKED-LAST"
|
||||
assert get_model_cost_key("lit-5853-walked-last") == "LIT-5853-WALKED-LAST"
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
assert "lit-5853-registered-during-the-rebuild" in litellm.model_cost
|
||||
assert "lit-5853-registered-during-add-known-models" in litellm.model_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
|
@ -5169,22 +5168,19 @@ class _EntryReadDuringARegistration(dict[str, object]):
|
|||
return super().__contains__(key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("walk", ["fal_ai", "fireworks_ai", "ollama", "gpt-5", "case-insensitive scan"])
|
||||
@pytest.mark.parametrize("walk", ["fal_ai", "ollama", "gpt-5"])
|
||||
def test_request_path_cost_map_walks_survive_a_registration_mid_walk(
|
||||
monkeypatch: pytest.MonkeyPatch, walk: str
|
||||
) -> None:
|
||||
from litellm.llms.fal_ai.cost_calculator import _keyed_rows
|
||||
from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
|
||||
from litellm.llms.ollama.common_utils import OllamaModelInfo
|
||||
from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort
|
||||
from litellm.utils import _handle_new_key_with_scan, _invalidate_model_cost_lowercase_map
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
walks: Final[Mapping[str, Callable[[], object]]] = {
|
||||
"fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"),
|
||||
"fireworks_ai": FireworksAIConfig._get_fireworks_index,
|
||||
"ollama": lambda: OllamaModelInfo._is_static_ollama_model("ollama/lit-5853-model"),
|
||||
"gpt-5": _catalogue_declares_default_effort,
|
||||
"case-insensitive scan": lambda: _handle_new_key_with_scan("lit-5853-model"),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
|
|
@ -10900,3 +10896,67 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_
|
|||
)
|
||||
|
||||
assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]}
|
||||
|
||||
|
||||
def test_get_model_info_sees_a_direct_write_without_an_invalidator(monkeypatch):
|
||||
from litellm.utils import cached_get_model_info_helper
|
||||
|
||||
litellm.get_model_info("gpt-4o")
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-4o", {**litellm.model_cost["gpt-4o"], "input_cost_per_token": 1.0})
|
||||
|
||||
assert litellm.get_model_info("gpt-4o")["input_cost_per_token"] == 1.0
|
||||
assert cached_get_model_info_helper("gpt-4o", "openai")["input_cost_per_token"] == 1.0
|
||||
|
||||
|
||||
def test_get_model_info_sees_a_nested_write_without_an_invalidator(monkeypatch):
|
||||
litellm.get_model_info("gpt-4o")
|
||||
litellm.supports_vision("gpt-4o")
|
||||
monkeypatch.setitem(litellm.model_cost, "gpt-4o", dict(litellm.model_cost["gpt-4o"]))
|
||||
|
||||
litellm.model_cost["gpt-4o"]["input_cost_per_token"] = 1.0
|
||||
litellm.model_cost["gpt-4o"]["supports_vision"] = False
|
||||
|
||||
assert litellm.get_model_info("gpt-4o")["input_cost_per_token"] == 1.0
|
||||
assert litellm.supports_vision("gpt-4o") is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"lookup",
|
||||
[
|
||||
pytest.param(lambda: litellm.get_model_info("acme-unknown-model-5982"), id="get_model_info"),
|
||||
pytest.param(lambda: litellm.supports_function_calling("acme-unknown-model-5982"), id="supports"),
|
||||
],
|
||||
)
|
||||
def test_an_unknown_model_prints_no_provider_banner(monkeypatch, capsys, lookup):
|
||||
monkeypatch.setattr(litellm, "suppress_debug_info", False)
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
lookup()
|
||||
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
|
||||
def test_a_key_added_by_a_direct_write_is_found_case_insensitively(monkeypatch):
|
||||
from litellm.utils import get_model_cost_key
|
||||
|
||||
assert get_model_cost_key("mixed-case-direct") is None
|
||||
monkeypatch.setitem(litellm.model_cost, "Mixed-Case-Direct", {"litellm_provider": "openai"})
|
||||
|
||||
assert get_model_cost_key("mixed-case-direct") == "Mixed-Case-Direct"
|
||||
|
||||
|
||||
def test_resolution_matches_mains_for_every_bundled_key_and_variant():
|
||||
reference = json.loads((Path(__file__).parent / "resolution_reference.json").read_text())
|
||||
|
||||
def resolved(model, provider):
|
||||
try:
|
||||
return litellm.get_model_info(model, provider)["key"]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
mismatches = [
|
||||
(model, provider, expected, actual)
|
||||
for model, provider, expected in reference["cases"]
|
||||
if (actual := resolved(model, provider)) != expected
|
||||
]
|
||||
assert mismatches == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue