mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(cost): port get_model_info key resolution and get_llm_provider inference
ModelInfoCatalog now resolves models the way Python's _get_model_info_helper does: the six candidate names from _get_potential_model_names, case-insensitive keys, bedrock routing and base-model stripping, the mantle region prefix, the fireworks cost key, vertex @latest and meta/ aliases, the azure alias table, the huggingface zero-rated entry, and the fallback_generalizations capability rules. Provider inference follows get_llm_provider instead of reading the cost-map litellm_provider: bare ids route through the provider model lists, the JSON provider registry, the static openai and cohere lists, and the routing rules. cost_per_token infers the provider when the caller passes none and fails with MissingProvider where Python raises BadRequestError. Bedrock and fal image pricing resolve through get_model_info, and Stability 1 rejects a steps value Python cannot compare with 50. A differential run over every cost-map key and its prefixed, cased, dated, and bedrock variants (21,365 lookups) matches Python except for ollama and lemonade, whose Python lookup queries a local server. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
eb627547c0
commit
edbf88daf2
35 changed files with 1677 additions and 333 deletions
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -2979,6 +2979,7 @@ name = "litellm-cost"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"fancy-regex 0.19.2",
|
||||
"jiff",
|
||||
"litellm-python-compat",
|
||||
"litellm-token-counter",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# litellm-cost port rules
|
||||
|
||||
- This crate is a standalone port of Python's cost calculation. Prices, usage, model metadata, response kinds, and feature flags are caller-supplied; there is no Python bridge, no proxy integration, no network, and no env or secret resolution
|
||||
- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline
|
||||
- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline. Model lookup goes through `ModelInfoCatalog::get_model_info` (`model_info.rs`, mirroring `utils.py::_get_model_info_helper`) and provider inference through `get_llm_provider.rs` and `fallback_generalizations.rs`; never read `entries()` by a hand-built key where Python calls `get_model_info`. Known gaps: provider configs whose `get_model_info` queries a live server (ollama, lemonade) and the `ModelInfoBase` field projection are not ported
|
||||
- Module `foo_cost.rs` mirrors the calculation portion of the matching Python module; `tests/python_foo_cost.rs` carries the mapping evidence, with each test naming the Python test it mirrors (`tests/test_litellm/` mirrors `litellm/` per the repo-root AGENTS.md)
|
||||
- Preserve Python's precedence and fallback orders exactly: cache hit before provider-reported before calculated cost, discount before margin, wall-clock pricing before provider totals, quality and size key selection order, model-key selection order including duplicate provider prefixes and regional aliases
|
||||
- Diverge only where Python is observably wrong, and pin each divergence with a test that would fail if either side changed
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ serde.workspace = true
|
|||
serde_json.workspace = true
|
||||
litellm-python-compat = { path = "../python-compat" }
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
fancy-regex.workspace = true
|
||||
jiff = "=0.2.37"
|
||||
regex = "=1.13.1"
|
||||
strum.workspace = true
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ pub fn azure_ai_cost_per_token(
|
|||
request_model: Option<&str>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
let model_info = catalog.entry(request.model, Some("azure_ai"), request.region);
|
||||
let model_info = model_info.as_deref();
|
||||
if let Some(cost) =
|
||||
model_info.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
||||
{
|
||||
|
|
@ -98,6 +99,6 @@ pub fn azure_ai_router_fee(
|
|||
let entry = catalog
|
||||
.entry(fee_entry, Some("azure_ai"), None)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, entry);
|
||||
let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, &entry);
|
||||
Ok((fee > 0.0).then_some(fee))
|
||||
}
|
||||
|
|
|
|||
103
litellm-rust/crates/cost/src/bedrock_common_utils.rs
Normal file
103
litellm-rust/crates/cost/src/bedrock_common_utils.rs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
const ROUTING_PREFIXES: [&str; 7] = [
|
||||
"bedrock/",
|
||||
"converse/",
|
||||
"invoke/",
|
||||
"openai/",
|
||||
"mantle/",
|
||||
"nova-2/",
|
||||
"nova/",
|
||||
];
|
||||
|
||||
const CROSS_REGION_INFERENCE_REGIONS: [&str; 7] =
|
||||
["global", "us", "eu", "apac", "jp", "au", "us-gov"];
|
||||
|
||||
const ALL_REGIONS: [&str; 23] = [
|
||||
"us-east-1",
|
||||
"us-east-2",
|
||||
"us-west-1",
|
||||
"us-west-2",
|
||||
"us-gov-east-1",
|
||||
"us-gov-west-1",
|
||||
"eu-west-1",
|
||||
"eu-west-2",
|
||||
"eu-west-3",
|
||||
"eu-central-1",
|
||||
"eu-central-2",
|
||||
"eu-south-1",
|
||||
"eu-south-2",
|
||||
"eu-north-1",
|
||||
"ap-northeast-1",
|
||||
"ap-northeast-2",
|
||||
"ap-northeast-3",
|
||||
"ap-south-1",
|
||||
"ap-south-2",
|
||||
"ap-southeast-1",
|
||||
"ap-southeast-2",
|
||||
"ca-central-1",
|
||||
"sa-east-1",
|
||||
];
|
||||
|
||||
static THROUGHPUT_SUFFIX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(:\d+):\d+k$").expect("valid throughput pattern"));
|
||||
static CONTEXT_WINDOW_SUFFIX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"\[\w+\]$").expect("valid context window pattern"));
|
||||
|
||||
pub fn is_bedrock_region(region: &str) -> bool {
|
||||
ALL_REGIONS.contains(®ion)
|
||||
}
|
||||
|
||||
pub fn strip_bedrock_routing_prefix(model: &str) -> &str {
|
||||
ROUTING_PREFIXES.iter().fold(model, |model, prefix| {
|
||||
if model.starts_with(prefix) {
|
||||
model.split_once('/').map_or(model, |(_, rest)| rest)
|
||||
} else {
|
||||
model
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn strip_bedrock_throughput_suffix(model: &str) -> String {
|
||||
let without_throughput = THROUGHPUT_SUFFIX.replace(model, "${1}");
|
||||
CONTEXT_WINDOW_SUFFIX
|
||||
.replace(&without_throughput, "")
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
pub fn extract_model_name_from_bedrock_arn(model: &str) -> &str {
|
||||
if model.to_lowercase().contains("arn") {
|
||||
model.rsplit('/').next().unwrap_or(model)
|
||||
} else {
|
||||
model
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_bedrock_base_model(model: &str) -> String {
|
||||
let spec = ["bedrock/converse/", "bedrock/", "converse/"]
|
||||
.into_iter()
|
||||
.find_map(|prefix| model.strip_prefix(prefix))
|
||||
.unwrap_or(model);
|
||||
if spec.starts_with("nova-2/") {
|
||||
return "amazon.nova-2-custom".to_owned();
|
||||
}
|
||||
if spec.starts_with("nova/") {
|
||||
return "amazon.nova-custom".to_owned();
|
||||
}
|
||||
let model = strip_bedrock_throughput_suffix(extract_model_name_from_bedrock_arn(
|
||||
strip_bedrock_routing_prefix(model),
|
||||
));
|
||||
if let Some((region, rest)) = model.split_once('.')
|
||||
&& CROSS_REGION_INFERENCE_REGIONS.contains(®ion)
|
||||
{
|
||||
return rest.to_owned();
|
||||
}
|
||||
if let Some((region, rest)) = model.split_once('/')
|
||||
&& is_bedrock_region(region)
|
||||
{
|
||||
return rest.to_owned();
|
||||
}
|
||||
model
|
||||
}
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
use crate::error::CostError;
|
||||
use crate::wire::py_real;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum BedrockImageFamily {
|
||||
Titan,
|
||||
|
|
@ -22,57 +24,48 @@ pub fn get_config_class(model: &str) -> BedrockImageFamily {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn stability1_pricing_key(model: &str, size: Option<&str>, optional_params: &Value) -> String {
|
||||
let steps = optional_params
|
||||
.get("steps")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(50.0);
|
||||
pub fn stability1_pricing_key(
|
||||
model: &str,
|
||||
size: Option<&str>,
|
||||
optional_params: &Value,
|
||||
) -> Result<String, CostError> {
|
||||
let steps = match optional_params.get("steps") {
|
||||
None => 50.0,
|
||||
Some(steps) => py_real(steps).ok_or(CostError::InvalidSteps)?,
|
||||
};
|
||||
let tier = if steps > 50.0 {
|
||||
"max-steps"
|
||||
} else {
|
||||
"50-steps"
|
||||
};
|
||||
format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024"))
|
||||
}
|
||||
|
||||
fn model_info<'a>(
|
||||
entries: &'a HashMap<String, Value>,
|
||||
model: &str,
|
||||
family: BedrockImageFamily,
|
||||
size: Option<&str>,
|
||||
optional_params: &Value,
|
||||
) -> Option<&'a Value> {
|
||||
let bare = model.strip_prefix("bedrock/").unwrap_or(model);
|
||||
match family {
|
||||
BedrockImageFamily::Stability1 => {
|
||||
let key = stability1_pricing_key(model, size, optional_params);
|
||||
let bare_key = stability1_pricing_key(bare, size, optional_params);
|
||||
[key, bare_key]
|
||||
.into_iter()
|
||||
.find_map(|candidate| entries.get(&candidate))
|
||||
}
|
||||
_ => [model.to_owned(), format!("bedrock/{bare}"), bare.to_owned()]
|
||||
.into_iter()
|
||||
.find_map(|candidate| entries.get(&candidate)),
|
||||
}
|
||||
Ok(format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024")))
|
||||
}
|
||||
|
||||
pub fn cost_calculator(
|
||||
catalog: &ModelInfoCatalog,
|
||||
model: &str,
|
||||
image_response: &Value,
|
||||
size: Option<&str>,
|
||||
optional_params: &Value,
|
||||
entries: &HashMap<String, Value>,
|
||||
) -> Option<f64> {
|
||||
let family = get_config_class(model);
|
||||
let info = model_info(entries, model, family, size, optional_params)?;
|
||||
let rate = info
|
||||
) -> Result<f64, CostError> {
|
||||
let model_info = match get_config_class(model) {
|
||||
BedrockImageFamily::Titan => catalog.get_model_info(model, None)?,
|
||||
BedrockImageFamily::NovaCanvas | BedrockImageFamily::Stability3 => {
|
||||
catalog.get_model_info(model, Some("bedrock"))?
|
||||
}
|
||||
BedrockImageFamily::Stability1 => catalog.get_model_info(
|
||||
&stability1_pricing_key(model, size.filter(|size| !size.is_empty()), optional_params)?,
|
||||
Some("bedrock"),
|
||||
)?,
|
||||
};
|
||||
let rate = model_info
|
||||
.info
|
||||
.get("output_cost_per_image")
|
||||
.and_then(Value::as_f64)
|
||||
.and_then(py_real)
|
||||
.unwrap_or(0.0);
|
||||
let images = image_response
|
||||
.get("data")
|
||||
.and_then(Value::as_array)
|
||||
.map_or(0, Vec::len);
|
||||
Some(rate * images as f64)
|
||||
Ok(rate * images as f64)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -190,12 +190,12 @@ pub fn calculate_token_type_cost_breakdown(
|
|||
}
|
||||
|
||||
fn billed_rates_request<'a>(
|
||||
catalog: &'a ModelInfoCatalog,
|
||||
model_info: Option<&'a Value>,
|
||||
request: ModelCostRequest<'a>,
|
||||
custom_cost_per_token: Option<CustomTokenRates>,
|
||||
) -> BilledRatesRequest<'a> {
|
||||
BilledRatesRequest {
|
||||
model_info: catalog.entry(request.model, request.provider, request.region),
|
||||
model_info,
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier: request.service_tier,
|
||||
|
|
@ -211,8 +211,9 @@ pub fn get_billed_token_rates(
|
|||
request: ModelCostRequest<'_>,
|
||||
custom_cost_per_token: Option<CustomTokenRates>,
|
||||
) -> Option<BilledTokenRates> {
|
||||
let model_info = catalog.entry(request.model, request.provider, request.region);
|
||||
calculate_billed_token_rates(billed_rates_request(
|
||||
catalog,
|
||||
model_info.as_deref(),
|
||||
request,
|
||||
custom_cost_per_token,
|
||||
))
|
||||
|
|
@ -223,8 +224,9 @@ pub fn get_token_type_cost_breakdown(
|
|||
request: ModelCostRequest<'_>,
|
||||
custom_cost_per_token: Option<CustomTokenRates>,
|
||||
) -> TokenTypeCostBreakdown {
|
||||
let model_info = catalog.entry(request.model, request.provider, request.region);
|
||||
calculate_token_type_cost_breakdown(billed_rates_request(
|
||||
catalog,
|
||||
model_info.as_deref(),
|
||||
request,
|
||||
custom_cost_per_token,
|
||||
))
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
|
|
@ -7,6 +8,11 @@ use crate::completion_input::{
|
|||
CompletionInputRequest, PreparedCompletionInput, prepare_completion_input,
|
||||
};
|
||||
use crate::error::CostError;
|
||||
use crate::fallback_generalizations::FallbackGeneralizations;
|
||||
use crate::get_llm_provider::{LlmProvider, ProviderModelSets, get_llm_provider};
|
||||
use crate::model_info::{ModelCost, ModelInfo, get_model_info_helper, lowercase_key_map};
|
||||
|
||||
const FALLBACK_GENERALIZATIONS_KEY: &str = "fallback_generalizations";
|
||||
use crate::model_selection::{
|
||||
ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc,
|
||||
};
|
||||
|
|
@ -15,6 +21,9 @@ use crate::responses_usage::ChatUsage;
|
|||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ModelInfoCatalog {
|
||||
entries: HashMap<String, Value>,
|
||||
lowercase_keys: HashMap<String, String>,
|
||||
model_sets: ProviderModelSets,
|
||||
generalizations: FallbackGeneralizations,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
|
|
@ -63,33 +72,13 @@ pub enum CostCall<'a> {
|
|||
},
|
||||
}
|
||||
|
||||
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
|
||||
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
|
||||
return true;
|
||||
};
|
||||
entry_provider == provider
|
||||
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|
||||
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|
||||
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|
||||
|| provider == "litellm_proxy"
|
||||
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|
||||
|| provider == "github"
|
||||
}
|
||||
|
||||
fn select_model_key<'a>(
|
||||
entries: &'a HashMap<String, Value>,
|
||||
model: &str,
|
||||
fn cost_map_model<'m>(
|
||||
entries: &HashMap<String, Value>,
|
||||
model: &'m str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
let match_provider = provider.map(|provider| match provider {
|
||||
"vertex_ai_beta" => "vertex_ai",
|
||||
provider => provider,
|
||||
});
|
||||
let normalized = match provider {
|
||||
) -> Cow<'m, str> {
|
||||
let model = match provider {
|
||||
Some(provider) => {
|
||||
let prefix = format!("{provider}/");
|
||||
let mut name = model;
|
||||
|
|
@ -106,44 +95,94 @@ fn select_model_key<'a>(
|
|||
let model_with_provider = match provider {
|
||||
Some(provider) => {
|
||||
let prefix = format!("{provider}/");
|
||||
let bare = normalized.strip_prefix(&prefix).unwrap_or(normalized);
|
||||
let bare = model.strip_prefix(&prefix).unwrap_or(model);
|
||||
let regional = region.map(|region| format!("{provider}/{region}/{bare}"));
|
||||
if let Some(regional) = regional.filter(|key| entries.contains_key(key)) {
|
||||
regional
|
||||
} else if normalized.starts_with(&prefix) {
|
||||
normalized.to_owned()
|
||||
Cow::Owned(regional)
|
||||
} else if model.starts_with(&prefix) {
|
||||
Cow::Borrowed(model)
|
||||
} else {
|
||||
format!("{provider}/{normalized}")
|
||||
Cow::Owned(format!("{provider}/{model}"))
|
||||
}
|
||||
}
|
||||
None => normalized.to_owned(),
|
||||
None => Cow::Borrowed(model),
|
||||
};
|
||||
let without_prefix = normalized
|
||||
let without_prefix = model
|
||||
.split_once('/')
|
||||
.map_or(normalized, |(_, remainder)| remainder);
|
||||
[model_with_provider.as_str(), normalized, without_prefix]
|
||||
.into_iter()
|
||||
.find_map(|candidate| {
|
||||
entries
|
||||
.get_key_value(candidate)
|
||||
.filter(|(_, model_info)| check_provider_match(model_info, match_provider))
|
||||
.map(|(key, _)| key.as_str())
|
||||
})
|
||||
.map_or(model, |(_, remainder)| remainder);
|
||||
if entries.contains_key(model_with_provider.as_ref()) {
|
||||
model_with_provider
|
||||
} else if entries.contains_key(model) {
|
||||
Cow::Borrowed(model)
|
||||
} else if entries.contains_key(without_prefix) {
|
||||
Cow::Borrowed(without_prefix)
|
||||
} else {
|
||||
Cow::Borrowed(model)
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelInfoCatalog {
|
||||
pub fn new(entries: HashMap<String, Value>) -> Self {
|
||||
Self { entries }
|
||||
pub fn new(mut entries: HashMap<String, Value>) -> Self {
|
||||
let generalizations = FallbackGeneralizations::from_block(
|
||||
entries.remove(FALLBACK_GENERALIZATIONS_KEY).as_ref(),
|
||||
);
|
||||
Self {
|
||||
lowercase_keys: lowercase_key_map(&entries),
|
||||
model_sets: ProviderModelSets::from_model_cost(&entries),
|
||||
generalizations,
|
||||
entries,
|
||||
}
|
||||
}
|
||||
|
||||
fn model_cost(&self) -> ModelCost<'_> {
|
||||
ModelCost {
|
||||
entries: &self.entries,
|
||||
lowercase_keys: &self.lowercase_keys,
|
||||
sets: &self.model_sets,
|
||||
generalizations: &self.generalizations,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> &HashMap<String, Value> {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
pub fn entry_for_key(&self, key: &str) -> &Value {
|
||||
self.entries
|
||||
.get(key)
|
||||
.expect("key came from select_model_key")
|
||||
pub fn get_llm_provider(&self, model: &str) -> Option<LlmProvider> {
|
||||
get_llm_provider(model, &self.model_sets, &self.generalizations)
|
||||
}
|
||||
|
||||
pub fn get_model_info(
|
||||
&self,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
) -> Result<ModelInfo<'_>, CostError> {
|
||||
get_model_info_helper(self.model_cost(), model, provider)
|
||||
}
|
||||
|
||||
pub fn cost_map_model<'m>(
|
||||
&self,
|
||||
model: &'m str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Cow<'m, str> {
|
||||
cost_map_model(&self.entries, model, provider, region)
|
||||
}
|
||||
|
||||
pub fn select_model_info(
|
||||
&self,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<ModelInfo<'_>> {
|
||||
let selected = cost_map_model(&self.entries, model, provider, region);
|
||||
let inferred = provider
|
||||
.is_none()
|
||||
.then(|| self.get_llm_provider(model))
|
||||
.flatten();
|
||||
let provider = provider.or(inferred
|
||||
.as_ref()
|
||||
.map(|inferred| inferred.custom_llm_provider.as_str()));
|
||||
self.get_model_info(&selected, provider).ok()
|
||||
}
|
||||
|
||||
pub fn entry(
|
||||
|
|
@ -151,45 +190,27 @@ impl ModelInfoCatalog {
|
|||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<&Value> {
|
||||
self.select_model_key(model, provider, region)
|
||||
.and_then(|key| self.entries.get(key))
|
||||
}
|
||||
|
||||
pub fn select_model_key<'a>(
|
||||
&'a self,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
select_model_key(&self.entries, model, provider, region)
|
||||
) -> Option<Cow<'_, Value>> {
|
||||
self.select_model_info(model, provider, region)
|
||||
.map(|model_info| model_info.info)
|
||||
}
|
||||
|
||||
pub fn contains_exact_model(&self, model: &str) -> bool {
|
||||
self.entries.contains_key(model)
|
||||
}
|
||||
|
||||
pub fn get_provider_for_cost_calc(
|
||||
&self,
|
||||
model: Option<&str>,
|
||||
provider: Option<&str>,
|
||||
known_providers: &[&str],
|
||||
) -> Option<String> {
|
||||
get_provider_for_cost_calc(model, provider, known_providers, &self.entries)
|
||||
}
|
||||
|
||||
pub fn select_model_name_for_cost_calc(
|
||||
&self,
|
||||
request: ModelSelectionRequest<'_>,
|
||||
) -> Option<String> {
|
||||
select_model_name_for_cost_calc(request, &self.entries)
|
||||
select_model_name_for_cost_calc(request, self)
|
||||
}
|
||||
|
||||
pub fn pricing_entry_for_cost_calc<'a>(
|
||||
&'a self,
|
||||
request: ModelSelectionRequest<'a>,
|
||||
logging_details: Option<&'a Value>,
|
||||
) -> Option<(&'a str, &'a Value)> {
|
||||
) -> Option<ModelInfo<'a>> {
|
||||
let deployment_info = if request.custom_pricing {
|
||||
request
|
||||
.router_model_id
|
||||
|
|
@ -209,15 +230,13 @@ impl ModelInfoCatalog {
|
|||
if let (Some(key), Some(info)) =
|
||||
(request.router_model_id.or(request.model), deployment_info)
|
||||
{
|
||||
return Some((key, info));
|
||||
return Some(ModelInfo {
|
||||
key: Cow::Borrowed(key),
|
||||
info: Cow::Borrowed(info),
|
||||
});
|
||||
}
|
||||
let selected = self.select_model_name_for_cost_calc(request);
|
||||
let provider = get_provider_for_cost_calc(
|
||||
request.model,
|
||||
request.provider,
|
||||
request.known_providers,
|
||||
&self.entries,
|
||||
);
|
||||
let provider = get_provider_for_cost_calc(request.model, request.provider, self);
|
||||
[
|
||||
selected.as_deref(),
|
||||
request
|
||||
|
|
@ -228,16 +247,13 @@ impl ModelInfoCatalog {
|
|||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|model| {
|
||||
self.select_model_key(model, provider.as_deref(), None)
|
||||
.and_then(|key| self.entries.get(key).map(|info| (key, info)))
|
||||
})
|
||||
.find_map(|model| self.select_model_info(model, provider.as_deref(), None))
|
||||
}
|
||||
|
||||
pub fn prepare_completion_input(
|
||||
&self,
|
||||
request: CompletionInputRequest<'_>,
|
||||
) -> Result<PreparedCompletionInput, CostError> {
|
||||
prepare_completion_input(request, &self.entries)
|
||||
prepare_completion_input(request, self)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::HashMap;
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -106,7 +106,7 @@ pub fn select_service_tier(
|
|||
|
||||
pub fn prepare_completion_input(
|
||||
request: CompletionInputRequest<'_>,
|
||||
cost_map: &HashMap<String, Value>,
|
||||
catalog: &ModelInfoCatalog,
|
||||
) -> Result<PreparedCompletionInput, CostError> {
|
||||
let response = request.model_selection.response;
|
||||
let call_type =
|
||||
|
|
@ -126,7 +126,7 @@ pub fn prepare_completion_input(
|
|||
model,
|
||||
..request.model_selection
|
||||
},
|
||||
cost_map,
|
||||
catalog,
|
||||
);
|
||||
let response_model = get_response_model(response).map(str::to_owned);
|
||||
let requested_model = model.map(str::to_owned);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ use crate::image_cost_router::{
|
|||
ImageCostRouteRequest, call_type_has_image_response, route_image_generation_cost_calculator,
|
||||
};
|
||||
use crate::mcp_cost::calculate_mcp_tool_call_cost;
|
||||
use crate::model_selection::get_provider_for_cost_calc;
|
||||
use crate::per_second::{DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, get_replicate_completion_pricing};
|
||||
use crate::provider::LlmProviders;
|
||||
use crate::realtime_cost::{
|
||||
|
|
@ -647,13 +648,7 @@ pub fn completion_cost_from_response(
|
|||
.model_candidates
|
||||
.iter()
|
||||
.flatten()
|
||||
.find_map(|model| {
|
||||
catalog.get_provider_for_cost_calc(
|
||||
Some(model),
|
||||
None,
|
||||
request.input.model_selection.known_providers,
|
||||
)
|
||||
})
|
||||
.find_map(|model| get_provider_for_cost_calc(Some(model), None, catalog))
|
||||
});
|
||||
let provider = inferred_provider.as_deref();
|
||||
let deployment_info = request
|
||||
|
|
|
|||
|
|
@ -96,6 +96,16 @@ pub fn cost_per_token_for_call(
|
|||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
call: CostCall<'_>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
with_inferred_provider(catalog, request, |catalog, request| {
|
||||
cost_per_token_for_provider(catalog, request, call)
|
||||
})
|
||||
}
|
||||
|
||||
fn cost_per_token_for_provider(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
call: CostCall<'_>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
let provider = request
|
||||
.provider
|
||||
|
|
@ -119,7 +129,7 @@ pub fn cost_per_token_for_call(
|
|||
if provider == Some(LlmProviders::AZURE_AI) {
|
||||
return azure_ai_cost_per_token(catalog, request, request_model);
|
||||
}
|
||||
Ok(cost_per_token(catalog, request)?)
|
||||
token_cost_per_token(catalog, request)
|
||||
}
|
||||
CostCall::Speech { prompt_characters } => speech_cost(catalog, request, prompt_characters),
|
||||
CostCall::Transcription { duration_seconds } => {
|
||||
|
|
@ -154,7 +164,7 @@ pub fn cost_per_token_for_call(
|
|||
deployment_info,
|
||||
} => {
|
||||
let published = catalog.entry(request.model, request.provider, request.region);
|
||||
Ok(ocr_cost(response, deployment_info, published)?)
|
||||
Ok(ocr_cost(response, deployment_info, published.as_deref())?)
|
||||
}
|
||||
CostCall::Batch { deployment_info } => {
|
||||
let published = catalog.entry(request.model, request.provider, request.region);
|
||||
|
|
@ -171,7 +181,7 @@ pub fn cost_per_token_for_call(
|
|||
let info = if has_deployment_rates {
|
||||
deployment_info
|
||||
} else {
|
||||
published.or(deployment_info)
|
||||
published.as_deref().or(deployment_info)
|
||||
};
|
||||
let Some(info) = info else {
|
||||
return Ok((0.0, 0.0));
|
||||
|
|
@ -190,10 +200,39 @@ pub fn cost_per_token_for_call(
|
|||
pub fn cost_per_token(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
with_inferred_provider(catalog, request, token_cost_per_token)
|
||||
}
|
||||
|
||||
fn with_inferred_provider(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
price: impl FnOnce(&ModelInfoCatalog, ModelCostRequest<'_>) -> Result<(f64, f64), CostError>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
if request.provider.is_some() {
|
||||
return price(catalog, request);
|
||||
}
|
||||
let inferred = catalog
|
||||
.get_llm_provider(request.model)
|
||||
.ok_or(CostError::MissingProvider)?;
|
||||
let selected = catalog.cost_map_model(request.model, None, request.region);
|
||||
price(
|
||||
catalog,
|
||||
ModelCostRequest {
|
||||
model: &selected,
|
||||
provider: Some(&inferred.custom_llm_provider),
|
||||
..request
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn token_cost_per_token(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
if let Some(cost) = catalog
|
||||
.entry(request.model, request.provider, request.region)
|
||||
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
||||
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
|
||||
{
|
||||
return Ok(cost);
|
||||
}
|
||||
|
|
@ -219,18 +258,17 @@ pub fn cost_per_token(
|
|||
{
|
||||
return Ok((0.0, cost));
|
||||
}
|
||||
let key = catalog
|
||||
.select_model_key(request.model, request.provider, request.region)
|
||||
let selected = catalog
|
||||
.select_model_info(request.model, request.provider, request.region)
|
||||
.or_else(|| {
|
||||
(provider == Some(LlmProviders::FIREWORKS_AI))
|
||||
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
|
||||
.and_then(|category| {
|
||||
catalog.select_model_key(category, request.provider, request.region)
|
||||
catalog.select_model_info(category, request.provider, request.region)
|
||||
})
|
||||
})
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let model_info =
|
||||
apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider);
|
||||
let model_info = apply_provider_cache_read_default(&selected.info, request.provider);
|
||||
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
|
||||
return Ok(cost);
|
||||
}
|
||||
|
|
@ -262,7 +300,7 @@ pub fn cost_per_token(
|
|||
Some(LlmProviders::FIREWORKS_AI) => {
|
||||
return Ok(fireworks_cost_per_token(
|
||||
request.usage,
|
||||
catalog.entry_for_key(key),
|
||||
&selected.info,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
|
|
@ -313,7 +351,7 @@ pub fn cost_per_token(
|
|||
_ => (request.service_tier, request.data_residency),
|
||||
};
|
||||
let cost = generic_cost_per_token(GenericCostRequest {
|
||||
model_info: catalog.entry_for_key(key),
|
||||
model_info: &selected.info,
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier,
|
||||
|
|
@ -369,6 +407,7 @@ pub fn speech_cost(
|
|||
let model_info = catalog
|
||||
.entry(request.model, request.provider, request.region)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let model_info = &*model_info;
|
||||
match select_cost_metric_for_model(model_info)? {
|
||||
SpeechCostMetric::PerCharacter => {
|
||||
let characters = prompt_characters.ok_or(CostError::MissingPromptCharacters)?;
|
||||
|
|
@ -399,6 +438,7 @@ pub fn transcription_cost(
|
|||
let model_info = catalog
|
||||
.entry(request.model, request.provider, request.region)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let model_info = &*model_info;
|
||||
if transcription_usage_has_token_details(request.usage) {
|
||||
return Ok(generic_cost_per_token(GenericCostRequest {
|
||||
model_info,
|
||||
|
|
@ -421,6 +461,7 @@ pub fn handle_realtime_transcription_cost_calculation(
|
|||
) -> f64 {
|
||||
let model = get_transcription_model_name_from_results(results).unwrap_or(requested_model);
|
||||
let model_info = catalog.entry(model, Some(provider), None);
|
||||
let model_info = model_info.as_deref();
|
||||
results
|
||||
.iter()
|
||||
.filter(|event| {
|
||||
|
|
@ -467,7 +508,7 @@ pub fn handle_realtime_stream_cost_calculation(
|
|||
.find_map(|model| {
|
||||
let model_info = catalog.entry(model, Some(provider), None)?;
|
||||
let cost = generic_cost_per_token(GenericCostRequest {
|
||||
model_info,
|
||||
model_info: &model_info,
|
||||
usage: combined_usage,
|
||||
provider: Some(provider),
|
||||
service_tier: None,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ pub fn databricks_cost_per_token(
|
|||
) -> Result<(f64, f64), CostError> {
|
||||
if let Some(cost) = catalog
|
||||
.entry(request.model, Some("databricks"), request.region)
|
||||
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
||||
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
|
||||
{
|
||||
return Ok(cost);
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ pub fn databricks_cost_per_token(
|
|||
.ok_or(CostError::ModelNotFound)?;
|
||||
Ok(calculate_generic_cost_from_model_info_with_region(
|
||||
request.usage,
|
||||
entry,
|
||||
&entry,
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ pub enum CostError {
|
|||
MissingProvider,
|
||||
#[error("no rate for the priced unit")]
|
||||
MissingRate,
|
||||
#[error("image steps are not a number")]
|
||||
InvalidSteps,
|
||||
#[error("no usage on the response")]
|
||||
MissingUsage,
|
||||
#[error("model is not in the cost map")]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use serde_json::Value;
|
|||
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
use crate::image_response_cost::resolve_image_model_info;
|
||||
use crate::wire::py_real;
|
||||
|
||||
pub const PIXELS_PER_MEGAPIXEL: u64 = 1_048_576;
|
||||
|
||||
|
|
@ -136,12 +137,13 @@ pub fn fal_ai_passthrough_cost_from_model_info(
|
|||
}
|
||||
|
||||
pub fn cost_calculator(
|
||||
catalog: &ModelInfoCatalog,
|
||||
model: &str,
|
||||
image_response: &Value,
|
||||
optional_params: &Value,
|
||||
deployment_prices: Option<&Value>,
|
||||
entries: &HashMap<String, Value>,
|
||||
) -> Option<f64> {
|
||||
let entries = catalog.entries();
|
||||
let images = image_response.get("data").and_then(Value::as_array);
|
||||
let count = images.map_or(0, Vec::len);
|
||||
if let Some(rate) = deployment_prices
|
||||
|
|
@ -160,15 +162,16 @@ pub fn cost_calculator(
|
|||
if keyed.iter().all(Option::is_some) {
|
||||
return Some(keyed.into_iter().flatten().sum());
|
||||
}
|
||||
let shared = entries.get(&format!("fal_ai/{normalized_model}"));
|
||||
let resolved = resolve_image_model_info(shared, deployment_prices)?;
|
||||
let shared = catalog
|
||||
.get_model_info(normalized_model, Some("fal_ai"))
|
||||
.ok()
|
||||
.map(|model_info| model_info.info);
|
||||
let resolved = resolve_image_model_info(shared.as_deref(), deployment_prices)?;
|
||||
let flat_rate = resolved
|
||||
.get("output_cost_per_image")
|
||||
.and_then(Value::as_f64)
|
||||
.and_then(py_real)
|
||||
.unwrap_or(0.0);
|
||||
let pixel_rate = resolved
|
||||
.get("output_cost_per_pixel")
|
||||
.and_then(Value::as_f64);
|
||||
let pixel_rate = resolved.get("output_cost_per_pixel").and_then(py_real);
|
||||
Some(
|
||||
images
|
||||
.into_iter()
|
||||
|
|
|
|||
147
litellm-rust/crates/cost/src/fallback_generalizations.rs
Normal file
147
litellm-rust/crates/cost/src/fallback_generalizations.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use fancy_regex::{Regex, RegexBuilder};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
const PROVIDER_KEY: &str = "litellm_provider";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RoutingRule {
|
||||
pattern: Regex,
|
||||
provider: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapabilityRule {
|
||||
pattern: Regex,
|
||||
model_info: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct FallbackGeneralizations {
|
||||
routing_rules: Vec<RoutingRule>,
|
||||
capability_rules: Vec<CapabilityRule>,
|
||||
}
|
||||
|
||||
fn resolve_legacy_extends(rules: &[Value]) -> Vec<Value> {
|
||||
let base_by_name: HashMap<&str, &Map<String, Value>> = rules
|
||||
.iter()
|
||||
.filter_map(|rule| {
|
||||
Some((
|
||||
rule.get("name")?.as_str()?,
|
||||
rule.get("model_info")?.as_object()?,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
rules
|
||||
.iter()
|
||||
.map(|rule| {
|
||||
let parent = rule
|
||||
.get("extends")
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|name| base_by_name.get(name));
|
||||
match (parent, rule.get("model_info").and_then(Value::as_object)) {
|
||||
(Some(parent), Some(own)) => {
|
||||
let mut merged = (*parent).clone();
|
||||
merged.extend(own.clone());
|
||||
let mut resolved = rule.as_object().cloned().unwrap_or_default();
|
||||
resolved.insert("model_info".to_owned(), Value::Object(merged));
|
||||
Value::Object(resolved)
|
||||
}
|
||||
_ => rule.clone(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn fill_missing_is_malformed(rule: &Value) -> bool {
|
||||
rule.get("fill_missing_for_providers")
|
||||
.is_some_and(|providers| {
|
||||
!providers
|
||||
.as_array()
|
||||
.is_some_and(|providers| providers.iter().all(Value::is_string))
|
||||
})
|
||||
}
|
||||
|
||||
fn compile_rule(rule: &Value) -> (Option<RoutingRule>, Option<CapabilityRule>) {
|
||||
let (Some(pattern), Some(model_info)) = (
|
||||
rule.get("pattern").and_then(Value::as_str),
|
||||
rule.get("model_info").and_then(Value::as_object),
|
||||
) else {
|
||||
return (None, None);
|
||||
};
|
||||
let Ok(pattern) = RegexBuilder::new(pattern).case_insensitive(true).build() else {
|
||||
return (None, None);
|
||||
};
|
||||
if fill_missing_is_malformed(rule) {
|
||||
return (None, None);
|
||||
}
|
||||
let capability = || CapabilityRule {
|
||||
pattern: pattern.clone(),
|
||||
model_info: model_info.clone(),
|
||||
};
|
||||
match model_info.get(PROVIDER_KEY) {
|
||||
None => (None, Some(capability())),
|
||||
Some(Value::String(provider)) => {
|
||||
let routing = RoutingRule {
|
||||
pattern: pattern.clone(),
|
||||
provider: provider.clone(),
|
||||
};
|
||||
let capability = (model_info.len() > 1).then(capability);
|
||||
(Some(routing), capability)
|
||||
}
|
||||
Some(_) => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_match(pattern: &Regex, model: &str) -> bool {
|
||||
pattern.is_match(model).unwrap_or(false)
|
||||
}
|
||||
|
||||
impl FallbackGeneralizations {
|
||||
pub fn from_block(block: Option<&Value>) -> Self {
|
||||
let rules = block
|
||||
.and_then(|block| block.get("rules"))
|
||||
.and_then(Value::as_array)
|
||||
.map(|rules| resolve_legacy_extends(rules))
|
||||
.unwrap_or_default();
|
||||
let (routing_rules, capability_rules): (Vec<_>, Vec<_>) =
|
||||
rules.iter().map(compile_rule).unzip();
|
||||
Self {
|
||||
routing_rules: routing_rules.into_iter().flatten().collect(),
|
||||
capability_rules: capability_rules.into_iter().flatten().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn match_routing(&self, model: &str) -> Option<&str> {
|
||||
if model.is_empty() {
|
||||
return None;
|
||||
}
|
||||
self.routing_rules
|
||||
.iter()
|
||||
.find(|rule| is_match(&rule.pattern, model))
|
||||
.map(|rule| rule.provider.as_str())
|
||||
}
|
||||
|
||||
pub fn match_capabilities(&self, model: &str) -> Option<Map<String, Value>> {
|
||||
if model.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let matched: Vec<&Map<String, Value>> = self
|
||||
.capability_rules
|
||||
.iter()
|
||||
.filter(|rule| is_match(&rule.pattern, model))
|
||||
.map(|rule| &rule.model_info)
|
||||
.collect();
|
||||
(!matched.is_empty()).then(|| {
|
||||
matched
|
||||
.into_iter()
|
||||
.flat_map(|model_info| {
|
||||
model_info
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
291
litellm-rust/crates/cost/src/get_llm_provider.rs
Normal file
291
litellm-rust/crates/cost/src/get_llm_provider.rs
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::str::FromStr;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::fallback_generalizations::FallbackGeneralizations;
|
||||
use crate::provider::LlmProviders;
|
||||
|
||||
const OPENAI_IMAGE_GENERATION_MODELS: [&str; 2] = ["dall-e-2", "dall-e-3"];
|
||||
const OPENAI_VIDEO_GENERATION_MODELS: [&str; 1] = ["sora-2"];
|
||||
const COHERE_EMBEDDING_MODELS: [&str; 7] = [
|
||||
"embed-v4.0",
|
||||
"embed-english-v3.0",
|
||||
"embed-english-light-v3.0",
|
||||
"embed-multilingual-v3.0",
|
||||
"embed-english-v2.0",
|
||||
"embed-english-light-v2.0",
|
||||
"embed-multilingual-v2.0",
|
||||
];
|
||||
const REPLICATE_MODEL_NAME_WITH_ID_LENGTH: usize = 64;
|
||||
const PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: [&str; 2] = ["github_copilot", "chatgpt"];
|
||||
const MODEL_PREFIX_PROVIDERS: [(&str, &str); 11] = [
|
||||
("bytez/", "bytez"),
|
||||
("gdc/", "gdc"),
|
||||
("lemonade/", "lemonade"),
|
||||
("heroku/", "heroku"),
|
||||
("cometapi/", "cometapi"),
|
||||
("oci/", "oci"),
|
||||
("compactifai/", "compactifai"),
|
||||
("ovhcloud/", "ovhcloud"),
|
||||
("clarifai/", "clarifai"),
|
||||
("amazon_nova", "amazon_nova"),
|
||||
("sap/", "sap"),
|
||||
];
|
||||
|
||||
static JSON_PROVIDERS: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
||||
serde_json::from_str::<serde_json::Map<String, Value>>(include_str!(
|
||||
"../../../../litellm/llms/openai_like/providers.json"
|
||||
))
|
||||
.expect("openai-like provider registry parses")
|
||||
.into_iter()
|
||||
.map(|(slug, _)| slug)
|
||||
.collect()
|
||||
});
|
||||
|
||||
static BEDROCK_PRICING_ONLY: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^bedrock/[a-zA-Z0-9_-]+/.+$").expect("valid bedrock pricing pattern")
|
||||
});
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum ModelList {
|
||||
OpenAiChatCompletion,
|
||||
OpenAiTextCompletion,
|
||||
Anthropic,
|
||||
Cohere,
|
||||
CohereChat,
|
||||
MistralChat,
|
||||
Openrouter,
|
||||
VertexText,
|
||||
VertexCodeText,
|
||||
VertexLanguage,
|
||||
VertexVision,
|
||||
VertexChat,
|
||||
VertexCodeChat,
|
||||
VertexEmbedding,
|
||||
VertexLlama3,
|
||||
VertexMistral,
|
||||
VertexAi21,
|
||||
VertexImage,
|
||||
VertexVideo,
|
||||
Ai21Chat,
|
||||
Ai21,
|
||||
NlpCloud,
|
||||
AlephAlpha,
|
||||
Bedrock,
|
||||
BedrockConverse,
|
||||
Watsonx,
|
||||
Empower,
|
||||
GradientAi,
|
||||
}
|
||||
|
||||
fn is_openai_finetune_model(key: &str) -> bool {
|
||||
key.starts_with("ft:") && key.matches(':').count() <= 1
|
||||
}
|
||||
|
||||
fn is_bedrock_pricing_only_model(key: &str) -> bool {
|
||||
BEDROCK_PRICING_ONLY.is_match(key)
|
||||
}
|
||||
|
||||
fn vertex_member(key: &str) -> String {
|
||||
key.replace("vertex_ai/", "")
|
||||
}
|
||||
|
||||
fn model_list_membership(key: &str, info: &Value) -> Option<(ModelList, String)> {
|
||||
let provider = info.get("litellm_provider").and_then(Value::as_str)?;
|
||||
let mode = info.get("mode").and_then(Value::as_str);
|
||||
let list = match provider {
|
||||
"openai" if !is_openai_finetune_model(key) => ModelList::OpenAiChatCompletion,
|
||||
"text-completion-openai" => ModelList::OpenAiTextCompletion,
|
||||
"cohere" => ModelList::Cohere,
|
||||
"cohere_chat" => ModelList::CohereChat,
|
||||
"mistral" => ModelList::MistralChat,
|
||||
"anthropic" => ModelList::Anthropic,
|
||||
"empower" => ModelList::Empower,
|
||||
"openrouter" => ModelList::Openrouter,
|
||||
"vertex_ai-text-models" => ModelList::VertexText,
|
||||
"vertex_ai-code-text-models" => ModelList::VertexCodeText,
|
||||
"vertex_ai-language-models" => ModelList::VertexLanguage,
|
||||
"vertex_ai-vision-models" => ModelList::VertexVision,
|
||||
"vertex_ai-chat-models" => ModelList::VertexChat,
|
||||
"vertex_ai-code-chat-models" => ModelList::VertexCodeChat,
|
||||
"vertex_ai-embedding-models" => ModelList::VertexEmbedding,
|
||||
"vertex_ai-llama_models" => return Some((ModelList::VertexLlama3, vertex_member(key))),
|
||||
"vertex_ai-mistral_models" => return Some((ModelList::VertexMistral, vertex_member(key))),
|
||||
"vertex_ai-ai21_models" => return Some((ModelList::VertexAi21, vertex_member(key))),
|
||||
"vertex_ai-image-models" => return Some((ModelList::VertexImage, vertex_member(key))),
|
||||
"vertex_ai-video-models" => return Some((ModelList::VertexVideo, vertex_member(key))),
|
||||
"ai21" if mode == Some("chat") => ModelList::Ai21Chat,
|
||||
"ai21" => ModelList::Ai21,
|
||||
"nlp_cloud" => ModelList::NlpCloud,
|
||||
"aleph_alpha" => ModelList::AlephAlpha,
|
||||
"bedrock" if mode == Some("guardrail") => return None,
|
||||
"bedrock" if !is_bedrock_pricing_only_model(key) => ModelList::Bedrock,
|
||||
"bedrock_converse" => ModelList::BedrockConverse,
|
||||
"watsonx" => ModelList::Watsonx,
|
||||
"gradient_ai" => ModelList::GradientAi,
|
||||
_ => return None,
|
||||
};
|
||||
Some((list, key.to_owned()))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ProviderModelSets {
|
||||
members: HashMap<ModelList, HashSet<String>>,
|
||||
}
|
||||
|
||||
impl ProviderModelSets {
|
||||
pub fn from_model_cost(model_cost: &HashMap<String, Value>) -> Self {
|
||||
let members = model_cost
|
||||
.iter()
|
||||
.filter_map(|(key, info)| model_list_membership(key, info))
|
||||
.fold(
|
||||
HashMap::<ModelList, HashSet<String>>::new(),
|
||||
|mut members, (list, name)| {
|
||||
members.entry(list).or_default().insert(name);
|
||||
members
|
||||
},
|
||||
);
|
||||
Self { members }
|
||||
}
|
||||
|
||||
pub fn contains(&self, list: ModelList, model: &str) -> bool {
|
||||
self.members
|
||||
.get(&list)
|
||||
.is_some_and(|models| models.contains(model))
|
||||
}
|
||||
|
||||
fn contains_any(&self, lists: &[ModelList], model: &str) -> bool {
|
||||
lists.iter().any(|list| self.contains(*list, model))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct LlmProvider {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
}
|
||||
|
||||
pub fn declared_authenticating_provider(model: &str) -> Option<&str> {
|
||||
model
|
||||
.split_once('/')
|
||||
.map(|(declared, _)| declared)
|
||||
.filter(|declared| PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO.contains(declared))
|
||||
}
|
||||
|
||||
fn is_anthropic_text_model(model: &str) -> bool {
|
||||
matches!(model, "claude-2" | "claude-instant-1")
|
||||
}
|
||||
|
||||
fn resolved(model: &str, provider: &str) -> Option<LlmProvider> {
|
||||
Some(LlmProvider {
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: provider.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
fn bare_model_provider<'a>(model: &str, sets: &ProviderModelSets) -> Option<&'a str> {
|
||||
use ModelList::*;
|
||||
let replicate_id = model
|
||||
.split(':')
|
||||
.nth(1)
|
||||
.is_some_and(|id| id.len() == REPLICATE_MODEL_NAME_WITH_ID_LENGTH);
|
||||
if sets.contains(OpenAiChatCompletion, model)
|
||||
|| model.contains("ft:gpt-3.5-turbo")
|
||||
|| model.contains("ft:gpt-4")
|
||||
|| OPENAI_IMAGE_GENERATION_MODELS.contains(&model)
|
||||
|| model.starts_with("gpt-image")
|
||||
|| OPENAI_VIDEO_GENERATION_MODELS.contains(&model)
|
||||
{
|
||||
Some("openai")
|
||||
} else if sets.contains(OpenAiTextCompletion, model) {
|
||||
Some("text-completion-openai")
|
||||
} else if sets.contains(Anthropic, model) {
|
||||
Some(if is_anthropic_text_model(model) {
|
||||
"anthropic_text"
|
||||
} else {
|
||||
"anthropic"
|
||||
})
|
||||
} else if sets.contains(Cohere, model) || COHERE_EMBEDDING_MODELS.contains(&model) {
|
||||
Some("cohere")
|
||||
} else if sets.contains(CohereChat, model) {
|
||||
Some("cohere_chat")
|
||||
} else if model.contains(':') && model.len() > REPLICATE_MODEL_NAME_WITH_ID_LENGTH {
|
||||
replicate_id.then_some("replicate")
|
||||
} else if sets.contains(Openrouter, model) {
|
||||
Some("openrouter")
|
||||
} else if sets.contains_any(
|
||||
&[
|
||||
VertexChat,
|
||||
VertexCodeChat,
|
||||
VertexText,
|
||||
VertexCodeText,
|
||||
VertexLanguage,
|
||||
VertexEmbedding,
|
||||
VertexVision,
|
||||
VertexImage,
|
||||
VertexVideo,
|
||||
],
|
||||
model,
|
||||
) {
|
||||
Some("vertex_ai")
|
||||
} else if sets.contains_any(&[Ai21Chat, Ai21], model) {
|
||||
Some("ai21_chat")
|
||||
} else if sets.contains(AlephAlpha, model) {
|
||||
Some("aleph_alpha")
|
||||
} else if sets.contains(NlpCloud, model) {
|
||||
Some("nlp_cloud")
|
||||
} else if sets.contains_any(&[Bedrock, BedrockConverse], model) {
|
||||
Some("bedrock")
|
||||
} else if sets.contains(Watsonx, model) {
|
||||
Some("watsonx")
|
||||
} else if sets.contains(Empower, model) {
|
||||
Some("empower")
|
||||
} else if sets.contains(GradientAi, model) {
|
||||
Some("gradient_ai")
|
||||
} else if model == "*" {
|
||||
Some("openai")
|
||||
} else {
|
||||
MODEL_PREFIX_PROVIDERS
|
||||
.iter()
|
||||
.find(|(prefix, _)| model.starts_with(prefix))
|
||||
.map(|(_, provider)| *provider)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_llm_provider(
|
||||
model: &str,
|
||||
sets: &ProviderModelSets,
|
||||
generalizations: &FallbackGeneralizations,
|
||||
) -> Option<LlmProvider> {
|
||||
if let Some((prefix, rest)) = model.split_once('/') {
|
||||
if prefix == "azure"
|
||||
&& (sets.contains(ModelList::CohereChat, rest)
|
||||
|| sets.contains(ModelList::MistralChat, &format!("mistral/{rest}")))
|
||||
{
|
||||
return resolved(model, "openai");
|
||||
}
|
||||
if prefix == "cohere" && sets.contains(ModelList::CohereChat, rest) {
|
||||
return resolved(rest, "cohere_chat");
|
||||
}
|
||||
if prefix == "anthropic" && is_anthropic_text_model(rest) {
|
||||
return resolved(rest, "anthropic_text");
|
||||
}
|
||||
if JSON_PROVIDERS.contains(prefix) {
|
||||
return resolved(rest, prefix);
|
||||
}
|
||||
if LlmProviders::from_str(prefix).is_ok() {
|
||||
let provider = if prefix == "ai21" {
|
||||
"ai21_chat"
|
||||
} else {
|
||||
prefix
|
||||
};
|
||||
return resolved(rest, provider);
|
||||
}
|
||||
}
|
||||
bare_model_provider(model, sets)
|
||||
.or_else(|| generalizations.match_routing(model))
|
||||
.and_then(|provider| resolved(model, provider))
|
||||
}
|
||||
|
|
@ -282,21 +282,20 @@ pub fn route_image_generation_cost_calculator<'a>(
|
|||
pricing.as_ref(),
|
||||
),
|
||||
Some(LlmProviders::FAL_AI) => fal_ai_image_cost_calculator(
|
||||
catalog,
|
||||
request.model,
|
||||
request.image_response,
|
||||
request.optional_params,
|
||||
pricing.as_ref(),
|
||||
catalog.entries(),
|
||||
)
|
||||
.ok_or(CostError::ModelNotFound),
|
||||
Some(LlmProviders::BEDROCK) => bedrock_image_cost_calculator(
|
||||
catalog,
|
||||
request.model,
|
||||
request.image_response,
|
||||
Some(resolved_size),
|
||||
request.optional_params,
|
||||
catalog.entries(),
|
||||
)
|
||||
.ok_or(CostError::ModelNotFound),
|
||||
),
|
||||
Some(provider @ (LlmProviders::OPENAI | LlmProviders::AZURE))
|
||||
if request.model.to_ascii_lowercase().contains("gpt-image") =>
|
||||
{
|
||||
|
|
@ -330,7 +329,11 @@ fn resolved_model_info(
|
|||
supplied_model_info: Option<&Value>,
|
||||
) -> Result<Value, CostError> {
|
||||
resolve_image_model_info(
|
||||
catalog.entry(model, Some(provider), None),
|
||||
catalog
|
||||
.get_model_info(model, Some(provider))
|
||||
.ok()
|
||||
.map(|model_info| model_info.info)
|
||||
.as_deref(),
|
||||
supplied_model_info,
|
||||
)
|
||||
.ok_or(CostError::ModelNotFound)
|
||||
|
|
@ -374,9 +377,8 @@ pub fn google_image_edit_cost(
|
|||
Ok(gemini_image_edit_cost(image_response, &model_info, at))
|
||||
}
|
||||
"vertex_ai" => catalog
|
||||
.entry(model, Some(provider), None)
|
||||
.map(|model_info| vertex_image_edit_cost(image_response, model_info))
|
||||
.ok_or(CostError::ModelNotFound),
|
||||
.get_model_info(model, Some(provider))
|
||||
.map(|model_info| vertex_image_edit_cost(image_response, &model_info.info)),
|
||||
_ => Err(CostError::ModelNotFound),
|
||||
}
|
||||
}
|
||||
|
|
@ -385,7 +387,8 @@ pub fn azure_ai_image_generation_cost(
|
|||
catalog: &ModelInfoCatalog,
|
||||
request: AzureAiImageCatalogRequest<'_>,
|
||||
) -> Result<f64, CostError> {
|
||||
let shared = catalog.entry(request.model, Some("azure_ai"), None);
|
||||
let selected = catalog.get_model_info(request.model, Some("azure_ai")).ok();
|
||||
let shared = selected.as_ref().map(|selected| &*selected.info);
|
||||
let model_info = resolve_image_model_info(shared, request.supplied_model_info)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let shared_pricing = model_info
|
||||
|
|
@ -395,9 +398,9 @@ pub fn azure_ai_image_generation_cost(
|
|||
.or(shared);
|
||||
azure_ai_image_cost_calculator(AzureAiImageRequest {
|
||||
catalog,
|
||||
model: catalog
|
||||
.select_model_key(request.model, Some("azure_ai"), None)
|
||||
.unwrap_or(request.model),
|
||||
model: selected
|
||||
.as_ref()
|
||||
.map_or(request.model, |selected| &selected.key),
|
||||
image_response: request.image_response,
|
||||
model_info: &model_info,
|
||||
supplied_model_info: request.supplied_model_info,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ pub mod background_cost_polling;
|
|||
pub mod base_rate_selection;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod batch;
|
||||
pub mod bedrock_common_utils;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod bedrock_image_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
|
|
@ -40,6 +41,8 @@ pub mod error;
|
|||
#[allow(clippy::disallowed_types)]
|
||||
pub mod fal_ai_image_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod fallback_generalizations;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod fireworks_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod gemini_cost;
|
||||
|
|
@ -51,6 +54,8 @@ pub mod generic_input;
|
|||
pub mod generic_output;
|
||||
pub mod generic_usage;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod get_llm_provider;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod groq_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod guardrail_cost;
|
||||
|
|
@ -65,6 +70,8 @@ pub mod lemonade_cost;
|
|||
#[allow(clippy::disallowed_types)]
|
||||
pub mod mcp_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod model_info;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod model_selection;
|
||||
pub mod non_token;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
|
|
|
|||
306
litellm-rust/crates/cost/src/model_info.rs
Normal file
306
litellm-rust/crates/cost/src/model_info.rs
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::bedrock_common_utils::{
|
||||
get_bedrock_base_model, is_bedrock_region, strip_bedrock_routing_prefix,
|
||||
};
|
||||
use crate::error::CostError;
|
||||
use crate::fallback_generalizations::FallbackGeneralizations;
|
||||
use crate::get_llm_provider::{
|
||||
ModelList, ProviderModelSets, declared_authenticating_provider, get_llm_provider,
|
||||
};
|
||||
|
||||
const AZURE_LLMS: [(&str, &str); 7] = [
|
||||
("gpt-35-turbo", "azure/gpt-35-turbo"),
|
||||
("gpt-35-turbo-16k", "azure/gpt-35-turbo-16k"),
|
||||
("gpt-35-turbo-instruct", "azure/gpt-35-turbo-instruct"),
|
||||
("azure/gpt-41", "gpt-4.1"),
|
||||
("azure/gpt-41-mini", "gpt-4.1-mini"),
|
||||
("azure/gpt-41-nano", "gpt-4.1-nano"),
|
||||
("ada", "azure/ada"),
|
||||
];
|
||||
|
||||
static STABLE_VERTEX_VERSION: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"-\d+$").expect("valid vertex version pattern"));
|
||||
static DATED_SNAPSHOT_SUFFIX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"-\d{4}-\d{2}-\d{2}$").expect("valid snapshot pattern"));
|
||||
static OPENAI_FINETUNE_SUFFIX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(:[^:]*){3}$").expect("valid finetune pattern"));
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct PotentialModelNames {
|
||||
pub split_model: String,
|
||||
pub combined_model_name: String,
|
||||
pub stripped_model_name: String,
|
||||
pub combined_stripped_model_name: String,
|
||||
pub provider_prefixed_model_name: String,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ModelInfo<'a> {
|
||||
pub key: Cow<'a, str>,
|
||||
pub info: Cow<'a, Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ModelCost<'a> {
|
||||
pub entries: &'a HashMap<String, Value>,
|
||||
pub lowercase_keys: &'a HashMap<String, String>,
|
||||
pub sets: &'a ProviderModelSets,
|
||||
pub generalizations: &'a FallbackGeneralizations,
|
||||
}
|
||||
|
||||
pub fn strip_model_name(model: &str, custom_llm_provider: Option<&str>) -> String {
|
||||
match custom_llm_provider {
|
||||
Some("bedrock" | "bedrock_converse") => get_bedrock_base_model(model),
|
||||
Some("vertex_ai" | "gemini" | "databricks") => {
|
||||
STABLE_VERTEX_VERSION.replace(model, "").into_owned()
|
||||
}
|
||||
_ if model.contains("ft:") => OPENAI_FINETUNE_SUFFIX.replace(model, "").into_owned(),
|
||||
_ => DATED_SNAPSHOT_SUFFIX.replace(model, "").into_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn split_mantle_region_prefix(model: &str) -> &str {
|
||||
match model.split_once('/') {
|
||||
Some((head, tail)) if is_bedrock_region(head) => tail,
|
||||
_ => model,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_fireworks_resource_name(model: &str) -> String {
|
||||
let stripped = model.strip_prefix("fireworks_ai/").unwrap_or(model);
|
||||
if stripped.starts_with("accounts/") || stripped.starts_with("FW-") || stripped.contains('#') {
|
||||
stripped.to_owned()
|
||||
} else if stripped.starts_with("routers/") || stripped.starts_with("models/") {
|
||||
format!("accounts/fireworks/{stripped}")
|
||||
} else if stripped.ends_with("-fast") {
|
||||
format!("accounts/fireworks/routers/{stripped}")
|
||||
} else {
|
||||
format!("accounts/fireworks/models/{stripped}")
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_potential_model_names(
|
||||
model_cost: ModelCost<'_>,
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> PotentialModelNames {
|
||||
let (split_model, combined, stripped, combined_stripped, provider_prefixed, provider) =
|
||||
match custom_llm_provider {
|
||||
None => {
|
||||
let inferred = get_llm_provider(model, model_cost.sets, model_cost.generalizations);
|
||||
let provider = inferred
|
||||
.as_ref()
|
||||
.map(|inferred| inferred.custom_llm_provider.clone());
|
||||
let stripped = strip_model_name(model, provider.as_deref());
|
||||
(
|
||||
inferred.map_or_else(|| model.to_owned(), |inferred| inferred.model),
|
||||
model.to_owned(),
|
||||
stripped.clone(),
|
||||
stripped,
|
||||
model.to_owned(),
|
||||
provider,
|
||||
)
|
||||
}
|
||||
Some(provider) => {
|
||||
let (split, combined, provider_prefixed) =
|
||||
match model.strip_prefix(&format!("{provider}/")) {
|
||||
Some(split) => (split, model.to_owned(), format!("{provider}/{model}")),
|
||||
None => (
|
||||
model,
|
||||
format!("{provider}/{model}"),
|
||||
format!("{provider}/{model}"),
|
||||
),
|
||||
};
|
||||
let stripped = strip_model_name(split, Some(provider));
|
||||
let combined_stripped = format!("{provider}/{stripped}");
|
||||
(
|
||||
split.to_owned(),
|
||||
combined,
|
||||
stripped,
|
||||
combined_stripped,
|
||||
provider_prefixed,
|
||||
Some(provider.to_owned()),
|
||||
)
|
||||
}
|
||||
};
|
||||
let split_model = match provider.as_deref() {
|
||||
Some("bedrock" | "bedrock_converse") => {
|
||||
strip_bedrock_routing_prefix(&split_model).to_owned()
|
||||
}
|
||||
_ => split_model,
|
||||
};
|
||||
let (split_model, combined_stripped) = match provider.as_deref() {
|
||||
Some(provider @ "bedrock_mantle") => {
|
||||
let region_free = split_mantle_region_prefix(&split_model).to_owned();
|
||||
let combined_stripped = format!(
|
||||
"bedrock_mantle/{}",
|
||||
strip_model_name(®ion_free, Some(provider))
|
||||
);
|
||||
(region_free, combined_stripped)
|
||||
}
|
||||
_ => (split_model, combined_stripped),
|
||||
};
|
||||
let provider_prefixed = match provider.as_deref() {
|
||||
Some("fireworks_ai") => format!(
|
||||
"fireworks_ai/{}",
|
||||
resolve_fireworks_resource_name(&split_model)
|
||||
),
|
||||
_ => provider_prefixed,
|
||||
};
|
||||
PotentialModelNames {
|
||||
split_model,
|
||||
combined_model_name: combined,
|
||||
stripped_model_name: stripped,
|
||||
combined_stripped_model_name: combined_stripped,
|
||||
provider_prefixed_model_name: provider_prefixed,
|
||||
custom_llm_provider: provider,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_model_cost_key<'a>(model_cost: ModelCost<'a>, potential_key: &str) -> Option<&'a str> {
|
||||
model_cost
|
||||
.entries
|
||||
.get_key_value(potential_key)
|
||||
.map(|(key, _)| key.as_str())
|
||||
.or_else(|| {
|
||||
model_cost
|
||||
.lowercase_keys
|
||||
.get(&potential_key.to_lowercase())
|
||||
.map(String::as_str)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn lowercase_key_map(model_cost: &HashMap<String, Value>) -> HashMap<String, String> {
|
||||
model_cost.keys().fold(HashMap::new(), |mut map, key| {
|
||||
map.entry(key.to_lowercase())
|
||||
.and_modify(|existing: &mut String| {
|
||||
if key < existing {
|
||||
existing.clone_from(key);
|
||||
}
|
||||
})
|
||||
.or_insert_with(|| key.clone());
|
||||
map
|
||||
})
|
||||
}
|
||||
|
||||
fn vertex_ai_model_alias(model: &str, sets: &ProviderModelSets) -> Option<String> {
|
||||
let meta = format!("meta/{model}");
|
||||
if sets.contains(ModelList::VertexLlama3, &meta) {
|
||||
return Some(meta);
|
||||
}
|
||||
let latest = format!("{model}@latest");
|
||||
(sets.contains(ModelList::VertexMistral, &latest)
|
||||
|| sets.contains(ModelList::VertexAi21, &latest))
|
||||
.then_some(latest)
|
||||
}
|
||||
|
||||
fn get_model_info_from_generalization(
|
||||
model_cost: ModelCost<'_>,
|
||||
candidates: &[&str],
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Option<(String, Value)> {
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| get_model_cost_key(model_cost, candidate).is_some())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
candidates.iter().find_map(|candidate| {
|
||||
let mut info = model_cost.generalizations.match_capabilities(candidate)?;
|
||||
if let Some(provider) = custom_llm_provider {
|
||||
info.insert(
|
||||
"litellm_provider".to_owned(),
|
||||
Value::String(provider.to_owned()),
|
||||
);
|
||||
}
|
||||
Some(((*candidate).to_owned(), Value::Object(info)))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_model_info_helper<'a>(
|
||||
model_cost: ModelCost<'a>,
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
) -> Result<ModelInfo<'a>, CostError> {
|
||||
let model = AZURE_LLMS
|
||||
.iter()
|
||||
.find(|(alias, _)| *alias == model)
|
||||
.map_or(model, |(_, target)| *target);
|
||||
let custom_llm_provider = custom_llm_provider.map(|provider| match provider {
|
||||
"vertex_ai_beta" => "vertex_ai",
|
||||
provider => provider,
|
||||
});
|
||||
let model = match custom_llm_provider {
|
||||
Some("vertex_ai") => {
|
||||
vertex_ai_model_alias(model, model_cost.sets).map_or(Cow::Borrowed(model), Cow::Owned)
|
||||
}
|
||||
_ => Cow::Borrowed(model),
|
||||
};
|
||||
let names = get_potential_model_names(
|
||||
model_cost,
|
||||
&model,
|
||||
custom_llm_provider.or_else(|| declared_authenticating_provider(&model)),
|
||||
);
|
||||
let provider = names.custom_llm_provider.as_deref();
|
||||
if provider == Some("huggingface") {
|
||||
return Ok(ModelInfo {
|
||||
key: Cow::Owned(model.into_owned()),
|
||||
info: Cow::Owned(json!({
|
||||
"input_cost_per_token": 0,
|
||||
"output_cost_per_token": 0,
|
||||
"litellm_provider": "huggingface",
|
||||
"mode": "chat"
|
||||
})),
|
||||
});
|
||||
}
|
||||
let candidates = [
|
||||
names.combined_model_name.as_str(),
|
||||
&model,
|
||||
&names.split_model,
|
||||
&names.combined_stripped_model_name,
|
||||
&names.stripped_model_name,
|
||||
&names.provider_prefixed_model_name,
|
||||
];
|
||||
let mapped = candidates
|
||||
.iter()
|
||||
.filter_map(|candidate| get_model_cost_key(model_cost, candidate))
|
||||
.find_map(|key| {
|
||||
let info = &model_cost.entries[key];
|
||||
check_provider_match(info, provider).then_some(ModelInfo {
|
||||
key: Cow::Borrowed(key),
|
||||
info: Cow::Borrowed(info),
|
||||
})
|
||||
});
|
||||
if let Some(mapped) = mapped {
|
||||
return Ok(mapped);
|
||||
}
|
||||
get_model_info_from_generalization(model_cost, &candidates, provider)
|
||||
.map(|(key, info)| ModelInfo {
|
||||
key: Cow::Owned(key),
|
||||
info: Cow::Owned(info),
|
||||
})
|
||||
.ok_or(CostError::ModelNotFound)
|
||||
}
|
||||
|
||||
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
|
||||
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
|
||||
return true;
|
||||
};
|
||||
entry_provider == provider
|
||||
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|
||||
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|
||||
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|
||||
|| provider == "litellm_proxy"
|
||||
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|
||||
|| provider == "github"
|
||||
}
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
use std::collections::HashMap;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
use crate::provider::LlmProviders;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ModelSelectionRequest<'a> {
|
||||
pub model: Option<&'a str>,
|
||||
|
|
@ -12,7 +15,6 @@ pub struct ModelSelectionRequest<'a> {
|
|||
pub provider: Option<&'a str>,
|
||||
pub router_model_id: Option<&'a str>,
|
||||
pub region_name: Option<&'a str>,
|
||||
pub known_providers: &'a [&'a str],
|
||||
}
|
||||
|
||||
pub fn get_response_model(response: Option<&Value>) -> Option<&str> {
|
||||
|
|
@ -29,17 +31,20 @@ pub fn get_hidden_str_for_cost_calc<'a>(
|
|||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
pub fn model_contains_known_llm_provider(model: &str, known_providers: &[&str]) -> bool {
|
||||
let prefix = model.split('/').next().unwrap_or(model);
|
||||
known_providers.contains(&prefix)
|
||||
fn is_llm_provider(name: &str) -> bool {
|
||||
LlmProviders::from_str(name).is_ok()
|
||||
}
|
||||
|
||||
pub fn model_contains_known_llm_provider(model: &str) -> bool {
|
||||
model.split('/').next().is_some_and(is_llm_provider)
|
||||
}
|
||||
|
||||
pub fn strip_unregistered_leading_segments(
|
||||
model: &str,
|
||||
region_name: Option<&str>,
|
||||
known_providers: &[&str],
|
||||
cost_map: &HashMap<String, Value>,
|
||||
catalog: &ModelInfoCatalog,
|
||||
) -> String {
|
||||
let cost_map = catalog.entries();
|
||||
let segments: Vec<&str> = model.split('/').collect();
|
||||
if segments.len() < 2 || cost_map.contains_key(&segments[1..].join("/")) {
|
||||
return model.to_owned();
|
||||
|
|
@ -55,7 +60,7 @@ pub fn strip_unregistered_leading_segments(
|
|||
let tail = &segments[head_len..];
|
||||
let first_provider = tail
|
||||
.iter()
|
||||
.position(|segment| known_providers.contains(segment))
|
||||
.position(|segment| is_llm_provider(segment))
|
||||
.unwrap_or(tail.len());
|
||||
(0..=first_provider.min(tail.len().saturating_sub(1)))
|
||||
.map(|start| format!("{head}/{}", tail[start..].join("/")))
|
||||
|
|
@ -76,38 +81,22 @@ fn has_explicit_pricing(entry: &Value) -> bool {
|
|||
|
||||
pub fn get_provider_for_cost_calc(
|
||||
model: Option<&str>,
|
||||
explicit_provider: Option<&str>,
|
||||
known_providers: &[&str],
|
||||
cost_map: &HashMap<String, Value>,
|
||||
custom_llm_provider: Option<&str>,
|
||||
catalog: &ModelInfoCatalog,
|
||||
) -> Option<String> {
|
||||
explicit_provider
|
||||
.map(str::to_owned)
|
||||
.or_else(|| {
|
||||
model
|
||||
.and_then(|model| model.split_once('/'))
|
||||
.map(|(provider, _)| provider)
|
||||
.filter(|provider| known_providers.contains(provider))
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.or_else(|| {
|
||||
model
|
||||
.and_then(|model| cost_map.get(model))
|
||||
.and_then(|info| info.get("litellm_provider"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
})
|
||||
custom_llm_provider.map(str::to_owned).or_else(|| {
|
||||
catalog
|
||||
.get_llm_provider(model?)
|
||||
.map(|resolved| resolved.custom_llm_provider)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn select_model_name_for_cost_calc(
|
||||
request: ModelSelectionRequest<'_>,
|
||||
cost_map: &HashMap<String, Value>,
|
||||
catalog: &ModelInfoCatalog,
|
||||
) -> Option<String> {
|
||||
let provider = get_provider_for_cost_calc(
|
||||
request.model,
|
||||
request.provider,
|
||||
request.known_providers,
|
||||
cost_map,
|
||||
);
|
||||
let cost_map = catalog.entries();
|
||||
let provider = get_provider_for_cost_calc(request.model, request.provider, catalog);
|
||||
let response_model = get_response_model(request.response);
|
||||
let private_model =
|
||||
get_hidden_str_for_cost_calc(request.hidden_params, "provider_response_model");
|
||||
|
|
@ -135,7 +124,7 @@ pub fn select_model_name_for_cost_calc(
|
|||
let Some(provider) = provider else {
|
||||
return Some(selected.to_owned());
|
||||
};
|
||||
if model_contains_known_llm_provider(selected, request.known_providers) {
|
||||
if model_contains_known_llm_provider(selected) {
|
||||
return Some(selected.to_owned());
|
||||
}
|
||||
let prefix =
|
||||
|
|
@ -143,7 +132,6 @@ pub fn select_model_name_for_cost_calc(
|
|||
Some(strip_unregistered_leading_segments(
|
||||
&format!("{prefix}/{selected}"),
|
||||
priced_region,
|
||||
request.known_providers,
|
||||
cost_map,
|
||||
catalog,
|
||||
))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ pub fn prompt_caching_savings_for_model(
|
|||
let model_info = catalog.entry(request.model, request.provider, request.region)?;
|
||||
Some(calculate_prompt_caching_savings(
|
||||
PromptCachingSavingsRequest {
|
||||
model_info,
|
||||
model_info: &model_info,
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier: request.service_tier,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ pub fn rerank_cost(
|
|||
) -> (f64, f64) {
|
||||
rerank_cost_from_model_info(
|
||||
provider,
|
||||
catalog.entry(model, Some(provider), region),
|
||||
catalog.entry(model, Some(provider), region).as_deref(),
|
||||
billed_units,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ pub fn search_provider_cost_per_query(
|
|||
.entry(model, provider, None)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
Ok(search_provider_cost_from_model_info(
|
||||
model_info,
|
||||
&model_info,
|
||||
number_of_queries,
|
||||
optional_params,
|
||||
))
|
||||
|
|
|
|||
|
|
@ -45,10 +45,17 @@ fn usage_reports_web_search(usage: &ChatUsage) -> bool {
|
|||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ResolvedModelInfo<'a> {
|
||||
pub model_info: Option<&'a Value>,
|
||||
pub provider: Option<&'a str>,
|
||||
pub model_info: Option<Cow<'a, Value>>,
|
||||
pub provider: Option<Cow<'a, str>>,
|
||||
}
|
||||
|
||||
fn litellm_provider<'a>(model_info: &Value) -> Option<Cow<'a, str>> {
|
||||
model_info
|
||||
.get("litellm_provider")
|
||||
.and_then(Value::as_str)
|
||||
.map(|provider| Cow::Owned(provider.to_owned()))
|
||||
}
|
||||
|
||||
pub fn resolve_model_info<'a>(
|
||||
|
|
@ -57,14 +64,13 @@ pub fn resolve_model_info<'a>(
|
|||
provider: Option<&'a str>,
|
||||
region: Option<&str>,
|
||||
) -> ResolvedModelInfo<'a> {
|
||||
let litellm_provider =
|
||||
|model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str);
|
||||
if let Some(direct) = catalog.entry(model, provider, region) {
|
||||
return ResolvedModelInfo {
|
||||
model_info: Some(direct),
|
||||
provider: provider
|
||||
.filter(|provider| !provider.is_empty())
|
||||
.or_else(|| litellm_provider(direct)),
|
||||
.map(Cow::Borrowed)
|
||||
.or_else(|| litellm_provider(&direct)),
|
||||
model_info: Some(direct),
|
||||
};
|
||||
}
|
||||
let by_prefix = model
|
||||
|
|
@ -73,12 +79,12 @@ pub fn resolve_model_info<'a>(
|
|||
.flatten();
|
||||
match by_prefix {
|
||||
Some(by_prefix) => ResolvedModelInfo {
|
||||
provider: litellm_provider(&by_prefix),
|
||||
model_info: Some(by_prefix),
|
||||
provider: litellm_provider(by_prefix),
|
||||
},
|
||||
None => ResolvedModelInfo {
|
||||
model_info: None,
|
||||
provider,
|
||||
provider: provider.map(Cow::Borrowed),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -158,24 +164,27 @@ fn usage_with_anthropic_web_search<'a>(
|
|||
Some(Cow::Owned(ChatUsage { extra, ..base }))
|
||||
}
|
||||
|
||||
fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 {
|
||||
fn maps_cost(usage: Option<&ChatUsage>, resolved: &ResolvedModelInfo<'_>) -> f64 {
|
||||
let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some())
|
||||
else {
|
||||
return 0.0;
|
||||
};
|
||||
match resolved {
|
||||
ResolvedModelInfo {
|
||||
model_info: Some(model_info),
|
||||
provider: Some(provider),
|
||||
} => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0),
|
||||
match (resolved.model_info.as_deref(), resolved.provider.as_deref()) {
|
||||
(Some(model_info), Some(provider)) => {
|
||||
get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0)
|
||||
}
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 {
|
||||
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: &ResolvedModelInfo<'_>) -> f64 {
|
||||
let usage =
|
||||
usage_with_anthropic_web_search(request.usage, request.response, request.response_kind);
|
||||
let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) {
|
||||
let routed = match (
|
||||
resolved.model_info.as_deref(),
|
||||
usage.as_deref(),
|
||||
resolved.provider.as_deref(),
|
||||
) {
|
||||
(Some(model_info), Some(usage), Some(provider)) => {
|
||||
get_cost_for_web_search_request(provider, usage, model_info, request.defaults)
|
||||
}
|
||||
|
|
@ -184,7 +193,7 @@ fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelI
|
|||
routed.unwrap_or_else(|| {
|
||||
get_cost_for_web_search(
|
||||
request.params.get("web_search_options"),
|
||||
resolved.model_info,
|
||||
resolved.model_info.as_deref(),
|
||||
) * count_web_search_calls(request.response, request.response_kind) as f64
|
||||
})
|
||||
}
|
||||
|
|
@ -243,17 +252,17 @@ pub fn get_cost_for_built_in_tools(
|
|||
request: BuiltInToolCostRequest<'_>,
|
||||
) -> f64 {
|
||||
let resolved = resolve_model_info(catalog, model, request.provider, region);
|
||||
let maps = maps_cost(request.usage, resolved);
|
||||
let maps = maps_cost(request.usage, &resolved);
|
||||
let web_search =
|
||||
response_object_includes_web_search_call(request.response, request.response_kind, None)
|
||||
|| (request.response_kind != ResponseKind::Responses
|
||||
&& request.usage.is_some_and(usage_reports_web_search));
|
||||
if web_search {
|
||||
return maps + web_search_cost(request, resolved);
|
||||
return maps + web_search_cost(request, &resolved);
|
||||
}
|
||||
let direct = catalog.entry(model, request.provider, region);
|
||||
if response_object_includes_file_search_call(request.response, request.response_kind) {
|
||||
return maps + file_search_cost(request, direct);
|
||||
return maps + file_search_cost(request, direct.as_deref());
|
||||
}
|
||||
maps + azure_assistant_cost(request, direct)
|
||||
maps + azure_assistant_cost(request, direct.as_deref())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -142,7 +142,7 @@ pub fn vertex_cost(
|
|||
) -> Result<(f64, f64), CostError> {
|
||||
if let Some(cost) = catalog
|
||||
.entry(request.model, request.provider, request.region)
|
||||
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
||||
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
|
||||
{
|
||||
return Ok(cost);
|
||||
}
|
||||
|
|
@ -164,7 +164,7 @@ pub fn vertex_cost(
|
|||
Ok(cost_per_character(
|
||||
model_without_prefix,
|
||||
request.usage,
|
||||
entry,
|
||||
&entry,
|
||||
(prompt_characters, completion_characters),
|
||||
request.service_tier,
|
||||
request.vertex_location,
|
||||
|
|
|
|||
|
|
@ -103,12 +103,12 @@ pub fn zero_cost_finding(
|
|||
return None;
|
||||
}
|
||||
let usage = get_usage_object(response).ok().flatten()?;
|
||||
let (pricing_model, pricing_entry) =
|
||||
let pricing =
|
||||
catalog.pricing_entry_for_cost_calc(request.model_selection, request.logging_details)?;
|
||||
let diagnostic = diagnose_zero_cost(
|
||||
&usage,
|
||||
pricing_model,
|
||||
pricing_entry,
|
||||
&pricing.key,
|
||||
&pricing.info,
|
||||
request.calculation_failed,
|
||||
)?;
|
||||
let model = request.model_selection.model.unwrap_or("None");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
// mirrors: test_litellm/llms/bedrock/test_bedrock_common_utils.py::test_get_bedrock_base_model
|
||||
// expected values recorded from litellm/llms/bedrock/common_utils.py
|
||||
|
||||
use litellm_cost::bedrock_common_utils::{get_bedrock_base_model, strip_bedrock_routing_prefix};
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
#[case(
|
||||
"bedrock/converse/us.anthropic.claude-x-v1:0",
|
||||
"us.anthropic.claude-x-v1:0"
|
||||
)]
|
||||
#[case("converse/bedrock/m", "bedrock/m")]
|
||||
#[case("openai/bedrock/m", "bedrock/m")]
|
||||
#[case("nova-2/arn:aws:x", "arn:aws:x")]
|
||||
#[case("bedrock/nova/arn:aws:y", "arn:aws:y")]
|
||||
#[case("invoke/mantle/nova/m", "m")]
|
||||
#[case("m", "m")]
|
||||
fn strip_bedrock_routing_prefix_strips_each_prefix_once_in_order(
|
||||
#[case] model: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(strip_bedrock_routing_prefix(model), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("us.meta.llama3-v1:0", "meta.llama3-v1:0")]
|
||||
#[case("bedrock/converse/model", "model")]
|
||||
#[case("bedrock/nova-2/arn:aws:bedrock:x", "amazon.nova-2-custom")]
|
||||
#[case("bedrock/nova/arn:aws:x", "amazon.nova-custom")]
|
||||
#[case("converse/nova-2/x", "amazon.nova-2-custom")]
|
||||
#[case("invoke/nova/x", "x")]
|
||||
#[case("anthropic.claude-v2:0:51k", "anthropic.claude-v2:0")]
|
||||
#[case("us.anthropic.claude-opus-v1[1m]", "anthropic.claude-opus-v1")]
|
||||
#[case(
|
||||
"arn:aws:bedrock:us-east-1:1:foundation-model/anthropic.claude-v2",
|
||||
"anthropic.claude-v2"
|
||||
)]
|
||||
#[case("us-east-1/amazon.titan", "amazon.titan")]
|
||||
#[case("ap-south-9/amazon.titan", "ap-south-9/amazon.titan")]
|
||||
#[case("global.x", "x")]
|
||||
#[case("mars.x", "mars.x")]
|
||||
fn get_bedrock_base_model_matches_python(#[case] model: &str, #[case] expected: &str) {
|
||||
assert_eq!(get_bedrock_base_model(model), expected);
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: image_gen_tests/test_bedrock_image_gen_unit_tests.py::test_cost_calculator_stability1
|
||||
// expected values recorded from litellm/llms/bedrock/image_generation/cost_calculator.py::cost_calculator
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
|
@ -10,6 +11,7 @@ use litellm_cost::bedrock_image_cost::{
|
|||
};
|
||||
use litellm_cost::call_type::CallTypes;
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
use litellm_cost::error::CostError;
|
||||
use litellm_cost::image_cost_router::{
|
||||
ImageCostRouteRequest, route_image_generation_cost_calculator,
|
||||
};
|
||||
|
|
@ -40,52 +42,75 @@ fn get_config_class_matches_model_family(
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(50, "50-steps")]
|
||||
#[case(51, "max-steps")]
|
||||
fn stability1_pricing_key_selects_step_tier(#[case] steps: u64, #[case] tier: &str) {
|
||||
let key = stability1_pricing_key(
|
||||
"stability.stable-diffusion-xl-v1",
|
||||
Some("1024-x-1024"),
|
||||
&json!({"steps": steps}),
|
||||
);
|
||||
#[case::at_the_threshold(json!({"steps": 50}), "50-steps")]
|
||||
#[case::above_the_threshold(json!({"steps": 51}), "max-steps")]
|
||||
#[case::float_steps(json!({"steps": 50.5}), "max-steps")]
|
||||
#[case::bool_steps_compare_as_int(json!({"steps": true}), "50-steps")]
|
||||
#[case::default_steps(json!({}), "50-steps")]
|
||||
fn stability1_pricing_key_selects_step_tier(#[case] params: Value, #[case] tier: &str) {
|
||||
assert_eq!(
|
||||
key,
|
||||
format!("1024-x-1024/{tier}/stability.stable-diffusion-xl-v1")
|
||||
stability1_pricing_key("stability.sd-zz-v1", Some("1024-x-1024"), ¶ms),
|
||||
Ok(format!("1024-x-1024/{tier}/stability.sd-zz-v1"))
|
||||
);
|
||||
}
|
||||
|
||||
fn bedrock_image_catalog() -> ModelInfoCatalog {
|
||||
let entry = |rate: f64| json!({"output_cost_per_image": rate, "litellm_provider": "bedrock", "mode": "image_generation"});
|
||||
ModelInfoCatalog::new(HashMap::from([
|
||||
(
|
||||
"1024-x-1024/50-steps/stability.sd-zz-v1".to_owned(),
|
||||
entry(0.04),
|
||||
),
|
||||
(
|
||||
"1024-x-1024/max-steps/stability.sd-zz-v1".to_owned(),
|
||||
entry(0.08),
|
||||
),
|
||||
("amazon.nova-canvas-zz".to_owned(), entry(0.03)),
|
||||
("amazon.titan-image-zz".to_owned(), entry(0.02)),
|
||||
("stability.sd3-zz".to_owned(), entry(0.05)),
|
||||
]))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn bedrock_stability1_cost_uses_size_and_steps_lookup() {
|
||||
let entries = HashMap::from([
|
||||
(
|
||||
"1024-x-1024/50-steps/stability.stable-diffusion-xl-v1".to_owned(),
|
||||
json!({"output_cost_per_image": 0.04}),
|
||||
),
|
||||
(
|
||||
"1024-x-1024/max-steps/stability.stable-diffusion-xl-v1".to_owned(),
|
||||
json!({"output_cost_per_image": 0.08}),
|
||||
),
|
||||
]);
|
||||
let response = json!({"data": [{}, {}]});
|
||||
#[case::stability1_default_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 50}), Ok(0.08))]
|
||||
#[case::stability1_max_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 60}), Ok(0.16))]
|
||||
#[case::stability1_missing_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({}), Ok(0.08))]
|
||||
#[case::stability1_default_size("stability.sd-zz-v1", None, json!({"steps": true}), Ok(0.08))]
|
||||
#[case::stability1_null_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": null}), Err(CostError::InvalidSteps))]
|
||||
#[case::stability1_string_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": "60"}), Err(CostError::InvalidSteps))]
|
||||
#[case::stability1_keeps_the_routing_prefix_in_the_key(
|
||||
"bedrock/stability.sd-zz-v1",
|
||||
Some("1024-x-1024"),
|
||||
json!({"steps": 50}),
|
||||
Err(CostError::ModelNotFound)
|
||||
)]
|
||||
#[case::stability1_key_is_case_insensitive("STABILITY.SD-ZZ-V1", Some("1024-x-1024"), json!({}), Ok(0.08))]
|
||||
#[case::nova_bare("amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
|
||||
#[case::nova_routing_prefix("bedrock/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
|
||||
#[case::nova_cross_region_prefix("us.amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
|
||||
#[case::nova_region_path("bedrock/us-east-1/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
|
||||
#[case::nova_invoke_route("invoke/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
|
||||
#[case::nova_unmapped("amazon.nova-canvas-missing", None, json!({}), Err(CostError::ModelNotFound))]
|
||||
#[case::titan_bare("amazon.titan-image-zz", None, json!({}), Ok(0.04))]
|
||||
#[case::titan_routing_prefix("bedrock/amazon.titan-image-zz", None, json!({}), Ok(0.04))]
|
||||
#[case::titan_infers_no_provider_for_a_cross_region_id(
|
||||
"us.amazon.titan-image-zz",
|
||||
None,
|
||||
json!({}),
|
||||
Err(CostError::ModelNotFound)
|
||||
)]
|
||||
#[case::stability3_bare("stability.sd3-zz", None, json!({}), Ok(0.1))]
|
||||
#[case::stability3_routing_prefix("bedrock/stability.sd3-zz", None, json!({}), Ok(0.1))]
|
||||
fn bedrock_image_cost_resolves_models_like_get_model_info(
|
||||
#[case] model: &str,
|
||||
#[case] size: Option<&str>,
|
||||
#[case] params: Value,
|
||||
#[case] expected: Result<f64, CostError>,
|
||||
) {
|
||||
let response = json!({"data": [{"url": "x"}, {"url": "y"}]});
|
||||
assert_eq!(
|
||||
cost_calculator(
|
||||
"bedrock/stability.stable-diffusion-xl-v1",
|
||||
&response,
|
||||
Some("1024-x-1024"),
|
||||
&json!({"steps": 50}),
|
||||
&entries
|
||||
),
|
||||
Some(0.08)
|
||||
);
|
||||
assert_eq!(
|
||||
cost_calculator(
|
||||
"bedrock/stability.stable-diffusion-xl-v1",
|
||||
&response,
|
||||
Some("1024-x-1024"),
|
||||
&json!({"steps": 60}),
|
||||
&entries
|
||||
),
|
||||
Some(0.16)
|
||||
cost_calculator(&bedrock_image_catalog(), model, &response, size, ¶ms),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -95,8 +120,8 @@ fn bedrock_stability1_cost_uses_size_and_steps_lookup() {
|
|||
#[case("stability.sd3-large-v1:0", 0.04)]
|
||||
fn bedrock_direct_families_bill_returned_images(#[case] model: &str, #[case] rate: f64) {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
format!("bedrock/{model}"),
|
||||
json!({"output_cost_per_image": rate}),
|
||||
model.to_owned(),
|
||||
json!({"output_cost_per_image": rate, "litellm_provider": "bedrock"}),
|
||||
)]));
|
||||
let response = json!({"data": [{}, {}, {}]});
|
||||
let supplied = json!({"output_cost_per_image": 1.0});
|
||||
|
|
|
|||
|
|
@ -15,8 +15,6 @@ use litellm_cost::model_selection::ModelSelectionRequest;
|
|||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const PROVIDERS: &[&str] = &["anthropic", "vertex_ai"];
|
||||
|
||||
#[rstest]
|
||||
#[case("ON_DEMAND_PRIORITY", Some("priority"))]
|
||||
#[case("on_demand_flex", Some("flex"))]
|
||||
|
|
@ -108,7 +106,6 @@ fn prepared_input_prices_served_priority_rate() {
|
|||
provider: Some("anthropic"),
|
||||
router_model_id: None,
|
||||
region_name: None,
|
||||
known_providers: PROVIDERS,
|
||||
},
|
||||
call_type: None,
|
||||
response_kind: Some(ResponseKind::Completion),
|
||||
|
|
|
|||
|
|
@ -19,17 +19,6 @@ use litellm_token_counter::{Error as TokenCounterError, TokenCounter, Tokenizer}
|
|||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const PROVIDERS: &[&str] = &[
|
||||
"anthropic",
|
||||
"azure_ai",
|
||||
"exa_ai",
|
||||
"openai",
|
||||
"recraft",
|
||||
"replicate",
|
||||
"vertex_ai",
|
||||
"xai",
|
||||
];
|
||||
|
||||
struct CharacterTokenizer;
|
||||
|
||||
impl Tokenizer for CharacterTokenizer {
|
||||
|
|
@ -56,7 +45,6 @@ fn request<'a>(
|
|||
provider,
|
||||
router_model_id: None,
|
||||
region_name: None,
|
||||
known_providers: PROVIDERS,
|
||||
},
|
||||
call_type: None,
|
||||
response_kind: Some(ResponseKind::Completion),
|
||||
|
|
@ -165,12 +153,13 @@ fn response_cost_calculator_prices_response_without_provider_override() {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("xai/served", json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))]
|
||||
#[case("served", json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))]
|
||||
fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[case] info: Value) {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(model.to_owned(), info)]));
|
||||
fn response_cost_infers_provider_for_specialized_pricing() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"xai/served".to_owned(),
|
||||
json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}),
|
||||
)]));
|
||||
let response = json!({
|
||||
"model": model,
|
||||
"model": "xai/served",
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 10,
|
||||
|
|
@ -181,12 +170,33 @@ fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[
|
|||
let empty = json!({});
|
||||
let result = completion_cost_from_response(
|
||||
&catalog,
|
||||
request(Some(&response), Some(model), None, &empty, &empty),
|
||||
request(Some(&response), Some("xai/served"), None, &empty, &empty),
|
||||
)
|
||||
.unwrap();
|
||||
assert!((result.cost.total - (100.0 * 0.01 + 30.0 * 0.02)).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn response_cost_rejects_a_bare_model_get_llm_provider_cannot_route() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"served".to_owned(),
|
||||
json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}),
|
||||
)]));
|
||||
let response = json!({
|
||||
"model": "served",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}
|
||||
});
|
||||
let empty = json!({});
|
||||
assert_eq!(
|
||||
completion_cost_from_response(
|
||||
&catalog,
|
||||
request(Some(&response), Some("served"), None, &empty, &empty),
|
||||
)
|
||||
.map(|result| result.cost.total),
|
||||
Err(CostError::MissingProvider)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn response_cost_infers_provider_from_priced_fallback_model() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ fn flat_cost_per_image_rounds_each_response_to_megapixels() {
|
|||
|
||||
#[rstest]
|
||||
fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() {
|
||||
let entries = HashMap::from([
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([
|
||||
(
|
||||
"fal_ai/high/1024-x-768/openai/model".to_owned(),
|
||||
json!({"output_cost_per_image": 0.10}),
|
||||
|
|
@ -107,42 +107,42 @@ fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() {
|
|||
"fal_ai/openai/model".to_owned(),
|
||||
json!({"output_cost_per_image": 0.50}),
|
||||
),
|
||||
]);
|
||||
]));
|
||||
let response = json!({"data": [{}, {}]});
|
||||
let supplied = json!({"output_cost_per_image": 0.25});
|
||||
assert_eq!(
|
||||
cost_calculator(
|
||||
&catalog,
|
||||
"fal_ai/openai/model",
|
||||
&response,
|
||||
&json!({}),
|
||||
Some(&supplied),
|
||||
&entries
|
||||
),
|
||||
Some(0.50)
|
||||
);
|
||||
assert_eq!(
|
||||
cost_calculator("fal_ai/openai/model", &response, &json!({}), None, &entries),
|
||||
cost_calculator(&catalog, "fal_ai/openai/model", &response, &json!({}), None),
|
||||
Some(0.20)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn fal_image_cost_falls_back_to_base_megapixel_and_flat_rates_per_image() {
|
||||
let entries = HashMap::from([(
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"fal_ai/fal-ai/flux/dev".to_owned(),
|
||||
json!({"output_cost_per_image": 0.5, "output_cost_per_pixel": 0.0001}),
|
||||
)]);
|
||||
)]));
|
||||
let response = json!({"data": [
|
||||
image(json!(1024), json!(1024)),
|
||||
image(json!(1920), json!(1080)),
|
||||
image(json!(true), json!(1024))
|
||||
]});
|
||||
let cost = cost_calculator(
|
||||
&catalog,
|
||||
"fal_ai/fal-ai/flux/dev",
|
||||
&response,
|
||||
&json!({}),
|
||||
None,
|
||||
&entries,
|
||||
)
|
||||
.unwrap();
|
||||
let expected = 0.0001 * PIXELS_PER_MEGAPIXEL as f64 * 3.0 + 0.5;
|
||||
|
|
@ -210,3 +210,33 @@ fn fal_passthrough_selects_resolution_rate_and_falls_back_to_base() {
|
|||
None
|
||||
);
|
||||
}
|
||||
|
||||
// expected values recorded from litellm/llms/fal_ai/cost_calculator.py::cost_calculator
|
||||
#[rstest]
|
||||
#[case::bare_key_owned_by_fal("fal_ai/flux-zz", Some(0.4))]
|
||||
#[case::case_insensitive_key("FLUX-ZZ", Some(0.4))]
|
||||
#[case::bool_rate_counts_as_int("fal_ai/flag-zz", Some(2.0))]
|
||||
#[case::string_rate_is_not_a_number("fal_ai/str-zz", Some(0.0))]
|
||||
#[case::unmapped_model("fal_ai/missing", None)]
|
||||
fn fal_image_flat_fallback_resolves_through_get_model_info(
|
||||
#[case] model: &str,
|
||||
#[case] expected: Option<f64>,
|
||||
) {
|
||||
let entry = |rate: Value| json!({"output_cost_per_image": rate, "litellm_provider": "fal_ai"});
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([
|
||||
("flux-zz".to_owned(), entry(json!(0.2))),
|
||||
("fal_ai/flag-zz".to_owned(), entry(json!(true))),
|
||||
("fal_ai/str-zz".to_owned(), entry(json!("0.3"))),
|
||||
]));
|
||||
let response = json!({"data": [{"url": "x"}, {"url": "y"}]});
|
||||
assert_eq!(
|
||||
cost_calculator(
|
||||
&catalog,
|
||||
model,
|
||||
&response,
|
||||
&json!({"image_size": "weird"}),
|
||||
None
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
|
|||
262
litellm-rust/crates/cost/tests/python_model_info.rs
Normal file
262
litellm-rust/crates/cost/tests/python_model_info.rs
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
// mirrors: test_litellm/test_utils.py::test_get_model_info_* (get_model_info key resolution)
|
||||
// expected values recorded from litellm/utils.py::_get_model_info_helper and
|
||||
// litellm/litellm_core_utils/get_llm_provider_logic.py::get_llm_provider on this catalog
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::catalog::ModelInfoCatalog;
|
||||
use litellm_cost::error::CostError;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn catalog() -> ModelInfoCatalog {
|
||||
let provider = |name: &str| json!({"litellm_provider": name});
|
||||
ModelInfoCatalog::new(HashMap::from([
|
||||
("openai/gpt-zz".to_owned(), provider("openai")),
|
||||
("gpt-zz".to_owned(), provider("openai")),
|
||||
("gemini/gemini-zz".to_owned(), provider("gemini")),
|
||||
(
|
||||
"gemini-zz-pro".to_owned(),
|
||||
provider("vertex_ai-language-models"),
|
||||
),
|
||||
("claude-zz".to_owned(), provider("anthropic")),
|
||||
("ft:gpt-yy".to_owned(), provider("openai")),
|
||||
(
|
||||
"vertex_ai/meta/llama-zz".to_owned(),
|
||||
provider("vertex_ai-llama_models"),
|
||||
),
|
||||
(
|
||||
"vertex_ai/mistral-zz@latest".to_owned(),
|
||||
provider("vertex_ai-mistral_models"),
|
||||
),
|
||||
(
|
||||
"anthropic.claude-zz-v1".to_owned(),
|
||||
provider("bedrock_converse"),
|
||||
),
|
||||
(
|
||||
"bedrock_mantle/model-zz".to_owned(),
|
||||
provider("bedrock_mantle"),
|
||||
),
|
||||
(
|
||||
"fireworks_ai/accounts/fireworks/models/fw-zz".to_owned(),
|
||||
provider("fireworks_ai"),
|
||||
),
|
||||
("gpt-4.1".to_owned(), provider("openai")),
|
||||
("Mixed-Case-ZZ".to_owned(), provider("together_ai")),
|
||||
(
|
||||
"fallback_generalizations".to_owned(),
|
||||
json!({"rules": [
|
||||
{"name": "zz-route", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}},
|
||||
{"name": "zz-cap", "pattern": "zzcap-\\d+", "model_info": {"mode": "chat", "max_tokens": 10}},
|
||||
{"name": "zz-cap2", "pattern": "zzcap-9", "model_info": {"max_tokens": 20}},
|
||||
{
|
||||
"name": "zz-legacy",
|
||||
"pattern": "^legacy-zz",
|
||||
"extends": "zz-cap",
|
||||
"model_info": {"litellm_provider": "xai", "max_tokens": 30}
|
||||
}
|
||||
]}),
|
||||
),
|
||||
]))
|
||||
}
|
||||
|
||||
type Resolved = Result<(&'static str, Option<&'static str>, Option<u64>), CostError>;
|
||||
|
||||
#[rstest]
|
||||
#[case::provider_prefixed_key_first("gpt-zz", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
|
||||
#[case::bare_model_without_provider("gpt-zz", None, Ok(("gpt-zz", Some("openai"), None)))]
|
||||
#[case::prefixed_model_without_provider("openai/gpt-zz", None, Ok(("openai/gpt-zz", Some("openai"), None)))]
|
||||
#[case::case_insensitive_key("GPT-ZZ", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
|
||||
#[case::dated_snapshot_suffix("gpt-zz-2025-01-01", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
|
||||
#[case::dated_snapshot_without_provider("gpt-zz-2025-01-01", None, Ok(("gpt-zz", Some("openai"), None)))]
|
||||
#[case::stable_version_suffix("gemini-zz-001", Some("gemini"), Ok(("gemini/gemini-zz", Some("gemini"), None)))]
|
||||
#[case::vertex_family_entry(
|
||||
"gemini-zz-pro",
|
||||
Some("vertex_ai"),
|
||||
Ok(("gemini-zz-pro", Some("vertex_ai-language-models"), None))
|
||||
)]
|
||||
#[case::mismatched_provider_is_skipped(
|
||||
"gemini/gemini-zz",
|
||||
Some("vertex_ai"),
|
||||
Err(CostError::ModelNotFound)
|
||||
)]
|
||||
#[case::openai_finetune("ft:gpt-yy:org:suffix:id", Some("openai"), Ok(("ft:gpt-yy", Some("openai"), None)))]
|
||||
#[case::vertex_llama_alias(
|
||||
"llama-zz",
|
||||
Some("vertex_ai"),
|
||||
Ok(("vertex_ai/meta/llama-zz", Some("vertex_ai-llama_models"), None))
|
||||
)]
|
||||
#[case::vertex_beta_latest_alias(
|
||||
"mistral-zz",
|
||||
Some("vertex_ai_beta"),
|
||||
Ok(("vertex_ai/mistral-zz@latest", Some("vertex_ai-mistral_models"), None))
|
||||
)]
|
||||
#[case::bedrock_cross_region_keeps_the_version(
|
||||
"us.anthropic.claude-zz-v1:0",
|
||||
Some("bedrock"),
|
||||
Err(CostError::ModelNotFound)
|
||||
)]
|
||||
#[case::bedrock_routing_prefix(
|
||||
"bedrock/converse/anthropic.claude-zz-v1",
|
||||
Some("bedrock"),
|
||||
Ok(("anthropic.claude-zz-v1", Some("bedrock_converse"), None))
|
||||
)]
|
||||
#[case::bedrock_throughput_suffix_keeps_the_version(
|
||||
"anthropic.claude-zz-v1:0:51k",
|
||||
Some("bedrock_converse"),
|
||||
Err(CostError::ModelNotFound)
|
||||
)]
|
||||
#[case::mantle_region_prefix(
|
||||
"bedrock_mantle/us-east-1/model-zz",
|
||||
Some("bedrock_mantle"),
|
||||
Ok(("bedrock_mantle/model-zz", Some("bedrock_mantle"), None))
|
||||
)]
|
||||
#[case::fireworks_resource_key(
|
||||
"fw-zz",
|
||||
Some("fireworks_ai"),
|
||||
Ok(("fireworks_ai/accounts/fireworks/models/fw-zz", Some("fireworks_ai"), None))
|
||||
)]
|
||||
#[case::azure_alias("azure/gpt-41", None, Ok(("gpt-4.1", Some("openai"), None)))]
|
||||
#[case::lowercase_query_finds_mixed_case_key(
|
||||
"mixed-case-zz",
|
||||
Some("together_ai"),
|
||||
Ok(("Mixed-Case-ZZ", Some("together_ai"), None))
|
||||
)]
|
||||
#[case::huggingface_is_zero_rated("any-model", Some("huggingface"), Ok(("any-model", Some("huggingface"), None)))]
|
||||
#[case::other_provider_entry("claude-zz", Some("openai"), Err(CostError::ModelNotFound))]
|
||||
#[case::capability_union_later_rule_wins("zzcap-9", None, Ok(("zzcap-9", None, Some(20))))]
|
||||
#[case::capability_backfills_the_provider("zzcap-9", Some("openai"), Ok(("openai/zzcap-9", Some("openai"), Some(20))))]
|
||||
#[case::routing_rule_alone_has_no_capabilities("zzroute-x", None, Err(CostError::ModelNotFound))]
|
||||
#[case::legacy_rule_routes_and_extends("legacy-zz-1", None, Ok(("legacy-zz-1", Some("xai"), Some(30))))]
|
||||
#[case::unmapped("nothing-matches", None, Err(CostError::ModelNotFound))]
|
||||
fn get_model_info_resolves_keys_like_python(
|
||||
#[case] model: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] expected: Resolved,
|
||||
) {
|
||||
let catalog = catalog();
|
||||
let actual = catalog.get_model_info(model, provider).map(|resolved| {
|
||||
(
|
||||
resolved.key.into_owned(),
|
||||
resolved
|
||||
.info
|
||||
.get("litellm_provider")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned),
|
||||
resolved.info.get("max_tokens").and_then(Value::as_u64),
|
||||
)
|
||||
});
|
||||
assert_eq!(
|
||||
actual,
|
||||
expected.map(|(key, provider, max_tokens)| {
|
||||
(key.to_owned(), provider.map(str::to_owned), max_tokens)
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn get_model_info_prices_huggingface_at_zero() {
|
||||
let catalog = catalog();
|
||||
let info = catalog
|
||||
.get_model_info("any-model", Some("huggingface"))
|
||||
.unwrap()
|
||||
.info;
|
||||
assert_eq!(info.get("input_cost_per_token"), Some(&json!(0)));
|
||||
assert_eq!(info.get("output_cost_per_token"), Some(&json!(0)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn capability_rules_lose_to_an_exact_key_owned_by_another_provider() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([
|
||||
("zzcap-1".to_owned(), json!({"litellm_provider": "gemini"})),
|
||||
(
|
||||
"fallback_generalizations".to_owned(),
|
||||
json!({"rules": [{"name": "cap", "pattern": "zzcap", "model_info": {"mode": "chat"}}]}),
|
||||
),
|
||||
]));
|
||||
assert_eq!(
|
||||
catalog
|
||||
.get_model_info("zzcap-1", Some("openai"))
|
||||
.map(|info| info.key.into_owned()),
|
||||
Err(CostError::ModelNotFound)
|
||||
);
|
||||
assert_eq!(
|
||||
catalog
|
||||
.get_model_info("zzcap-2", Some("openai"))
|
||||
.map(|info| info.key.into_owned()),
|
||||
Ok("openai/zzcap-2".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cataloged_bare("gpt-zz", Some("openai"))]
|
||||
#[case::provider_prefix("gemini/anything", Some("gemini"))]
|
||||
#[case::vertex_family("gemini-zz-pro", Some("vertex_ai"))]
|
||||
#[case::bedrock_converse_family("anthropic.claude-zz-v1", Some("bedrock"))]
|
||||
#[case::routing_rule("zzroute-x", Some("anthropic"))]
|
||||
#[case::legacy_routing_rule("legacy-zz-1", Some("xai"))]
|
||||
#[case::azure_prefix_stays_azure("azure/gpt-41", Some("azure"))]
|
||||
#[case::json_registry_provider("gmi/any", Some("gmi"))]
|
||||
#[case::static_cohere_embedding("embed-v4.0", Some("cohere"))]
|
||||
#[case::static_openai_image("dall-e-3", Some("openai"))]
|
||||
#[case::unroutable("GPT-ZZ", None)]
|
||||
fn get_llm_provider_infers_like_python(#[case] model: &str, #[case] expected: Option<&str>) {
|
||||
assert_eq!(
|
||||
catalog()
|
||||
.get_llm_provider(model)
|
||||
.map(|resolved| resolved.custom_llm_provider)
|
||||
.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
fn usage() -> litellm_cost::responses_usage::ChatUsage {
|
||||
litellm_cost::usage_dispatch::get_usage_object(
|
||||
&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}}),
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// expected values recorded from litellm.completion_cost for unmapped ids the shipped rules generalize
|
||||
#[rstest]
|
||||
#[case::routed_and_generalized("claude-newfam-9", None, Ok((0.0, 0.0)))]
|
||||
#[case::generalized_under_explicit_provider("claude-newfam-9", Some("anthropic"), Ok((0.0, 0.0)))]
|
||||
#[case::generalized_bedrock_id("anthropic.claude-newfam-9", Some("bedrock"), Ok((0.0, 0.0)))]
|
||||
#[case::unroutable("totally-unknown", None, Err(CostError::MissingProvider))]
|
||||
#[case::routable_but_unmapped("zzroute-x", None, Err(CostError::ModelNotFound))]
|
||||
fn unmapped_models_price_through_capability_rules(
|
||||
#[case] model: &str,
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] expected: Result<(f64, f64), CostError>,
|
||||
) {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"fallback_generalizations".to_owned(),
|
||||
json!({"rules": [
|
||||
{"name": "route-bedrock", "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-", "model_info": {"litellm_provider": "bedrock"}},
|
||||
{"name": "route", "pattern": "^claude-[a-z]+-\\d+$", "model_info": {"litellm_provider": "anthropic"}},
|
||||
{"name": "route-only", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}},
|
||||
{"name": "baseline", "pattern": "claude-[a-z]+-\\d+", "model_info": {"mode": "chat"}}
|
||||
]}),
|
||||
)]));
|
||||
let usage = usage();
|
||||
assert_eq!(
|
||||
litellm_cost::cost_calculator::cost_per_token(
|
||||
&catalog,
|
||||
litellm_cost::catalog::ModelCostRequest {
|
||||
model,
|
||||
provider,
|
||||
region: None,
|
||||
usage: &usage,
|
||||
service_tier: None,
|
||||
data_residency: None,
|
||||
vertex_location: None,
|
||||
at: "2026-01-01T12:00Z".parse().unwrap(),
|
||||
response_time_ms: None,
|
||||
},
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
@ -7,7 +7,8 @@ use litellm_cost::error::CostError;
|
|||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match};
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::model_info::check_provider_match;
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
@ -84,7 +85,10 @@ fn cost_per_token_resolves_model_key_in_python_order(
|
|||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
catalog().select_model_key(model, provider, region),
|
||||
catalog()
|
||||
.select_model_info(model, provider, region)
|
||||
.map(|selected| selected.key.into_owned())
|
||||
.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
@ -221,7 +225,11 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
|
|||
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
|
||||
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
|
||||
#[case::fireworks_request_rejects_other_providers(Some("openai"), Some("fireworks_ai"), false)]
|
||||
#[case::fireworks_entry_needs_a_fireworks_request(Some("fireworks_ai-embedding-models"), Some("together_ai"), false)]
|
||||
#[case::fireworks_entry_needs_a_fireworks_request(
|
||||
Some("fireworks_ai-embedding-models"),
|
||||
Some("together_ai"),
|
||||
false
|
||||
)]
|
||||
#[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)]
|
||||
#[case::bedrock_request_rejects_other_providers(Some("openai"), Some("bedrock"), false)]
|
||||
#[case::bedrock_entry_needs_a_bedrock_request(Some("bedrock"), Some("sagemaker"), false)]
|
||||
|
|
@ -252,7 +260,10 @@ fn select_model_key_skips_entries_owned_by_another_provider(
|
|||
json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}),
|
||||
)]));
|
||||
assert_eq!(
|
||||
catalog.select_model_key("gemini/gemini-x", provider, None),
|
||||
catalog
|
||||
.select_model_info("gemini/gemini-x", provider, None)
|
||||
.map(|selected| selected.key.into_owned())
|
||||
.as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
@ -264,7 +275,10 @@ fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() {
|
|||
json!({"litellm_provider": "vertex_ai-language-models"}),
|
||||
)]));
|
||||
assert_eq!(
|
||||
catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None),
|
||||
catalog
|
||||
.select_model_info("vertex_ai/gemini-x", Some("vertex_ai_beta"), None)
|
||||
.map(|selected| selected.key.into_owned())
|
||||
.as_deref(),
|
||||
Some("vertex_ai/gemini-x")
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,15 @@ use litellm_cost::model_selection::{
|
|||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const PROVIDERS: &[&str] = &["openai", "bedrock", "vertex_ai", "anthropic", "dashscope"];
|
||||
fn pricing<'a>(
|
||||
catalog: &'a ModelInfoCatalog,
|
||||
request: ModelSelectionRequest<'a>,
|
||||
logging_details: Option<&'a Value>,
|
||||
) -> Option<(String, Value)> {
|
||||
catalog
|
||||
.pricing_entry_for_cost_calc(request, logging_details)
|
||||
.map(|selected| (selected.key.into_owned(), selected.info.into_owned()))
|
||||
}
|
||||
|
||||
fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelectionRequest<'a> {
|
||||
ModelSelectionRequest {
|
||||
|
|
@ -24,7 +32,6 @@ fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelect
|
|||
provider,
|
||||
router_model_id: None,
|
||||
region_name: None,
|
||||
known_providers: PROVIDERS,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -166,31 +173,64 @@ fn model_selection_helpers_ignore_invalid_hidden_values_and_stop_at_provider_seg
|
|||
get_hidden_str_for_cost_calc(Some(&json!({"model": ""})), "model"),
|
||||
None
|
||||
);
|
||||
assert!(model_contains_known_llm_provider("openai/model", PROVIDERS));
|
||||
assert!(!model_contains_known_llm_provider("team/model", PROVIDERS));
|
||||
let catalog = HashMap::from([(
|
||||
assert!(model_contains_known_llm_provider("openai/model"));
|
||||
assert!(!model_contains_known_llm_provider("team/model"));
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"vertex_ai/claude".to_owned(),
|
||||
json!({"input_cost_per_token": 1e-6}),
|
||||
)]);
|
||||
)]));
|
||||
assert_eq!(
|
||||
strip_unregistered_leading_segments("vertex_ai/openai/claude", None, PROVIDERS, &catalog,),
|
||||
strip_unregistered_leading_segments("vertex_ai/openai/claude", None, &catalog),
|
||||
"vertex_ai/openai/claude"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(Some("xai/model"), None, Some("xai"))]
|
||||
#[case(Some("custom"), None, Some("xai"))]
|
||||
#[case(Some("xai/model"), Some("anthropic"), Some("anthropic"))]
|
||||
#[case(Some("unknown"), None, None)]
|
||||
fn provider_inference_uses_explicit_prefix_or_catalog_metadata(
|
||||
#[case::provider_prefix(Some("xai/model"), None, Some("xai"))]
|
||||
#[case::explicit_provider_wins(Some("xai/model"), Some("anthropic"), Some("anthropic"))]
|
||||
#[case::no_model(None, None, None)]
|
||||
#[case::unlisted_bare_provider_is_not_inferred(Some("custom-xai"), None, None)]
|
||||
#[case::unknown_bare_model(Some("unknown"), None, None)]
|
||||
#[case::bedrock_converse_routes_to_bedrock(Some("converse-model"), None, Some("bedrock"))]
|
||||
#[case::vertex_family_routes_to_vertex_ai(Some("vertex-model"), None, Some("vertex_ai"))]
|
||||
#[case::ai21_chat(Some("ai21-model"), None, Some("ai21_chat"))]
|
||||
#[case::ai21_prefix(Some("ai21/any"), None, Some("ai21_chat"))]
|
||||
#[case::cohere_chat_under_cohere_prefix(Some("cohere/chat-model"), None, Some("cohere_chat"))]
|
||||
#[case::azure_hosted_mistral(Some("azure/mistral-model"), None, Some("openai"))]
|
||||
#[case::anthropic_text(Some("anthropic/claude-2"), None, Some("anthropic_text"))]
|
||||
#[case::openai_finetune(Some("ft:gpt-4o:org::id"), None, Some("openai"))]
|
||||
#[case::replicate_version_id(Some(&*format!("owner/model:{}", "a".repeat(64))), None, Some("replicate"))]
|
||||
#[case::wildcard(Some("*"), None, Some("openai"))]
|
||||
fn provider_inference_follows_get_llm_provider(
|
||||
#[case] model: Option<&str>,
|
||||
#[case] explicit: Option<&str>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let catalog = HashMap::from([("custom".to_owned(), json!({"litellm_provider": "xai"}))]);
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([
|
||||
("custom-xai".to_owned(), json!({"litellm_provider": "xai"})),
|
||||
(
|
||||
"converse-model".to_owned(),
|
||||
json!({"litellm_provider": "bedrock_converse"}),
|
||||
),
|
||||
(
|
||||
"vertex-model".to_owned(),
|
||||
json!({"litellm_provider": "vertex_ai-language-models"}),
|
||||
),
|
||||
(
|
||||
"ai21-model".to_owned(),
|
||||
json!({"litellm_provider": "ai21", "mode": "chat"}),
|
||||
),
|
||||
(
|
||||
"chat-model".to_owned(),
|
||||
json!({"litellm_provider": "cohere_chat"}),
|
||||
),
|
||||
(
|
||||
"mistral/mistral-model".to_owned(),
|
||||
json!({"litellm_provider": "mistral"}),
|
||||
),
|
||||
]));
|
||||
assert_eq!(
|
||||
get_provider_for_cost_calc(model, explicit, &["xai", "anthropic"], &catalog).as_deref(),
|
||||
get_provider_for_cost_calc(model, explicit, &catalog).as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
@ -215,8 +255,11 @@ fn pricing_entry_prefers_registered_deployment_to_served_model() {
|
|||
..request(Some("requested"), Some("openai"))
|
||||
};
|
||||
assert_eq!(
|
||||
catalog.pricing_entry_for_cost_calc(request, None),
|
||||
Some(("router-id", &json!({"input_cost_per_token": 0.03})))
|
||||
pricing(&catalog, request, None),
|
||||
Some((
|
||||
"router-id".to_owned(),
|
||||
json!({"input_cost_per_token": 0.03})
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -237,18 +280,25 @@ fn pricing_entry_reads_deployment_metadata_before_published_price() {
|
|||
..request(Some("requested"), Some("openai"))
|
||||
};
|
||||
assert_eq!(
|
||||
catalog.pricing_entry_for_cost_calc(selected, Some(&logging)),
|
||||
Some(("requested", &json!({"input_cost_per_token": 0.02})))
|
||||
pricing(&catalog, selected, Some(&logging)),
|
||||
Some((
|
||||
"requested".to_owned(),
|
||||
json!({"input_cost_per_token": 0.02})
|
||||
))
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.pricing_entry_for_cost_calc(
|
||||
pricing(
|
||||
&catalog,
|
||||
ModelSelectionRequest {
|
||||
custom_pricing: false,
|
||||
..selected
|
||||
},
|
||||
Some(&logging)
|
||||
),
|
||||
Some(("openai/served", &json!({"input_cost_per_token": 0.01})))
|
||||
Some((
|
||||
"openai/served".to_owned(),
|
||||
json!({"input_cost_per_token": 0.01})
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -260,7 +310,8 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() {
|
|||
)]));
|
||||
let response = json!({"model": "served"});
|
||||
assert_eq!(
|
||||
catalog.pricing_entry_for_cost_calc(
|
||||
pricing(
|
||||
&catalog,
|
||||
ModelSelectionRequest {
|
||||
response: Some(&response),
|
||||
base_model: Some("unpriced"),
|
||||
|
|
@ -268,7 +319,10 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() {
|
|||
},
|
||||
None,
|
||||
),
|
||||
Some(("openai/served", &json!({"input_cost_per_token": 0.01})))
|
||||
Some((
|
||||
"openai/served".to_owned(),
|
||||
json!({"input_cost_per_token": 0.01})
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -138,7 +138,6 @@ fn finding_request<'a>(response: &'a Value) -> ZeroCostFindingRequest<'a> {
|
|||
provider: Some("openai"),
|
||||
router_model_id: None,
|
||||
region_name: None,
|
||||
known_providers: &["openai"],
|
||||
},
|
||||
logging_details: None,
|
||||
metadata: None,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue