feat(cost): port get_model_info key resolution and get_llm_provider inference

ModelInfoCatalog now resolves models the way Python's _get_model_info_helper
does: the six candidate names from _get_potential_model_names, case-insensitive
keys, bedrock routing and base-model stripping, the mantle region prefix, the
fireworks cost key, vertex @latest and meta/ aliases, the azure alias table, the
huggingface zero-rated entry, and the fallback_generalizations capability rules.

Provider inference follows get_llm_provider instead of reading the cost-map
litellm_provider: bare ids route through the provider model lists, the JSON
provider registry, the static openai and cohere lists, and the routing rules.
cost_per_token infers the provider when the caller passes none and fails with
MissingProvider where Python raises BadRequestError.

Bedrock and fal image pricing resolve through get_model_info, and Stability 1
rejects a steps value Python cannot compare with 50.

A differential run over every cost-map key and its prefixed, cased, dated, and
bedrock variants (21,365 lookups) matches Python except for ollama and lemonade,
whose Python lookup queries a local server.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-23 22:43:03 -07:00
parent eb627547c0
commit edbf88daf2
35 changed files with 1677 additions and 333 deletions

View file

@ -2979,6 +2979,7 @@ name = "litellm-cost"
version = "0.1.0"
dependencies = [
"criterion",
"fancy-regex 0.19.2",
"jiff",
"litellm-python-compat",
"litellm-token-counter",

View file

@ -1,7 +1,7 @@
# litellm-cost port rules
- This crate is a standalone port of Python's cost calculation. Prices, usage, model metadata, response kinds, and feature flags are caller-supplied; there is no Python bridge, no proxy integration, no network, and no env or secret resolution
- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline
- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline. Model lookup goes through `ModelInfoCatalog::get_model_info` (`model_info.rs`, mirroring `utils.py::_get_model_info_helper`) and provider inference through `get_llm_provider.rs` and `fallback_generalizations.rs`; never read `entries()` by a hand-built key where Python calls `get_model_info`. Known gaps: provider configs whose `get_model_info` queries a live server (ollama, lemonade) and the `ModelInfoBase` field projection are not ported
- Module `foo_cost.rs` mirrors the calculation portion of the matching Python module; `tests/python_foo_cost.rs` carries the mapping evidence, with each test naming the Python test it mirrors (`tests/test_litellm/` mirrors `litellm/` per the repo-root AGENTS.md)
- Preserve Python's precedence and fallback orders exactly: cache hit before provider-reported before calculated cost, discount before margin, wall-clock pricing before provider totals, quality and size key selection order, model-key selection order including duplicate provider prefixes and regional aliases
- Diverge only where Python is observably wrong, and pin each divergence with a test that would fail if either side changed

View file

@ -10,6 +10,7 @@ serde.workspace = true
serde_json.workspace = true
litellm-python-compat = { path = "../python-compat" }
litellm-token-counter = { path = "../token-counter", default-features = false }
fancy-regex.workspace = true
jiff = "=0.2.37"
regex = "=1.13.1"
strum.workspace = true

View file

@ -57,6 +57,7 @@ pub fn azure_ai_cost_per_token(
request_model: Option<&str>,
) -> Result<(f64, f64), CostError> {
let model_info = catalog.entry(request.model, Some("azure_ai"), request.region);
let model_info = model_info.as_deref();
if let Some(cost) =
model_info.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
{
@ -98,6 +99,6 @@ pub fn azure_ai_router_fee(
let entry = catalog
.entry(fee_entry, Some("azure_ai"), None)
.ok_or(CostError::ModelNotFound)?;
let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, entry);
let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, &entry);
Ok((fee > 0.0).then_some(fee))
}

View file

@ -0,0 +1,103 @@
use std::sync::LazyLock;
use regex::Regex;
const ROUTING_PREFIXES: [&str; 7] = [
"bedrock/",
"converse/",
"invoke/",
"openai/",
"mantle/",
"nova-2/",
"nova/",
];
const CROSS_REGION_INFERENCE_REGIONS: [&str; 7] =
["global", "us", "eu", "apac", "jp", "au", "us-gov"];
const ALL_REGIONS: [&str; 23] = [
"us-east-1",
"us-east-2",
"us-west-1",
"us-west-2",
"us-gov-east-1",
"us-gov-west-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"eu-central-1",
"eu-central-2",
"eu-south-1",
"eu-south-2",
"eu-north-1",
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-south-1",
"ap-south-2",
"ap-southeast-1",
"ap-southeast-2",
"ca-central-1",
"sa-east-1",
];
static THROUGHPUT_SUFFIX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(:\d+):\d+k$").expect("valid throughput pattern"));
static CONTEXT_WINDOW_SUFFIX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\[\w+\]$").expect("valid context window pattern"));
pub fn is_bedrock_region(region: &str) -> bool {
ALL_REGIONS.contains(&region)
}
pub fn strip_bedrock_routing_prefix(model: &str) -> &str {
ROUTING_PREFIXES.iter().fold(model, |model, prefix| {
if model.starts_with(prefix) {
model.split_once('/').map_or(model, |(_, rest)| rest)
} else {
model
}
})
}
pub fn strip_bedrock_throughput_suffix(model: &str) -> String {
let without_throughput = THROUGHPUT_SUFFIX.replace(model, "${1}");
CONTEXT_WINDOW_SUFFIX
.replace(&without_throughput, "")
.into_owned()
}
pub fn extract_model_name_from_bedrock_arn(model: &str) -> &str {
if model.to_lowercase().contains("arn") {
model.rsplit('/').next().unwrap_or(model)
} else {
model
}
}
pub fn get_bedrock_base_model(model: &str) -> String {
let spec = ["bedrock/converse/", "bedrock/", "converse/"]
.into_iter()
.find_map(|prefix| model.strip_prefix(prefix))
.unwrap_or(model);
if spec.starts_with("nova-2/") {
return "amazon.nova-2-custom".to_owned();
}
if spec.starts_with("nova/") {
return "amazon.nova-custom".to_owned();
}
let model = strip_bedrock_throughput_suffix(extract_model_name_from_bedrock_arn(
strip_bedrock_routing_prefix(model),
));
if let Some((region, rest)) = model.split_once('.')
&& CROSS_REGION_INFERENCE_REGIONS.contains(&region)
{
return rest.to_owned();
}
if let Some((region, rest)) = model.split_once('/')
&& is_bedrock_region(region)
{
return rest.to_owned();
}
model
}

View file

@ -1,7 +1,9 @@
use std::collections::HashMap;
use serde_json::Value;
use crate::catalog::ModelInfoCatalog;
use crate::error::CostError;
use crate::wire::py_real;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum BedrockImageFamily {
Titan,
@ -22,57 +24,48 @@ pub fn get_config_class(model: &str) -> BedrockImageFamily {
}
}
pub fn stability1_pricing_key(model: &str, size: Option<&str>, optional_params: &Value) -> String {
let steps = optional_params
.get("steps")
.and_then(Value::as_f64)
.unwrap_or(50.0);
pub fn stability1_pricing_key(
model: &str,
size: Option<&str>,
optional_params: &Value,
) -> Result<String, CostError> {
let steps = match optional_params.get("steps") {
None => 50.0,
Some(steps) => py_real(steps).ok_or(CostError::InvalidSteps)?,
};
let tier = if steps > 50.0 {
"max-steps"
} else {
"50-steps"
};
format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024"))
}
fn model_info<'a>(
entries: &'a HashMap<String, Value>,
model: &str,
family: BedrockImageFamily,
size: Option<&str>,
optional_params: &Value,
) -> Option<&'a Value> {
let bare = model.strip_prefix("bedrock/").unwrap_or(model);
match family {
BedrockImageFamily::Stability1 => {
let key = stability1_pricing_key(model, size, optional_params);
let bare_key = stability1_pricing_key(bare, size, optional_params);
[key, bare_key]
.into_iter()
.find_map(|candidate| entries.get(&candidate))
}
_ => [model.to_owned(), format!("bedrock/{bare}"), bare.to_owned()]
.into_iter()
.find_map(|candidate| entries.get(&candidate)),
}
Ok(format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024")))
}
pub fn cost_calculator(
catalog: &ModelInfoCatalog,
model: &str,
image_response: &Value,
size: Option<&str>,
optional_params: &Value,
entries: &HashMap<String, Value>,
) -> Option<f64> {
let family = get_config_class(model);
let info = model_info(entries, model, family, size, optional_params)?;
let rate = info
) -> Result<f64, CostError> {
let model_info = match get_config_class(model) {
BedrockImageFamily::Titan => catalog.get_model_info(model, None)?,
BedrockImageFamily::NovaCanvas | BedrockImageFamily::Stability3 => {
catalog.get_model_info(model, Some("bedrock"))?
}
BedrockImageFamily::Stability1 => catalog.get_model_info(
&stability1_pricing_key(model, size.filter(|size| !size.is_empty()), optional_params)?,
Some("bedrock"),
)?,
};
let rate = model_info
.info
.get("output_cost_per_image")
.and_then(Value::as_f64)
.and_then(py_real)
.unwrap_or(0.0);
let images = image_response
.get("data")
.and_then(Value::as_array)
.map_or(0, Vec::len);
Some(rate * images as f64)
Ok(rate * images as f64)
}

View file

@ -190,12 +190,12 @@ pub fn calculate_token_type_cost_breakdown(
}
fn billed_rates_request<'a>(
catalog: &'a ModelInfoCatalog,
model_info: Option<&'a Value>,
request: ModelCostRequest<'a>,
custom_cost_per_token: Option<CustomTokenRates>,
) -> BilledRatesRequest<'a> {
BilledRatesRequest {
model_info: catalog.entry(request.model, request.provider, request.region),
model_info,
usage: request.usage,
provider: request.provider,
service_tier: request.service_tier,
@ -211,8 +211,9 @@ pub fn get_billed_token_rates(
request: ModelCostRequest<'_>,
custom_cost_per_token: Option<CustomTokenRates>,
) -> Option<BilledTokenRates> {
let model_info = catalog.entry(request.model, request.provider, request.region);
calculate_billed_token_rates(billed_rates_request(
catalog,
model_info.as_deref(),
request,
custom_cost_per_token,
))
@ -223,8 +224,9 @@ pub fn get_token_type_cost_breakdown(
request: ModelCostRequest<'_>,
custom_cost_per_token: Option<CustomTokenRates>,
) -> TokenTypeCostBreakdown {
let model_info = catalog.entry(request.model, request.provider, request.region);
calculate_token_type_cost_breakdown(billed_rates_request(
catalog,
model_info.as_deref(),
request,
custom_cost_per_token,
))

View file

@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::collections::HashMap;
use jiff::Timestamp;
@ -7,6 +8,11 @@ use crate::completion_input::{
CompletionInputRequest, PreparedCompletionInput, prepare_completion_input,
};
use crate::error::CostError;
use crate::fallback_generalizations::FallbackGeneralizations;
use crate::get_llm_provider::{LlmProvider, ProviderModelSets, get_llm_provider};
use crate::model_info::{ModelCost, ModelInfo, get_model_info_helper, lowercase_key_map};
const FALLBACK_GENERALIZATIONS_KEY: &str = "fallback_generalizations";
use crate::model_selection::{
ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc,
};
@ -15,6 +21,9 @@ use crate::responses_usage::ChatUsage;
#[derive(Clone, Debug, Default)]
pub struct ModelInfoCatalog {
entries: HashMap<String, Value>,
lowercase_keys: HashMap<String, String>,
model_sets: ProviderModelSets,
generalizations: FallbackGeneralizations,
}
#[derive(Clone, Copy, Debug)]
@ -63,33 +72,13 @@ pub enum CostCall<'a> {
},
}
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
return true;
};
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
return true;
};
entry_provider == provider
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|| provider == "litellm_proxy"
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|| provider == "github"
}
fn select_model_key<'a>(
entries: &'a HashMap<String, Value>,
model: &str,
fn cost_map_model<'m>(
entries: &HashMap<String, Value>,
model: &'m str,
provider: Option<&str>,
region: Option<&str>,
) -> Option<&'a str> {
let match_provider = provider.map(|provider| match provider {
"vertex_ai_beta" => "vertex_ai",
provider => provider,
});
let normalized = match provider {
) -> Cow<'m, str> {
let model = match provider {
Some(provider) => {
let prefix = format!("{provider}/");
let mut name = model;
@ -106,44 +95,94 @@ fn select_model_key<'a>(
let model_with_provider = match provider {
Some(provider) => {
let prefix = format!("{provider}/");
let bare = normalized.strip_prefix(&prefix).unwrap_or(normalized);
let bare = model.strip_prefix(&prefix).unwrap_or(model);
let regional = region.map(|region| format!("{provider}/{region}/{bare}"));
if let Some(regional) = regional.filter(|key| entries.contains_key(key)) {
regional
} else if normalized.starts_with(&prefix) {
normalized.to_owned()
Cow::Owned(regional)
} else if model.starts_with(&prefix) {
Cow::Borrowed(model)
} else {
format!("{provider}/{normalized}")
Cow::Owned(format!("{provider}/{model}"))
}
}
None => normalized.to_owned(),
None => Cow::Borrowed(model),
};
let without_prefix = normalized
let without_prefix = model
.split_once('/')
.map_or(normalized, |(_, remainder)| remainder);
[model_with_provider.as_str(), normalized, without_prefix]
.into_iter()
.find_map(|candidate| {
entries
.get_key_value(candidate)
.filter(|(_, model_info)| check_provider_match(model_info, match_provider))
.map(|(key, _)| key.as_str())
})
.map_or(model, |(_, remainder)| remainder);
if entries.contains_key(model_with_provider.as_ref()) {
model_with_provider
} else if entries.contains_key(model) {
Cow::Borrowed(model)
} else if entries.contains_key(without_prefix) {
Cow::Borrowed(without_prefix)
} else {
Cow::Borrowed(model)
}
}
impl ModelInfoCatalog {
pub fn new(entries: HashMap<String, Value>) -> Self {
Self { entries }
pub fn new(mut entries: HashMap<String, Value>) -> Self {
let generalizations = FallbackGeneralizations::from_block(
entries.remove(FALLBACK_GENERALIZATIONS_KEY).as_ref(),
);
Self {
lowercase_keys: lowercase_key_map(&entries),
model_sets: ProviderModelSets::from_model_cost(&entries),
generalizations,
entries,
}
}
fn model_cost(&self) -> ModelCost<'_> {
ModelCost {
entries: &self.entries,
lowercase_keys: &self.lowercase_keys,
sets: &self.model_sets,
generalizations: &self.generalizations,
}
}
pub fn entries(&self) -> &HashMap<String, Value> {
&self.entries
}
pub fn entry_for_key(&self, key: &str) -> &Value {
self.entries
.get(key)
.expect("key came from select_model_key")
pub fn get_llm_provider(&self, model: &str) -> Option<LlmProvider> {
get_llm_provider(model, &self.model_sets, &self.generalizations)
}
pub fn get_model_info(
&self,
model: &str,
provider: Option<&str>,
) -> Result<ModelInfo<'_>, CostError> {
get_model_info_helper(self.model_cost(), model, provider)
}
pub fn cost_map_model<'m>(
&self,
model: &'m str,
provider: Option<&str>,
region: Option<&str>,
) -> Cow<'m, str> {
cost_map_model(&self.entries, model, provider, region)
}
pub fn select_model_info(
&self,
model: &str,
provider: Option<&str>,
region: Option<&str>,
) -> Option<ModelInfo<'_>> {
let selected = cost_map_model(&self.entries, model, provider, region);
let inferred = provider
.is_none()
.then(|| self.get_llm_provider(model))
.flatten();
let provider = provider.or(inferred
.as_ref()
.map(|inferred| inferred.custom_llm_provider.as_str()));
self.get_model_info(&selected, provider).ok()
}
pub fn entry(
@ -151,45 +190,27 @@ impl ModelInfoCatalog {
model: &str,
provider: Option<&str>,
region: Option<&str>,
) -> Option<&Value> {
self.select_model_key(model, provider, region)
.and_then(|key| self.entries.get(key))
}
pub fn select_model_key<'a>(
&'a self,
model: &str,
provider: Option<&str>,
region: Option<&str>,
) -> Option<&'a str> {
select_model_key(&self.entries, model, provider, region)
) -> Option<Cow<'_, Value>> {
self.select_model_info(model, provider, region)
.map(|model_info| model_info.info)
}
pub fn contains_exact_model(&self, model: &str) -> bool {
self.entries.contains_key(model)
}
pub fn get_provider_for_cost_calc(
&self,
model: Option<&str>,
provider: Option<&str>,
known_providers: &[&str],
) -> Option<String> {
get_provider_for_cost_calc(model, provider, known_providers, &self.entries)
}
pub fn select_model_name_for_cost_calc(
&self,
request: ModelSelectionRequest<'_>,
) -> Option<String> {
select_model_name_for_cost_calc(request, &self.entries)
select_model_name_for_cost_calc(request, self)
}
pub fn pricing_entry_for_cost_calc<'a>(
&'a self,
request: ModelSelectionRequest<'a>,
logging_details: Option<&'a Value>,
) -> Option<(&'a str, &'a Value)> {
) -> Option<ModelInfo<'a>> {
let deployment_info = if request.custom_pricing {
request
.router_model_id
@ -209,15 +230,13 @@ impl ModelInfoCatalog {
if let (Some(key), Some(info)) =
(request.router_model_id.or(request.model), deployment_info)
{
return Some((key, info));
return Some(ModelInfo {
key: Cow::Borrowed(key),
info: Cow::Borrowed(info),
});
}
let selected = self.select_model_name_for_cost_calc(request);
let provider = get_provider_for_cost_calc(
request.model,
request.provider,
request.known_providers,
&self.entries,
);
let provider = get_provider_for_cost_calc(request.model, request.provider, self);
[
selected.as_deref(),
request
@ -228,16 +247,13 @@ impl ModelInfoCatalog {
]
.into_iter()
.flatten()
.find_map(|model| {
self.select_model_key(model, provider.as_deref(), None)
.and_then(|key| self.entries.get(key).map(|info| (key, info)))
})
.find_map(|model| self.select_model_info(model, provider.as_deref(), None))
}
pub fn prepare_completion_input(
&self,
request: CompletionInputRequest<'_>,
) -> Result<PreparedCompletionInput, CostError> {
prepare_completion_input(request, &self.entries)
prepare_completion_input(request, self)
}
}

View file

@ -1,4 +1,4 @@
use std::collections::HashMap;
use crate::catalog::ModelInfoCatalog;
use serde_json::Value;
@ -106,7 +106,7 @@ pub fn select_service_tier(
pub fn prepare_completion_input(
request: CompletionInputRequest<'_>,
cost_map: &HashMap<String, Value>,
catalog: &ModelInfoCatalog,
) -> Result<PreparedCompletionInput, CostError> {
let response = request.model_selection.response;
let call_type =
@ -126,7 +126,7 @@ pub fn prepare_completion_input(
model,
..request.model_selection
},
cost_map,
catalog,
);
let response_model = get_response_model(response).map(str::to_owned);
let requested_model = model.map(str::to_owned);

View file

@ -25,6 +25,7 @@ use crate::image_cost_router::{
ImageCostRouteRequest, call_type_has_image_response, route_image_generation_cost_calculator,
};
use crate::mcp_cost::calculate_mcp_tool_call_cost;
use crate::model_selection::get_provider_for_cost_calc;
use crate::per_second::{DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, get_replicate_completion_pricing};
use crate::provider::LlmProviders;
use crate::realtime_cost::{
@ -647,13 +648,7 @@ pub fn completion_cost_from_response(
.model_candidates
.iter()
.flatten()
.find_map(|model| {
catalog.get_provider_for_cost_calc(
Some(model),
None,
request.input.model_selection.known_providers,
)
})
.find_map(|model| get_provider_for_cost_calc(Some(model), None, catalog))
});
let provider = inferred_provider.as_deref();
let deployment_info = request

View file

@ -96,6 +96,16 @@ pub fn cost_per_token_for_call(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
call: CostCall<'_>,
) -> Result<(f64, f64), CostError> {
with_inferred_provider(catalog, request, |catalog, request| {
cost_per_token_for_provider(catalog, request, call)
})
}
fn cost_per_token_for_provider(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
call: CostCall<'_>,
) -> Result<(f64, f64), CostError> {
let provider = request
.provider
@ -119,7 +129,7 @@ pub fn cost_per_token_for_call(
if provider == Some(LlmProviders::AZURE_AI) {
return azure_ai_cost_per_token(catalog, request, request_model);
}
Ok(cost_per_token(catalog, request)?)
token_cost_per_token(catalog, request)
}
CostCall::Speech { prompt_characters } => speech_cost(catalog, request, prompt_characters),
CostCall::Transcription { duration_seconds } => {
@ -154,7 +164,7 @@ pub fn cost_per_token_for_call(
deployment_info,
} => {
let published = catalog.entry(request.model, request.provider, request.region);
Ok(ocr_cost(response, deployment_info, published)?)
Ok(ocr_cost(response, deployment_info, published.as_deref())?)
}
CostCall::Batch { deployment_info } => {
let published = catalog.entry(request.model, request.provider, request.region);
@ -171,7 +181,7 @@ pub fn cost_per_token_for_call(
let info = if has_deployment_rates {
deployment_info
} else {
published.or(deployment_info)
published.as_deref().or(deployment_info)
};
let Some(info) = info else {
return Ok((0.0, 0.0));
@ -190,10 +200,39 @@ pub fn cost_per_token_for_call(
pub fn cost_per_token(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
) -> Result<(f64, f64), CostError> {
with_inferred_provider(catalog, request, token_cost_per_token)
}
fn with_inferred_provider(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
price: impl FnOnce(&ModelInfoCatalog, ModelCostRequest<'_>) -> Result<(f64, f64), CostError>,
) -> Result<(f64, f64), CostError> {
if request.provider.is_some() {
return price(catalog, request);
}
let inferred = catalog
.get_llm_provider(request.model)
.ok_or(CostError::MissingProvider)?;
let selected = catalog.cost_map_model(request.model, None, request.region);
price(
catalog,
ModelCostRequest {
model: &selected,
provider: Some(&inferred.custom_llm_provider),
..request
},
)
}
fn token_cost_per_token(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
) -> Result<(f64, f64), CostError> {
if let Some(cost) = catalog
.entry(request.model, request.provider, request.region)
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
{
return Ok(cost);
}
@ -219,18 +258,17 @@ pub fn cost_per_token(
{
return Ok((0.0, cost));
}
let key = catalog
.select_model_key(request.model, request.provider, request.region)
let selected = catalog
.select_model_info(request.model, request.provider, request.region)
.or_else(|| {
(provider == Some(LlmProviders::FIREWORKS_AI))
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
.and_then(|category| {
catalog.select_model_key(category, request.provider, request.region)
catalog.select_model_info(category, request.provider, request.region)
})
})
.ok_or(CostError::ModelNotFound)?;
let model_info =
apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider);
let model_info = apply_provider_cache_read_default(&selected.info, request.provider);
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
return Ok(cost);
}
@ -262,7 +300,7 @@ pub fn cost_per_token(
Some(LlmProviders::FIREWORKS_AI) => {
return Ok(fireworks_cost_per_token(
request.usage,
catalog.entry_for_key(key),
&selected.info,
request.at,
));
}
@ -313,7 +351,7 @@ pub fn cost_per_token(
_ => (request.service_tier, request.data_residency),
};
let cost = generic_cost_per_token(GenericCostRequest {
model_info: catalog.entry_for_key(key),
model_info: &selected.info,
usage: request.usage,
provider: request.provider,
service_tier,
@ -369,6 +407,7 @@ pub fn speech_cost(
let model_info = catalog
.entry(request.model, request.provider, request.region)
.ok_or(CostError::ModelNotFound)?;
let model_info = &*model_info;
match select_cost_metric_for_model(model_info)? {
SpeechCostMetric::PerCharacter => {
let characters = prompt_characters.ok_or(CostError::MissingPromptCharacters)?;
@ -399,6 +438,7 @@ pub fn transcription_cost(
let model_info = catalog
.entry(request.model, request.provider, request.region)
.ok_or(CostError::ModelNotFound)?;
let model_info = &*model_info;
if transcription_usage_has_token_details(request.usage) {
return Ok(generic_cost_per_token(GenericCostRequest {
model_info,
@ -421,6 +461,7 @@ pub fn handle_realtime_transcription_cost_calculation(
) -> f64 {
let model = get_transcription_model_name_from_results(results).unwrap_or(requested_model);
let model_info = catalog.entry(model, Some(provider), None);
let model_info = model_info.as_deref();
results
.iter()
.filter(|event| {
@ -467,7 +508,7 @@ pub fn handle_realtime_stream_cost_calculation(
.find_map(|model| {
let model_info = catalog.entry(model, Some(provider), None)?;
let cost = generic_cost_per_token(GenericCostRequest {
model_info,
model_info: &model_info,
usage: combined_usage,
provider: Some(provider),
service_tier: None,

View file

@ -34,7 +34,7 @@ pub fn databricks_cost_per_token(
) -> Result<(f64, f64), CostError> {
if let Some(cost) = catalog
.entry(request.model, Some("databricks"), request.region)
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
{
return Ok(cost);
}
@ -47,7 +47,7 @@ pub fn databricks_cost_per_token(
.ok_or(CostError::ModelNotFound)?;
Ok(calculate_generic_cost_from_model_info_with_region(
request.usage,
entry,
&entry,
None,
false,
None,

View file

@ -32,6 +32,8 @@ pub enum CostError {
MissingProvider,
#[error("no rate for the priced unit")]
MissingRate,
#[error("image steps are not a number")]
InvalidSteps,
#[error("no usage on the response")]
MissingUsage,
#[error("model is not in the cost map")]

View file

@ -4,6 +4,7 @@ use serde_json::Value;
use crate::catalog::ModelInfoCatalog;
use crate::image_response_cost::resolve_image_model_info;
use crate::wire::py_real;
pub const PIXELS_PER_MEGAPIXEL: u64 = 1_048_576;
@ -136,12 +137,13 @@ pub fn fal_ai_passthrough_cost_from_model_info(
}
pub fn cost_calculator(
catalog: &ModelInfoCatalog,
model: &str,
image_response: &Value,
optional_params: &Value,
deployment_prices: Option<&Value>,
entries: &HashMap<String, Value>,
) -> Option<f64> {
let entries = catalog.entries();
let images = image_response.get("data").and_then(Value::as_array);
let count = images.map_or(0, Vec::len);
if let Some(rate) = deployment_prices
@ -160,15 +162,16 @@ pub fn cost_calculator(
if keyed.iter().all(Option::is_some) {
return Some(keyed.into_iter().flatten().sum());
}
let shared = entries.get(&format!("fal_ai/{normalized_model}"));
let resolved = resolve_image_model_info(shared, deployment_prices)?;
let shared = catalog
.get_model_info(normalized_model, Some("fal_ai"))
.ok()
.map(|model_info| model_info.info);
let resolved = resolve_image_model_info(shared.as_deref(), deployment_prices)?;
let flat_rate = resolved
.get("output_cost_per_image")
.and_then(Value::as_f64)
.and_then(py_real)
.unwrap_or(0.0);
let pixel_rate = resolved
.get("output_cost_per_pixel")
.and_then(Value::as_f64);
let pixel_rate = resolved.get("output_cost_per_pixel").and_then(py_real);
Some(
images
.into_iter()

View file

@ -0,0 +1,147 @@
use std::collections::HashMap;
use fancy_regex::{Regex, RegexBuilder};
use serde_json::{Map, Value};
const PROVIDER_KEY: &str = "litellm_provider";
#[derive(Clone, Debug)]
struct RoutingRule {
pattern: Regex,
provider: String,
}
#[derive(Clone, Debug)]
struct CapabilityRule {
pattern: Regex,
model_info: Map<String, Value>,
}
#[derive(Clone, Debug, Default)]
pub struct FallbackGeneralizations {
routing_rules: Vec<RoutingRule>,
capability_rules: Vec<CapabilityRule>,
}
fn resolve_legacy_extends(rules: &[Value]) -> Vec<Value> {
let base_by_name: HashMap<&str, &Map<String, Value>> = rules
.iter()
.filter_map(|rule| {
Some((
rule.get("name")?.as_str()?,
rule.get("model_info")?.as_object()?,
))
})
.collect();
rules
.iter()
.map(|rule| {
let parent = rule
.get("extends")
.and_then(Value::as_str)
.and_then(|name| base_by_name.get(name));
match (parent, rule.get("model_info").and_then(Value::as_object)) {
(Some(parent), Some(own)) => {
let mut merged = (*parent).clone();
merged.extend(own.clone());
let mut resolved = rule.as_object().cloned().unwrap_or_default();
resolved.insert("model_info".to_owned(), Value::Object(merged));
Value::Object(resolved)
}
_ => rule.clone(),
}
})
.collect()
}
fn fill_missing_is_malformed(rule: &Value) -> bool {
rule.get("fill_missing_for_providers")
.is_some_and(|providers| {
!providers
.as_array()
.is_some_and(|providers| providers.iter().all(Value::is_string))
})
}
fn compile_rule(rule: &Value) -> (Option<RoutingRule>, Option<CapabilityRule>) {
let (Some(pattern), Some(model_info)) = (
rule.get("pattern").and_then(Value::as_str),
rule.get("model_info").and_then(Value::as_object),
) else {
return (None, None);
};
let Ok(pattern) = RegexBuilder::new(pattern).case_insensitive(true).build() else {
return (None, None);
};
if fill_missing_is_malformed(rule) {
return (None, None);
}
let capability = || CapabilityRule {
pattern: pattern.clone(),
model_info: model_info.clone(),
};
match model_info.get(PROVIDER_KEY) {
None => (None, Some(capability())),
Some(Value::String(provider)) => {
let routing = RoutingRule {
pattern: pattern.clone(),
provider: provider.clone(),
};
let capability = (model_info.len() > 1).then(capability);
(Some(routing), capability)
}
Some(_) => (None, None),
}
}
fn is_match(pattern: &Regex, model: &str) -> bool {
pattern.is_match(model).unwrap_or(false)
}
impl FallbackGeneralizations {
pub fn from_block(block: Option<&Value>) -> Self {
let rules = block
.and_then(|block| block.get("rules"))
.and_then(Value::as_array)
.map(|rules| resolve_legacy_extends(rules))
.unwrap_or_default();
let (routing_rules, capability_rules): (Vec<_>, Vec<_>) =
rules.iter().map(compile_rule).unzip();
Self {
routing_rules: routing_rules.into_iter().flatten().collect(),
capability_rules: capability_rules.into_iter().flatten().collect(),
}
}
pub fn match_routing(&self, model: &str) -> Option<&str> {
if model.is_empty() {
return None;
}
self.routing_rules
.iter()
.find(|rule| is_match(&rule.pattern, model))
.map(|rule| rule.provider.as_str())
}
pub fn match_capabilities(&self, model: &str) -> Option<Map<String, Value>> {
if model.is_empty() {
return None;
}
let matched: Vec<&Map<String, Value>> = self
.capability_rules
.iter()
.filter(|rule| is_match(&rule.pattern, model))
.map(|rule| &rule.model_info)
.collect();
(!matched.is_empty()).then(|| {
matched
.into_iter()
.flat_map(|model_info| {
model_info
.iter()
.map(|(key, value)| (key.clone(), value.clone()))
})
.collect()
})
}
}

View file

@ -0,0 +1,291 @@
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::LazyLock;
use regex::Regex;
use serde_json::Value;
use crate::fallback_generalizations::FallbackGeneralizations;
use crate::provider::LlmProviders;
const OPENAI_IMAGE_GENERATION_MODELS: [&str; 2] = ["dall-e-2", "dall-e-3"];
const OPENAI_VIDEO_GENERATION_MODELS: [&str; 1] = ["sora-2"];
const COHERE_EMBEDDING_MODELS: [&str; 7] = [
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
"embed-multilingual-v3.0",
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
];
const REPLICATE_MODEL_NAME_WITH_ID_LENGTH: usize = 64;
const PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: [&str; 2] = ["github_copilot", "chatgpt"];
const MODEL_PREFIX_PROVIDERS: [(&str, &str); 11] = [
("bytez/", "bytez"),
("gdc/", "gdc"),
("lemonade/", "lemonade"),
("heroku/", "heroku"),
("cometapi/", "cometapi"),
("oci/", "oci"),
("compactifai/", "compactifai"),
("ovhcloud/", "ovhcloud"),
("clarifai/", "clarifai"),
("amazon_nova", "amazon_nova"),
("sap/", "sap"),
];
static JSON_PROVIDERS: LazyLock<HashSet<String>> = LazyLock::new(|| {
serde_json::from_str::<serde_json::Map<String, Value>>(include_str!(
"../../../../litellm/llms/openai_like/providers.json"
))
.expect("openai-like provider registry parses")
.into_iter()
.map(|(slug, _)| slug)
.collect()
});
static BEDROCK_PRICING_ONLY: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^bedrock/[a-zA-Z0-9_-]+/.+$").expect("valid bedrock pricing pattern")
});
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum ModelList {
OpenAiChatCompletion,
OpenAiTextCompletion,
Anthropic,
Cohere,
CohereChat,
MistralChat,
Openrouter,
VertexText,
VertexCodeText,
VertexLanguage,
VertexVision,
VertexChat,
VertexCodeChat,
VertexEmbedding,
VertexLlama3,
VertexMistral,
VertexAi21,
VertexImage,
VertexVideo,
Ai21Chat,
Ai21,
NlpCloud,
AlephAlpha,
Bedrock,
BedrockConverse,
Watsonx,
Empower,
GradientAi,
}
fn is_openai_finetune_model(key: &str) -> bool {
key.starts_with("ft:") && key.matches(':').count() <= 1
}
fn is_bedrock_pricing_only_model(key: &str) -> bool {
BEDROCK_PRICING_ONLY.is_match(key)
}
fn vertex_member(key: &str) -> String {
key.replace("vertex_ai/", "")
}
fn model_list_membership(key: &str, info: &Value) -> Option<(ModelList, String)> {
let provider = info.get("litellm_provider").and_then(Value::as_str)?;
let mode = info.get("mode").and_then(Value::as_str);
let list = match provider {
"openai" if !is_openai_finetune_model(key) => ModelList::OpenAiChatCompletion,
"text-completion-openai" => ModelList::OpenAiTextCompletion,
"cohere" => ModelList::Cohere,
"cohere_chat" => ModelList::CohereChat,
"mistral" => ModelList::MistralChat,
"anthropic" => ModelList::Anthropic,
"empower" => ModelList::Empower,
"openrouter" => ModelList::Openrouter,
"vertex_ai-text-models" => ModelList::VertexText,
"vertex_ai-code-text-models" => ModelList::VertexCodeText,
"vertex_ai-language-models" => ModelList::VertexLanguage,
"vertex_ai-vision-models" => ModelList::VertexVision,
"vertex_ai-chat-models" => ModelList::VertexChat,
"vertex_ai-code-chat-models" => ModelList::VertexCodeChat,
"vertex_ai-embedding-models" => ModelList::VertexEmbedding,
"vertex_ai-llama_models" => return Some((ModelList::VertexLlama3, vertex_member(key))),
"vertex_ai-mistral_models" => return Some((ModelList::VertexMistral, vertex_member(key))),
"vertex_ai-ai21_models" => return Some((ModelList::VertexAi21, vertex_member(key))),
"vertex_ai-image-models" => return Some((ModelList::VertexImage, vertex_member(key))),
"vertex_ai-video-models" => return Some((ModelList::VertexVideo, vertex_member(key))),
"ai21" if mode == Some("chat") => ModelList::Ai21Chat,
"ai21" => ModelList::Ai21,
"nlp_cloud" => ModelList::NlpCloud,
"aleph_alpha" => ModelList::AlephAlpha,
"bedrock" if mode == Some("guardrail") => return None,
"bedrock" if !is_bedrock_pricing_only_model(key) => ModelList::Bedrock,
"bedrock_converse" => ModelList::BedrockConverse,
"watsonx" => ModelList::Watsonx,
"gradient_ai" => ModelList::GradientAi,
_ => return None,
};
Some((list, key.to_owned()))
}
#[derive(Clone, Debug, Default)]
pub struct ProviderModelSets {
members: HashMap<ModelList, HashSet<String>>,
}
impl ProviderModelSets {
pub fn from_model_cost(model_cost: &HashMap<String, Value>) -> Self {
let members = model_cost
.iter()
.filter_map(|(key, info)| model_list_membership(key, info))
.fold(
HashMap::<ModelList, HashSet<String>>::new(),
|mut members, (list, name)| {
members.entry(list).or_default().insert(name);
members
},
);
Self { members }
}
pub fn contains(&self, list: ModelList, model: &str) -> bool {
self.members
.get(&list)
.is_some_and(|models| models.contains(model))
}
fn contains_any(&self, lists: &[ModelList], model: &str) -> bool {
lists.iter().any(|list| self.contains(*list, model))
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LlmProvider {
pub model: String,
pub custom_llm_provider: String,
}
pub fn declared_authenticating_provider(model: &str) -> Option<&str> {
model
.split_once('/')
.map(|(declared, _)| declared)
.filter(|declared| PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO.contains(declared))
}
fn is_anthropic_text_model(model: &str) -> bool {
matches!(model, "claude-2" | "claude-instant-1")
}
fn resolved(model: &str, provider: &str) -> Option<LlmProvider> {
Some(LlmProvider {
model: model.to_owned(),
custom_llm_provider: provider.to_owned(),
})
}
fn bare_model_provider<'a>(model: &str, sets: &ProviderModelSets) -> Option<&'a str> {
use ModelList::*;
let replicate_id = model
.split(':')
.nth(1)
.is_some_and(|id| id.len() == REPLICATE_MODEL_NAME_WITH_ID_LENGTH);
if sets.contains(OpenAiChatCompletion, model)
|| model.contains("ft:gpt-3.5-turbo")
|| model.contains("ft:gpt-4")
|| OPENAI_IMAGE_GENERATION_MODELS.contains(&model)
|| model.starts_with("gpt-image")
|| OPENAI_VIDEO_GENERATION_MODELS.contains(&model)
{
Some("openai")
} else if sets.contains(OpenAiTextCompletion, model) {
Some("text-completion-openai")
} else if sets.contains(Anthropic, model) {
Some(if is_anthropic_text_model(model) {
"anthropic_text"
} else {
"anthropic"
})
} else if sets.contains(Cohere, model) || COHERE_EMBEDDING_MODELS.contains(&model) {
Some("cohere")
} else if sets.contains(CohereChat, model) {
Some("cohere_chat")
} else if model.contains(':') && model.len() > REPLICATE_MODEL_NAME_WITH_ID_LENGTH {
replicate_id.then_some("replicate")
} else if sets.contains(Openrouter, model) {
Some("openrouter")
} else if sets.contains_any(
&[
VertexChat,
VertexCodeChat,
VertexText,
VertexCodeText,
VertexLanguage,
VertexEmbedding,
VertexVision,
VertexImage,
VertexVideo,
],
model,
) {
Some("vertex_ai")
} else if sets.contains_any(&[Ai21Chat, Ai21], model) {
Some("ai21_chat")
} else if sets.contains(AlephAlpha, model) {
Some("aleph_alpha")
} else if sets.contains(NlpCloud, model) {
Some("nlp_cloud")
} else if sets.contains_any(&[Bedrock, BedrockConverse], model) {
Some("bedrock")
} else if sets.contains(Watsonx, model) {
Some("watsonx")
} else if sets.contains(Empower, model) {
Some("empower")
} else if sets.contains(GradientAi, model) {
Some("gradient_ai")
} else if model == "*" {
Some("openai")
} else {
MODEL_PREFIX_PROVIDERS
.iter()
.find(|(prefix, _)| model.starts_with(prefix))
.map(|(_, provider)| *provider)
}
}
pub fn get_llm_provider(
model: &str,
sets: &ProviderModelSets,
generalizations: &FallbackGeneralizations,
) -> Option<LlmProvider> {
if let Some((prefix, rest)) = model.split_once('/') {
if prefix == "azure"
&& (sets.contains(ModelList::CohereChat, rest)
|| sets.contains(ModelList::MistralChat, &format!("mistral/{rest}")))
{
return resolved(model, "openai");
}
if prefix == "cohere" && sets.contains(ModelList::CohereChat, rest) {
return resolved(rest, "cohere_chat");
}
if prefix == "anthropic" && is_anthropic_text_model(rest) {
return resolved(rest, "anthropic_text");
}
if JSON_PROVIDERS.contains(prefix) {
return resolved(rest, prefix);
}
if LlmProviders::from_str(prefix).is_ok() {
let provider = if prefix == "ai21" {
"ai21_chat"
} else {
prefix
};
return resolved(rest, provider);
}
}
bare_model_provider(model, sets)
.or_else(|| generalizations.match_routing(model))
.and_then(|provider| resolved(model, provider))
}

View file

@ -282,21 +282,20 @@ pub fn route_image_generation_cost_calculator<'a>(
pricing.as_ref(),
),
Some(LlmProviders::FAL_AI) => fal_ai_image_cost_calculator(
catalog,
request.model,
request.image_response,
request.optional_params,
pricing.as_ref(),
catalog.entries(),
)
.ok_or(CostError::ModelNotFound),
Some(LlmProviders::BEDROCK) => bedrock_image_cost_calculator(
catalog,
request.model,
request.image_response,
Some(resolved_size),
request.optional_params,
catalog.entries(),
)
.ok_or(CostError::ModelNotFound),
),
Some(provider @ (LlmProviders::OPENAI | LlmProviders::AZURE))
if request.model.to_ascii_lowercase().contains("gpt-image") =>
{
@ -330,7 +329,11 @@ fn resolved_model_info(
supplied_model_info: Option<&Value>,
) -> Result<Value, CostError> {
resolve_image_model_info(
catalog.entry(model, Some(provider), None),
catalog
.get_model_info(model, Some(provider))
.ok()
.map(|model_info| model_info.info)
.as_deref(),
supplied_model_info,
)
.ok_or(CostError::ModelNotFound)
@ -374,9 +377,8 @@ pub fn google_image_edit_cost(
Ok(gemini_image_edit_cost(image_response, &model_info, at))
}
"vertex_ai" => catalog
.entry(model, Some(provider), None)
.map(|model_info| vertex_image_edit_cost(image_response, model_info))
.ok_or(CostError::ModelNotFound),
.get_model_info(model, Some(provider))
.map(|model_info| vertex_image_edit_cost(image_response, &model_info.info)),
_ => Err(CostError::ModelNotFound),
}
}
@ -385,7 +387,8 @@ pub fn azure_ai_image_generation_cost(
catalog: &ModelInfoCatalog,
request: AzureAiImageCatalogRequest<'_>,
) -> Result<f64, CostError> {
let shared = catalog.entry(request.model, Some("azure_ai"), None);
let selected = catalog.get_model_info(request.model, Some("azure_ai")).ok();
let shared = selected.as_ref().map(|selected| &*selected.info);
let model_info = resolve_image_model_info(shared, request.supplied_model_info)
.ok_or(CostError::ModelNotFound)?;
let shared_pricing = model_info
@ -395,9 +398,9 @@ pub fn azure_ai_image_generation_cost(
.or(shared);
azure_ai_image_cost_calculator(AzureAiImageRequest {
catalog,
model: catalog
.select_model_key(request.model, Some("azure_ai"), None)
.unwrap_or(request.model),
model: selected
.as_ref()
.map_or(request.model, |selected| &selected.key),
image_response: request.image_response,
model_info: &model_info,
supplied_model_info: request.supplied_model_info,

View file

@ -15,6 +15,7 @@ pub mod background_cost_polling;
pub mod base_rate_selection;
#[allow(clippy::disallowed_types)]
pub mod batch;
pub mod bedrock_common_utils;
#[allow(clippy::disallowed_types)]
pub mod bedrock_image_cost;
#[allow(clippy::disallowed_types)]
@ -40,6 +41,8 @@ pub mod error;
#[allow(clippy::disallowed_types)]
pub mod fal_ai_image_cost;
#[allow(clippy::disallowed_types)]
pub mod fallback_generalizations;
#[allow(clippy::disallowed_types)]
pub mod fireworks_cost;
#[allow(clippy::disallowed_types)]
pub mod gemini_cost;
@ -51,6 +54,8 @@ pub mod generic_input;
pub mod generic_output;
pub mod generic_usage;
#[allow(clippy::disallowed_types)]
pub mod get_llm_provider;
#[allow(clippy::disallowed_types)]
pub mod groq_cost;
#[allow(clippy::disallowed_types)]
pub mod guardrail_cost;
@ -65,6 +70,8 @@ pub mod lemonade_cost;
#[allow(clippy::disallowed_types)]
pub mod mcp_cost;
#[allow(clippy::disallowed_types)]
pub mod model_info;
#[allow(clippy::disallowed_types)]
pub mod model_selection;
pub mod non_token;
#[allow(clippy::disallowed_types)]

View file

@ -0,0 +1,306 @@
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::LazyLock;
use regex::Regex;
use serde_json::{Value, json};
use crate::bedrock_common_utils::{
get_bedrock_base_model, is_bedrock_region, strip_bedrock_routing_prefix,
};
use crate::error::CostError;
use crate::fallback_generalizations::FallbackGeneralizations;
use crate::get_llm_provider::{
ModelList, ProviderModelSets, declared_authenticating_provider, get_llm_provider,
};
const AZURE_LLMS: [(&str, &str); 7] = [
("gpt-35-turbo", "azure/gpt-35-turbo"),
("gpt-35-turbo-16k", "azure/gpt-35-turbo-16k"),
("gpt-35-turbo-instruct", "azure/gpt-35-turbo-instruct"),
("azure/gpt-41", "gpt-4.1"),
("azure/gpt-41-mini", "gpt-4.1-mini"),
("azure/gpt-41-nano", "gpt-4.1-nano"),
("ada", "azure/ada"),
];
static STABLE_VERTEX_VERSION: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"-\d+$").expect("valid vertex version pattern"));
static DATED_SNAPSHOT_SUFFIX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"-\d{4}-\d{2}-\d{2}$").expect("valid snapshot pattern"));
static OPENAI_FINETUNE_SUFFIX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(:[^:]*){3}$").expect("valid finetune pattern"));
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PotentialModelNames {
pub split_model: String,
pub combined_model_name: String,
pub stripped_model_name: String,
pub combined_stripped_model_name: String,
pub provider_prefixed_model_name: String,
pub custom_llm_provider: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct ModelInfo<'a> {
pub key: Cow<'a, str>,
pub info: Cow<'a, Value>,
}
#[derive(Clone, Copy, Debug)]
pub struct ModelCost<'a> {
pub entries: &'a HashMap<String, Value>,
pub lowercase_keys: &'a HashMap<String, String>,
pub sets: &'a ProviderModelSets,
pub generalizations: &'a FallbackGeneralizations,
}
pub fn strip_model_name(model: &str, custom_llm_provider: Option<&str>) -> String {
match custom_llm_provider {
Some("bedrock" | "bedrock_converse") => get_bedrock_base_model(model),
Some("vertex_ai" | "gemini" | "databricks") => {
STABLE_VERTEX_VERSION.replace(model, "").into_owned()
}
_ if model.contains("ft:") => OPENAI_FINETUNE_SUFFIX.replace(model, "").into_owned(),
_ => DATED_SNAPSHOT_SUFFIX.replace(model, "").into_owned(),
}
}
fn split_mantle_region_prefix(model: &str) -> &str {
match model.split_once('/') {
Some((head, tail)) if is_bedrock_region(head) => tail,
_ => model,
}
}
fn resolve_fireworks_resource_name(model: &str) -> String {
let stripped = model.strip_prefix("fireworks_ai/").unwrap_or(model);
if stripped.starts_with("accounts/") || stripped.starts_with("FW-") || stripped.contains('#') {
stripped.to_owned()
} else if stripped.starts_with("routers/") || stripped.starts_with("models/") {
format!("accounts/fireworks/{stripped}")
} else if stripped.ends_with("-fast") {
format!("accounts/fireworks/routers/{stripped}")
} else {
format!("accounts/fireworks/models/{stripped}")
}
}
pub fn get_potential_model_names(
model_cost: ModelCost<'_>,
model: &str,
custom_llm_provider: Option<&str>,
) -> PotentialModelNames {
let (split_model, combined, stripped, combined_stripped, provider_prefixed, provider) =
match custom_llm_provider {
None => {
let inferred = get_llm_provider(model, model_cost.sets, model_cost.generalizations);
let provider = inferred
.as_ref()
.map(|inferred| inferred.custom_llm_provider.clone());
let stripped = strip_model_name(model, provider.as_deref());
(
inferred.map_or_else(|| model.to_owned(), |inferred| inferred.model),
model.to_owned(),
stripped.clone(),
stripped,
model.to_owned(),
provider,
)
}
Some(provider) => {
let (split, combined, provider_prefixed) =
match model.strip_prefix(&format!("{provider}/")) {
Some(split) => (split, model.to_owned(), format!("{provider}/{model}")),
None => (
model,
format!("{provider}/{model}"),
format!("{provider}/{model}"),
),
};
let stripped = strip_model_name(split, Some(provider));
let combined_stripped = format!("{provider}/{stripped}");
(
split.to_owned(),
combined,
stripped,
combined_stripped,
provider_prefixed,
Some(provider.to_owned()),
)
}
};
let split_model = match provider.as_deref() {
Some("bedrock" | "bedrock_converse") => {
strip_bedrock_routing_prefix(&split_model).to_owned()
}
_ => split_model,
};
let (split_model, combined_stripped) = match provider.as_deref() {
Some(provider @ "bedrock_mantle") => {
let region_free = split_mantle_region_prefix(&split_model).to_owned();
let combined_stripped = format!(
"bedrock_mantle/{}",
strip_model_name(&region_free, Some(provider))
);
(region_free, combined_stripped)
}
_ => (split_model, combined_stripped),
};
let provider_prefixed = match provider.as_deref() {
Some("fireworks_ai") => format!(
"fireworks_ai/{}",
resolve_fireworks_resource_name(&split_model)
),
_ => provider_prefixed,
};
PotentialModelNames {
split_model,
combined_model_name: combined,
stripped_model_name: stripped,
combined_stripped_model_name: combined_stripped,
provider_prefixed_model_name: provider_prefixed,
custom_llm_provider: provider,
}
}
pub fn get_model_cost_key<'a>(model_cost: ModelCost<'a>, potential_key: &str) -> Option<&'a str> {
model_cost
.entries
.get_key_value(potential_key)
.map(|(key, _)| key.as_str())
.or_else(|| {
model_cost
.lowercase_keys
.get(&potential_key.to_lowercase())
.map(String::as_str)
})
}
pub fn lowercase_key_map(model_cost: &HashMap<String, Value>) -> HashMap<String, String> {
model_cost.keys().fold(HashMap::new(), |mut map, key| {
map.entry(key.to_lowercase())
.and_modify(|existing: &mut String| {
if key < existing {
existing.clone_from(key);
}
})
.or_insert_with(|| key.clone());
map
})
}
fn vertex_ai_model_alias(model: &str, sets: &ProviderModelSets) -> Option<String> {
let meta = format!("meta/{model}");
if sets.contains(ModelList::VertexLlama3, &meta) {
return Some(meta);
}
let latest = format!("{model}@latest");
(sets.contains(ModelList::VertexMistral, &latest)
|| sets.contains(ModelList::VertexAi21, &latest))
.then_some(latest)
}
fn get_model_info_from_generalization(
model_cost: ModelCost<'_>,
candidates: &[&str],
custom_llm_provider: Option<&str>,
) -> Option<(String, Value)> {
if candidates
.iter()
.any(|candidate| get_model_cost_key(model_cost, candidate).is_some())
{
return None;
}
candidates.iter().find_map(|candidate| {
let mut info = model_cost.generalizations.match_capabilities(candidate)?;
if let Some(provider) = custom_llm_provider {
info.insert(
"litellm_provider".to_owned(),
Value::String(provider.to_owned()),
);
}
Some(((*candidate).to_owned(), Value::Object(info)))
})
}
pub fn get_model_info_helper<'a>(
model_cost: ModelCost<'a>,
model: &str,
custom_llm_provider: Option<&str>,
) -> Result<ModelInfo<'a>, CostError> {
let model = AZURE_LLMS
.iter()
.find(|(alias, _)| *alias == model)
.map_or(model, |(_, target)| *target);
let custom_llm_provider = custom_llm_provider.map(|provider| match provider {
"vertex_ai_beta" => "vertex_ai",
provider => provider,
});
let model = match custom_llm_provider {
Some("vertex_ai") => {
vertex_ai_model_alias(model, model_cost.sets).map_or(Cow::Borrowed(model), Cow::Owned)
}
_ => Cow::Borrowed(model),
};
let names = get_potential_model_names(
model_cost,
&model,
custom_llm_provider.or_else(|| declared_authenticating_provider(&model)),
);
let provider = names.custom_llm_provider.as_deref();
if provider == Some("huggingface") {
return Ok(ModelInfo {
key: Cow::Owned(model.into_owned()),
info: Cow::Owned(json!({
"input_cost_per_token": 0,
"output_cost_per_token": 0,
"litellm_provider": "huggingface",
"mode": "chat"
})),
});
}
let candidates = [
names.combined_model_name.as_str(),
&model,
&names.split_model,
&names.combined_stripped_model_name,
&names.stripped_model_name,
&names.provider_prefixed_model_name,
];
let mapped = candidates
.iter()
.filter_map(|candidate| get_model_cost_key(model_cost, candidate))
.find_map(|key| {
let info = &model_cost.entries[key];
check_provider_match(info, provider).then_some(ModelInfo {
key: Cow::Borrowed(key),
info: Cow::Borrowed(info),
})
});
if let Some(mapped) = mapped {
return Ok(mapped);
}
get_model_info_from_generalization(model_cost, &candidates, provider)
.map(|(key, info)| ModelInfo {
key: Cow::Owned(key),
info: Cow::Owned(info),
})
.ok_or(CostError::ModelNotFound)
}
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
return true;
};
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
return true;
};
entry_provider == provider
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|| provider == "litellm_proxy"
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|| provider == "github"
}

View file

@ -1,7 +1,10 @@
use std::collections::HashMap;
use std::str::FromStr;
use serde_json::Value;
use crate::catalog::ModelInfoCatalog;
use crate::provider::LlmProviders;
#[derive(Clone, Copy, Debug)]
pub struct ModelSelectionRequest<'a> {
pub model: Option<&'a str>,
@ -12,7 +15,6 @@ pub struct ModelSelectionRequest<'a> {
pub provider: Option<&'a str>,
pub router_model_id: Option<&'a str>,
pub region_name: Option<&'a str>,
pub known_providers: &'a [&'a str],
}
pub fn get_response_model(response: Option<&Value>) -> Option<&str> {
@ -29,17 +31,20 @@ pub fn get_hidden_str_for_cost_calc<'a>(
.filter(|value| !value.is_empty())
}
pub fn model_contains_known_llm_provider(model: &str, known_providers: &[&str]) -> bool {
let prefix = model.split('/').next().unwrap_or(model);
known_providers.contains(&prefix)
fn is_llm_provider(name: &str) -> bool {
LlmProviders::from_str(name).is_ok()
}
pub fn model_contains_known_llm_provider(model: &str) -> bool {
model.split('/').next().is_some_and(is_llm_provider)
}
pub fn strip_unregistered_leading_segments(
model: &str,
region_name: Option<&str>,
known_providers: &[&str],
cost_map: &HashMap<String, Value>,
catalog: &ModelInfoCatalog,
) -> String {
let cost_map = catalog.entries();
let segments: Vec<&str> = model.split('/').collect();
if segments.len() < 2 || cost_map.contains_key(&segments[1..].join("/")) {
return model.to_owned();
@ -55,7 +60,7 @@ pub fn strip_unregistered_leading_segments(
let tail = &segments[head_len..];
let first_provider = tail
.iter()
.position(|segment| known_providers.contains(segment))
.position(|segment| is_llm_provider(segment))
.unwrap_or(tail.len());
(0..=first_provider.min(tail.len().saturating_sub(1)))
.map(|start| format!("{head}/{}", tail[start..].join("/")))
@ -76,38 +81,22 @@ fn has_explicit_pricing(entry: &Value) -> bool {
pub fn get_provider_for_cost_calc(
model: Option<&str>,
explicit_provider: Option<&str>,
known_providers: &[&str],
cost_map: &HashMap<String, Value>,
custom_llm_provider: Option<&str>,
catalog: &ModelInfoCatalog,
) -> Option<String> {
explicit_provider
.map(str::to_owned)
.or_else(|| {
model
.and_then(|model| model.split_once('/'))
.map(|(provider, _)| provider)
.filter(|provider| known_providers.contains(provider))
.map(str::to_owned)
})
.or_else(|| {
model
.and_then(|model| cost_map.get(model))
.and_then(|info| info.get("litellm_provider"))
.and_then(Value::as_str)
.map(str::to_owned)
})
custom_llm_provider.map(str::to_owned).or_else(|| {
catalog
.get_llm_provider(model?)
.map(|resolved| resolved.custom_llm_provider)
})
}
pub fn select_model_name_for_cost_calc(
request: ModelSelectionRequest<'_>,
cost_map: &HashMap<String, Value>,
catalog: &ModelInfoCatalog,
) -> Option<String> {
let provider = get_provider_for_cost_calc(
request.model,
request.provider,
request.known_providers,
cost_map,
);
let cost_map = catalog.entries();
let provider = get_provider_for_cost_calc(request.model, request.provider, catalog);
let response_model = get_response_model(request.response);
let private_model =
get_hidden_str_for_cost_calc(request.hidden_params, "provider_response_model");
@ -135,7 +124,7 @@ pub fn select_model_name_for_cost_calc(
let Some(provider) = provider else {
return Some(selected.to_owned());
};
if model_contains_known_llm_provider(selected, request.known_providers) {
if model_contains_known_llm_provider(selected) {
return Some(selected.to_owned());
}
let prefix =
@ -143,7 +132,6 @@ pub fn select_model_name_for_cost_calc(
Some(strip_unregistered_leading_segments(
&format!("{prefix}/{selected}"),
priced_region,
request.known_providers,
cost_map,
catalog,
))
}

View file

@ -60,7 +60,7 @@ pub fn prompt_caching_savings_for_model(
let model_info = catalog.entry(request.model, request.provider, request.region)?;
Some(calculate_prompt_caching_savings(
PromptCachingSavingsRequest {
model_info,
model_info: &model_info,
usage: request.usage,
provider: request.provider,
service_tier: request.service_tier,

View file

@ -47,7 +47,7 @@ pub fn rerank_cost(
) -> (f64, f64) {
rerank_cost_from_model_info(
provider,
catalog.entry(model, Some(provider), region),
catalog.entry(model, Some(provider), region).as_deref(),
billed_units,
)
}

View file

@ -141,7 +141,7 @@ pub fn search_provider_cost_per_query(
.entry(model, provider, None)
.ok_or(CostError::ModelNotFound)?;
Ok(search_provider_cost_from_model_info(
model_info,
&model_info,
number_of_queries,
optional_params,
))

View file

@ -45,10 +45,17 @@ fn usage_reports_web_search(usage: &ChatUsage) -> bool {
)
}
#[derive(Clone, Copy, Debug, Default)]
#[derive(Clone, Debug, Default)]
pub struct ResolvedModelInfo<'a> {
pub model_info: Option<&'a Value>,
pub provider: Option<&'a str>,
pub model_info: Option<Cow<'a, Value>>,
pub provider: Option<Cow<'a, str>>,
}
fn litellm_provider<'a>(model_info: &Value) -> Option<Cow<'a, str>> {
model_info
.get("litellm_provider")
.and_then(Value::as_str)
.map(|provider| Cow::Owned(provider.to_owned()))
}
pub fn resolve_model_info<'a>(
@ -57,14 +64,13 @@ pub fn resolve_model_info<'a>(
provider: Option<&'a str>,
region: Option<&str>,
) -> ResolvedModelInfo<'a> {
let litellm_provider =
|model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str);
if let Some(direct) = catalog.entry(model, provider, region) {
return ResolvedModelInfo {
model_info: Some(direct),
provider: provider
.filter(|provider| !provider.is_empty())
.or_else(|| litellm_provider(direct)),
.map(Cow::Borrowed)
.or_else(|| litellm_provider(&direct)),
model_info: Some(direct),
};
}
let by_prefix = model
@ -73,12 +79,12 @@ pub fn resolve_model_info<'a>(
.flatten();
match by_prefix {
Some(by_prefix) => ResolvedModelInfo {
provider: litellm_provider(&by_prefix),
model_info: Some(by_prefix),
provider: litellm_provider(by_prefix),
},
None => ResolvedModelInfo {
model_info: None,
provider,
provider: provider.map(Cow::Borrowed),
},
}
}
@ -158,24 +164,27 @@ fn usage_with_anthropic_web_search<'a>(
Some(Cow::Owned(ChatUsage { extra, ..base }))
}
fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 {
fn maps_cost(usage: Option<&ChatUsage>, resolved: &ResolvedModelInfo<'_>) -> f64 {
let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some())
else {
return 0.0;
};
match resolved {
ResolvedModelInfo {
model_info: Some(model_info),
provider: Some(provider),
} => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0),
match (resolved.model_info.as_deref(), resolved.provider.as_deref()) {
(Some(model_info), Some(provider)) => {
get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0)
}
_ => 0.0,
}
}
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 {
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: &ResolvedModelInfo<'_>) -> f64 {
let usage =
usage_with_anthropic_web_search(request.usage, request.response, request.response_kind);
let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) {
let routed = match (
resolved.model_info.as_deref(),
usage.as_deref(),
resolved.provider.as_deref(),
) {
(Some(model_info), Some(usage), Some(provider)) => {
get_cost_for_web_search_request(provider, usage, model_info, request.defaults)
}
@ -184,7 +193,7 @@ fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelI
routed.unwrap_or_else(|| {
get_cost_for_web_search(
request.params.get("web_search_options"),
resolved.model_info,
resolved.model_info.as_deref(),
) * count_web_search_calls(request.response, request.response_kind) as f64
})
}
@ -243,17 +252,17 @@ pub fn get_cost_for_built_in_tools(
request: BuiltInToolCostRequest<'_>,
) -> f64 {
let resolved = resolve_model_info(catalog, model, request.provider, region);
let maps = maps_cost(request.usage, resolved);
let maps = maps_cost(request.usage, &resolved);
let web_search =
response_object_includes_web_search_call(request.response, request.response_kind, None)
|| (request.response_kind != ResponseKind::Responses
&& request.usage.is_some_and(usage_reports_web_search));
if web_search {
return maps + web_search_cost(request, resolved);
return maps + web_search_cost(request, &resolved);
}
let direct = catalog.entry(model, request.provider, region);
if response_object_includes_file_search_call(request.response, request.response_kind) {
return maps + file_search_cost(request, direct);
return maps + file_search_cost(request, direct.as_deref());
}
maps + azure_assistant_cost(request, direct)
maps + azure_assistant_cost(request, direct.as_deref())
}

View file

@ -142,7 +142,7 @@ pub fn vertex_cost(
) -> Result<(f64, f64), CostError> {
if let Some(cost) = catalog
.entry(request.model, request.provider, request.region)
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
.and_then(|info| per_second_pricing_cost(&info, request.response_time_ms))
{
return Ok(cost);
}
@ -164,7 +164,7 @@ pub fn vertex_cost(
Ok(cost_per_character(
model_without_prefix,
request.usage,
entry,
&entry,
(prompt_characters, completion_characters),
request.service_tier,
request.vertex_location,

View file

@ -103,12 +103,12 @@ pub fn zero_cost_finding(
return None;
}
let usage = get_usage_object(response).ok().flatten()?;
let (pricing_model, pricing_entry) =
let pricing =
catalog.pricing_entry_for_cost_calc(request.model_selection, request.logging_details)?;
let diagnostic = diagnose_zero_cost(
&usage,
pricing_model,
pricing_entry,
&pricing.key,
&pricing.info,
request.calculation_failed,
)?;
let model = request.model_selection.model.unwrap_or("None");

View file

@ -0,0 +1,44 @@
// mirrors: test_litellm/llms/bedrock/test_bedrock_common_utils.py::test_get_bedrock_base_model
// expected values recorded from litellm/llms/bedrock/common_utils.py
use litellm_cost::bedrock_common_utils::{get_bedrock_base_model, strip_bedrock_routing_prefix};
use rstest::rstest;
#[rstest]
#[case(
"bedrock/converse/us.anthropic.claude-x-v1:0",
"us.anthropic.claude-x-v1:0"
)]
#[case("converse/bedrock/m", "bedrock/m")]
#[case("openai/bedrock/m", "bedrock/m")]
#[case("nova-2/arn:aws:x", "arn:aws:x")]
#[case("bedrock/nova/arn:aws:y", "arn:aws:y")]
#[case("invoke/mantle/nova/m", "m")]
#[case("m", "m")]
fn strip_bedrock_routing_prefix_strips_each_prefix_once_in_order(
#[case] model: &str,
#[case] expected: &str,
) {
assert_eq!(strip_bedrock_routing_prefix(model), expected);
}
#[rstest]
#[case("us.meta.llama3-v1:0", "meta.llama3-v1:0")]
#[case("bedrock/converse/model", "model")]
#[case("bedrock/nova-2/arn:aws:bedrock:x", "amazon.nova-2-custom")]
#[case("bedrock/nova/arn:aws:x", "amazon.nova-custom")]
#[case("converse/nova-2/x", "amazon.nova-2-custom")]
#[case("invoke/nova/x", "x")]
#[case("anthropic.claude-v2:0:51k", "anthropic.claude-v2:0")]
#[case("us.anthropic.claude-opus-v1[1m]", "anthropic.claude-opus-v1")]
#[case(
"arn:aws:bedrock:us-east-1:1:foundation-model/anthropic.claude-v2",
"anthropic.claude-v2"
)]
#[case("us-east-1/amazon.titan", "amazon.titan")]
#[case("ap-south-9/amazon.titan", "ap-south-9/amazon.titan")]
#[case("global.x", "x")]
#[case("mars.x", "mars.x")]
fn get_bedrock_base_model_matches_python(#[case] model: &str, #[case] expected: &str) {
assert_eq!(get_bedrock_base_model(model), expected);
}

View file

@ -1,6 +1,7 @@
#![allow(clippy::disallowed_types)]
// mirrors: image_gen_tests/test_bedrock_image_gen_unit_tests.py::test_cost_calculator_stability1
// expected values recorded from litellm/llms/bedrock/image_generation/cost_calculator.py::cost_calculator
use std::collections::HashMap;
@ -10,6 +11,7 @@ use litellm_cost::bedrock_image_cost::{
};
use litellm_cost::call_type::CallTypes;
use litellm_cost::catalog::ModelInfoCatalog;
use litellm_cost::error::CostError;
use litellm_cost::image_cost_router::{
ImageCostRouteRequest, route_image_generation_cost_calculator,
};
@ -40,52 +42,75 @@ fn get_config_class_matches_model_family(
}
#[rstest]
#[case(50, "50-steps")]
#[case(51, "max-steps")]
fn stability1_pricing_key_selects_step_tier(#[case] steps: u64, #[case] tier: &str) {
let key = stability1_pricing_key(
"stability.stable-diffusion-xl-v1",
Some("1024-x-1024"),
&json!({"steps": steps}),
);
#[case::at_the_threshold(json!({"steps": 50}), "50-steps")]
#[case::above_the_threshold(json!({"steps": 51}), "max-steps")]
#[case::float_steps(json!({"steps": 50.5}), "max-steps")]
#[case::bool_steps_compare_as_int(json!({"steps": true}), "50-steps")]
#[case::default_steps(json!({}), "50-steps")]
fn stability1_pricing_key_selects_step_tier(#[case] params: Value, #[case] tier: &str) {
assert_eq!(
key,
format!("1024-x-1024/{tier}/stability.stable-diffusion-xl-v1")
stability1_pricing_key("stability.sd-zz-v1", Some("1024-x-1024"), &params),
Ok(format!("1024-x-1024/{tier}/stability.sd-zz-v1"))
);
}
fn bedrock_image_catalog() -> ModelInfoCatalog {
let entry = |rate: f64| json!({"output_cost_per_image": rate, "litellm_provider": "bedrock", "mode": "image_generation"});
ModelInfoCatalog::new(HashMap::from([
(
"1024-x-1024/50-steps/stability.sd-zz-v1".to_owned(),
entry(0.04),
),
(
"1024-x-1024/max-steps/stability.sd-zz-v1".to_owned(),
entry(0.08),
),
("amazon.nova-canvas-zz".to_owned(), entry(0.03)),
("amazon.titan-image-zz".to_owned(), entry(0.02)),
("stability.sd3-zz".to_owned(), entry(0.05)),
]))
}
#[rstest]
fn bedrock_stability1_cost_uses_size_and_steps_lookup() {
let entries = HashMap::from([
(
"1024-x-1024/50-steps/stability.stable-diffusion-xl-v1".to_owned(),
json!({"output_cost_per_image": 0.04}),
),
(
"1024-x-1024/max-steps/stability.stable-diffusion-xl-v1".to_owned(),
json!({"output_cost_per_image": 0.08}),
),
]);
let response = json!({"data": [{}, {}]});
#[case::stability1_default_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 50}), Ok(0.08))]
#[case::stability1_max_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 60}), Ok(0.16))]
#[case::stability1_missing_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({}), Ok(0.08))]
#[case::stability1_default_size("stability.sd-zz-v1", None, json!({"steps": true}), Ok(0.08))]
#[case::stability1_null_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": null}), Err(CostError::InvalidSteps))]
#[case::stability1_string_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": "60"}), Err(CostError::InvalidSteps))]
#[case::stability1_keeps_the_routing_prefix_in_the_key(
"bedrock/stability.sd-zz-v1",
Some("1024-x-1024"),
json!({"steps": 50}),
Err(CostError::ModelNotFound)
)]
#[case::stability1_key_is_case_insensitive("STABILITY.SD-ZZ-V1", Some("1024-x-1024"), json!({}), Ok(0.08))]
#[case::nova_bare("amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
#[case::nova_routing_prefix("bedrock/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
#[case::nova_cross_region_prefix("us.amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
#[case::nova_region_path("bedrock/us-east-1/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
#[case::nova_invoke_route("invoke/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))]
#[case::nova_unmapped("amazon.nova-canvas-missing", None, json!({}), Err(CostError::ModelNotFound))]
#[case::titan_bare("amazon.titan-image-zz", None, json!({}), Ok(0.04))]
#[case::titan_routing_prefix("bedrock/amazon.titan-image-zz", None, json!({}), Ok(0.04))]
#[case::titan_infers_no_provider_for_a_cross_region_id(
"us.amazon.titan-image-zz",
None,
json!({}),
Err(CostError::ModelNotFound)
)]
#[case::stability3_bare("stability.sd3-zz", None, json!({}), Ok(0.1))]
#[case::stability3_routing_prefix("bedrock/stability.sd3-zz", None, json!({}), Ok(0.1))]
fn bedrock_image_cost_resolves_models_like_get_model_info(
#[case] model: &str,
#[case] size: Option<&str>,
#[case] params: Value,
#[case] expected: Result<f64, CostError>,
) {
let response = json!({"data": [{"url": "x"}, {"url": "y"}]});
assert_eq!(
cost_calculator(
"bedrock/stability.stable-diffusion-xl-v1",
&response,
Some("1024-x-1024"),
&json!({"steps": 50}),
&entries
),
Some(0.08)
);
assert_eq!(
cost_calculator(
"bedrock/stability.stable-diffusion-xl-v1",
&response,
Some("1024-x-1024"),
&json!({"steps": 60}),
&entries
),
Some(0.16)
cost_calculator(&bedrock_image_catalog(), model, &response, size, &params),
expected
);
}
@ -95,8 +120,8 @@ fn bedrock_stability1_cost_uses_size_and_steps_lookup() {
#[case("stability.sd3-large-v1:0", 0.04)]
fn bedrock_direct_families_bill_returned_images(#[case] model: &str, #[case] rate: f64) {
let catalog = ModelInfoCatalog::new(HashMap::from([(
format!("bedrock/{model}"),
json!({"output_cost_per_image": rate}),
model.to_owned(),
json!({"output_cost_per_image": rate, "litellm_provider": "bedrock"}),
)]));
let response = json!({"data": [{}, {}, {}]});
let supplied = json!({"output_cost_per_image": 1.0});

View file

@ -15,8 +15,6 @@ use litellm_cost::model_selection::ModelSelectionRequest;
use rstest::rstest;
use serde_json::{Value, json};
const PROVIDERS: &[&str] = &["anthropic", "vertex_ai"];
#[rstest]
#[case("ON_DEMAND_PRIORITY", Some("priority"))]
#[case("on_demand_flex", Some("flex"))]
@ -108,7 +106,6 @@ fn prepared_input_prices_served_priority_rate() {
provider: Some("anthropic"),
router_model_id: None,
region_name: None,
known_providers: PROVIDERS,
},
call_type: None,
response_kind: Some(ResponseKind::Completion),

View file

@ -19,17 +19,6 @@ use litellm_token_counter::{Error as TokenCounterError, TokenCounter, Tokenizer}
use rstest::rstest;
use serde_json::{Value, json};
const PROVIDERS: &[&str] = &[
"anthropic",
"azure_ai",
"exa_ai",
"openai",
"recraft",
"replicate",
"vertex_ai",
"xai",
];
struct CharacterTokenizer;
impl Tokenizer for CharacterTokenizer {
@ -56,7 +45,6 @@ fn request<'a>(
provider,
router_model_id: None,
region_name: None,
known_providers: PROVIDERS,
},
call_type: None,
response_kind: Some(ResponseKind::Completion),
@ -165,12 +153,13 @@ fn response_cost_calculator_prices_response_without_provider_override() {
}
#[rstest]
#[case("xai/served", json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))]
#[case("served", json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))]
fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[case] info: Value) {
let catalog = ModelInfoCatalog::new(HashMap::from([(model.to_owned(), info)]));
fn response_cost_infers_provider_for_specialized_pricing() {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"xai/served".to_owned(),
json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}),
)]));
let response = json!({
"model": model,
"model": "xai/served",
"usage": {
"prompt_tokens": 100,
"completion_tokens": 10,
@ -181,12 +170,33 @@ fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[
let empty = json!({});
let result = completion_cost_from_response(
&catalog,
request(Some(&response), Some(model), None, &empty, &empty),
request(Some(&response), Some("xai/served"), None, &empty, &empty),
)
.unwrap();
assert!((result.cost.total - (100.0 * 0.01 + 30.0 * 0.02)).abs() < 1e-12);
}
#[rstest]
fn response_cost_rejects_a_bare_model_get_llm_provider_cannot_route() {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"served".to_owned(),
json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}),
)]));
let response = json!({
"model": "served",
"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}
});
let empty = json!({});
assert_eq!(
completion_cost_from_response(
&catalog,
request(Some(&response), Some("served"), None, &empty, &empty),
)
.map(|result| result.cost.total),
Err(CostError::MissingProvider)
);
}
#[rstest]
fn response_cost_infers_provider_from_priced_fallback_model() {
let catalog = ModelInfoCatalog::new(HashMap::from([(

View file

@ -98,7 +98,7 @@ fn flat_cost_per_image_rounds_each_response_to_megapixels() {
#[rstest]
fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() {
let entries = HashMap::from([
let catalog = ModelInfoCatalog::new(HashMap::from([
(
"fal_ai/high/1024-x-768/openai/model".to_owned(),
json!({"output_cost_per_image": 0.10}),
@ -107,42 +107,42 @@ fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() {
"fal_ai/openai/model".to_owned(),
json!({"output_cost_per_image": 0.50}),
),
]);
]));
let response = json!({"data": [{}, {}]});
let supplied = json!({"output_cost_per_image": 0.25});
assert_eq!(
cost_calculator(
&catalog,
"fal_ai/openai/model",
&response,
&json!({}),
Some(&supplied),
&entries
),
Some(0.50)
);
assert_eq!(
cost_calculator("fal_ai/openai/model", &response, &json!({}), None, &entries),
cost_calculator(&catalog, "fal_ai/openai/model", &response, &json!({}), None),
Some(0.20)
);
}
#[rstest]
fn fal_image_cost_falls_back_to_base_megapixel_and_flat_rates_per_image() {
let entries = HashMap::from([(
let catalog = ModelInfoCatalog::new(HashMap::from([(
"fal_ai/fal-ai/flux/dev".to_owned(),
json!({"output_cost_per_image": 0.5, "output_cost_per_pixel": 0.0001}),
)]);
)]));
let response = json!({"data": [
image(json!(1024), json!(1024)),
image(json!(1920), json!(1080)),
image(json!(true), json!(1024))
]});
let cost = cost_calculator(
&catalog,
"fal_ai/fal-ai/flux/dev",
&response,
&json!({}),
None,
&entries,
)
.unwrap();
let expected = 0.0001 * PIXELS_PER_MEGAPIXEL as f64 * 3.0 + 0.5;
@ -210,3 +210,33 @@ fn fal_passthrough_selects_resolution_rate_and_falls_back_to_base() {
None
);
}
// expected values recorded from litellm/llms/fal_ai/cost_calculator.py::cost_calculator
#[rstest]
#[case::bare_key_owned_by_fal("fal_ai/flux-zz", Some(0.4))]
#[case::case_insensitive_key("FLUX-ZZ", Some(0.4))]
#[case::bool_rate_counts_as_int("fal_ai/flag-zz", Some(2.0))]
#[case::string_rate_is_not_a_number("fal_ai/str-zz", Some(0.0))]
#[case::unmapped_model("fal_ai/missing", None)]
fn fal_image_flat_fallback_resolves_through_get_model_info(
#[case] model: &str,
#[case] expected: Option<f64>,
) {
let entry = |rate: Value| json!({"output_cost_per_image": rate, "litellm_provider": "fal_ai"});
let catalog = ModelInfoCatalog::new(HashMap::from([
("flux-zz".to_owned(), entry(json!(0.2))),
("fal_ai/flag-zz".to_owned(), entry(json!(true))),
("fal_ai/str-zz".to_owned(), entry(json!("0.3"))),
]));
let response = json!({"data": [{"url": "x"}, {"url": "y"}]});
assert_eq!(
cost_calculator(
&catalog,
model,
&response,
&json!({"image_size": "weird"}),
None
),
expected
);
}

View file

@ -0,0 +1,262 @@
#![allow(clippy::disallowed_types)]
// mirrors: test_litellm/test_utils.py::test_get_model_info_* (get_model_info key resolution)
// expected values recorded from litellm/utils.py::_get_model_info_helper and
// litellm/litellm_core_utils/get_llm_provider_logic.py::get_llm_provider on this catalog
use std::collections::HashMap;
use litellm_cost::catalog::ModelInfoCatalog;
use litellm_cost::error::CostError;
use rstest::rstest;
use serde_json::{Value, json};
fn catalog() -> ModelInfoCatalog {
let provider = |name: &str| json!({"litellm_provider": name});
ModelInfoCatalog::new(HashMap::from([
("openai/gpt-zz".to_owned(), provider("openai")),
("gpt-zz".to_owned(), provider("openai")),
("gemini/gemini-zz".to_owned(), provider("gemini")),
(
"gemini-zz-pro".to_owned(),
provider("vertex_ai-language-models"),
),
("claude-zz".to_owned(), provider("anthropic")),
("ft:gpt-yy".to_owned(), provider("openai")),
(
"vertex_ai/meta/llama-zz".to_owned(),
provider("vertex_ai-llama_models"),
),
(
"vertex_ai/mistral-zz@latest".to_owned(),
provider("vertex_ai-mistral_models"),
),
(
"anthropic.claude-zz-v1".to_owned(),
provider("bedrock_converse"),
),
(
"bedrock_mantle/model-zz".to_owned(),
provider("bedrock_mantle"),
),
(
"fireworks_ai/accounts/fireworks/models/fw-zz".to_owned(),
provider("fireworks_ai"),
),
("gpt-4.1".to_owned(), provider("openai")),
("Mixed-Case-ZZ".to_owned(), provider("together_ai")),
(
"fallback_generalizations".to_owned(),
json!({"rules": [
{"name": "zz-route", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}},
{"name": "zz-cap", "pattern": "zzcap-\\d+", "model_info": {"mode": "chat", "max_tokens": 10}},
{"name": "zz-cap2", "pattern": "zzcap-9", "model_info": {"max_tokens": 20}},
{
"name": "zz-legacy",
"pattern": "^legacy-zz",
"extends": "zz-cap",
"model_info": {"litellm_provider": "xai", "max_tokens": 30}
}
]}),
),
]))
}
type Resolved = Result<(&'static str, Option<&'static str>, Option<u64>), CostError>;
#[rstest]
#[case::provider_prefixed_key_first("gpt-zz", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
#[case::bare_model_without_provider("gpt-zz", None, Ok(("gpt-zz", Some("openai"), None)))]
#[case::prefixed_model_without_provider("openai/gpt-zz", None, Ok(("openai/gpt-zz", Some("openai"), None)))]
#[case::case_insensitive_key("GPT-ZZ", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
#[case::dated_snapshot_suffix("gpt-zz-2025-01-01", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))]
#[case::dated_snapshot_without_provider("gpt-zz-2025-01-01", None, Ok(("gpt-zz", Some("openai"), None)))]
#[case::stable_version_suffix("gemini-zz-001", Some("gemini"), Ok(("gemini/gemini-zz", Some("gemini"), None)))]
#[case::vertex_family_entry(
"gemini-zz-pro",
Some("vertex_ai"),
Ok(("gemini-zz-pro", Some("vertex_ai-language-models"), None))
)]
#[case::mismatched_provider_is_skipped(
"gemini/gemini-zz",
Some("vertex_ai"),
Err(CostError::ModelNotFound)
)]
#[case::openai_finetune("ft:gpt-yy:org:suffix:id", Some("openai"), Ok(("ft:gpt-yy", Some("openai"), None)))]
#[case::vertex_llama_alias(
"llama-zz",
Some("vertex_ai"),
Ok(("vertex_ai/meta/llama-zz", Some("vertex_ai-llama_models"), None))
)]
#[case::vertex_beta_latest_alias(
"mistral-zz",
Some("vertex_ai_beta"),
Ok(("vertex_ai/mistral-zz@latest", Some("vertex_ai-mistral_models"), None))
)]
#[case::bedrock_cross_region_keeps_the_version(
"us.anthropic.claude-zz-v1:0",
Some("bedrock"),
Err(CostError::ModelNotFound)
)]
#[case::bedrock_routing_prefix(
"bedrock/converse/anthropic.claude-zz-v1",
Some("bedrock"),
Ok(("anthropic.claude-zz-v1", Some("bedrock_converse"), None))
)]
#[case::bedrock_throughput_suffix_keeps_the_version(
"anthropic.claude-zz-v1:0:51k",
Some("bedrock_converse"),
Err(CostError::ModelNotFound)
)]
#[case::mantle_region_prefix(
"bedrock_mantle/us-east-1/model-zz",
Some("bedrock_mantle"),
Ok(("bedrock_mantle/model-zz", Some("bedrock_mantle"), None))
)]
#[case::fireworks_resource_key(
"fw-zz",
Some("fireworks_ai"),
Ok(("fireworks_ai/accounts/fireworks/models/fw-zz", Some("fireworks_ai"), None))
)]
#[case::azure_alias("azure/gpt-41", None, Ok(("gpt-4.1", Some("openai"), None)))]
#[case::lowercase_query_finds_mixed_case_key(
"mixed-case-zz",
Some("together_ai"),
Ok(("Mixed-Case-ZZ", Some("together_ai"), None))
)]
#[case::huggingface_is_zero_rated("any-model", Some("huggingface"), Ok(("any-model", Some("huggingface"), None)))]
#[case::other_provider_entry("claude-zz", Some("openai"), Err(CostError::ModelNotFound))]
#[case::capability_union_later_rule_wins("zzcap-9", None, Ok(("zzcap-9", None, Some(20))))]
#[case::capability_backfills_the_provider("zzcap-9", Some("openai"), Ok(("openai/zzcap-9", Some("openai"), Some(20))))]
#[case::routing_rule_alone_has_no_capabilities("zzroute-x", None, Err(CostError::ModelNotFound))]
#[case::legacy_rule_routes_and_extends("legacy-zz-1", None, Ok(("legacy-zz-1", Some("xai"), Some(30))))]
#[case::unmapped("nothing-matches", None, Err(CostError::ModelNotFound))]
fn get_model_info_resolves_keys_like_python(
#[case] model: &str,
#[case] provider: Option<&str>,
#[case] expected: Resolved,
) {
let catalog = catalog();
let actual = catalog.get_model_info(model, provider).map(|resolved| {
(
resolved.key.into_owned(),
resolved
.info
.get("litellm_provider")
.and_then(Value::as_str)
.map(str::to_owned),
resolved.info.get("max_tokens").and_then(Value::as_u64),
)
});
assert_eq!(
actual,
expected.map(|(key, provider, max_tokens)| {
(key.to_owned(), provider.map(str::to_owned), max_tokens)
})
);
}
#[rstest]
fn get_model_info_prices_huggingface_at_zero() {
let catalog = catalog();
let info = catalog
.get_model_info("any-model", Some("huggingface"))
.unwrap()
.info;
assert_eq!(info.get("input_cost_per_token"), Some(&json!(0)));
assert_eq!(info.get("output_cost_per_token"), Some(&json!(0)));
}
#[rstest]
fn capability_rules_lose_to_an_exact_key_owned_by_another_provider() {
let catalog = ModelInfoCatalog::new(HashMap::from([
("zzcap-1".to_owned(), json!({"litellm_provider": "gemini"})),
(
"fallback_generalizations".to_owned(),
json!({"rules": [{"name": "cap", "pattern": "zzcap", "model_info": {"mode": "chat"}}]}),
),
]));
assert_eq!(
catalog
.get_model_info("zzcap-1", Some("openai"))
.map(|info| info.key.into_owned()),
Err(CostError::ModelNotFound)
);
assert_eq!(
catalog
.get_model_info("zzcap-2", Some("openai"))
.map(|info| info.key.into_owned()),
Ok("openai/zzcap-2".to_owned())
);
}
#[rstest]
#[case::cataloged_bare("gpt-zz", Some("openai"))]
#[case::provider_prefix("gemini/anything", Some("gemini"))]
#[case::vertex_family("gemini-zz-pro", Some("vertex_ai"))]
#[case::bedrock_converse_family("anthropic.claude-zz-v1", Some("bedrock"))]
#[case::routing_rule("zzroute-x", Some("anthropic"))]
#[case::legacy_routing_rule("legacy-zz-1", Some("xai"))]
#[case::azure_prefix_stays_azure("azure/gpt-41", Some("azure"))]
#[case::json_registry_provider("gmi/any", Some("gmi"))]
#[case::static_cohere_embedding("embed-v4.0", Some("cohere"))]
#[case::static_openai_image("dall-e-3", Some("openai"))]
#[case::unroutable("GPT-ZZ", None)]
fn get_llm_provider_infers_like_python(#[case] model: &str, #[case] expected: Option<&str>) {
assert_eq!(
catalog()
.get_llm_provider(model)
.map(|resolved| resolved.custom_llm_provider)
.as_deref(),
expected
);
}
fn usage() -> litellm_cost::responses_usage::ChatUsage {
litellm_cost::usage_dispatch::get_usage_object(
&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}}),
)
.unwrap()
.unwrap()
}
// expected values recorded from litellm.completion_cost for unmapped ids the shipped rules generalize
#[rstest]
#[case::routed_and_generalized("claude-newfam-9", None, Ok((0.0, 0.0)))]
#[case::generalized_under_explicit_provider("claude-newfam-9", Some("anthropic"), Ok((0.0, 0.0)))]
#[case::generalized_bedrock_id("anthropic.claude-newfam-9", Some("bedrock"), Ok((0.0, 0.0)))]
#[case::unroutable("totally-unknown", None, Err(CostError::MissingProvider))]
#[case::routable_but_unmapped("zzroute-x", None, Err(CostError::ModelNotFound))]
fn unmapped_models_price_through_capability_rules(
#[case] model: &str,
#[case] provider: Option<&str>,
#[case] expected: Result<(f64, f64), CostError>,
) {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"fallback_generalizations".to_owned(),
json!({"rules": [
{"name": "route-bedrock", "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-", "model_info": {"litellm_provider": "bedrock"}},
{"name": "route", "pattern": "^claude-[a-z]+-\\d+$", "model_info": {"litellm_provider": "anthropic"}},
{"name": "route-only", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}},
{"name": "baseline", "pattern": "claude-[a-z]+-\\d+", "model_info": {"mode": "chat"}}
]}),
)]));
let usage = usage();
assert_eq!(
litellm_cost::cost_calculator::cost_per_token(
&catalog,
litellm_cost::catalog::ModelCostRequest {
model,
provider,
region: None,
usage: &usage,
service_tier: None,
data_residency: None,
vertex_location: None,
at: "2026-01-01T12:00Z".parse().unwrap(),
response_time_ms: None,
},
),
expected
);
}

View file

@ -7,7 +7,8 @@ use litellm_cost::error::CostError;
use std::collections::HashMap;
use jiff::Timestamp;
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match};
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::model_info::check_provider_match;
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::json;
@ -84,7 +85,10 @@ fn cost_per_token_resolves_model_key_in_python_order(
#[case] expected: Option<&str>,
) {
assert_eq!(
catalog().select_model_key(model, provider, region),
catalog()
.select_model_info(model, provider, region)
.map(|selected| selected.key.into_owned())
.as_deref(),
expected
);
}
@ -221,7 +225,11 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
#[case::fireworks_request_rejects_other_providers(Some("openai"), Some("fireworks_ai"), false)]
#[case::fireworks_entry_needs_a_fireworks_request(Some("fireworks_ai-embedding-models"), Some("together_ai"), false)]
#[case::fireworks_entry_needs_a_fireworks_request(
Some("fireworks_ai-embedding-models"),
Some("together_ai"),
false
)]
#[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)]
#[case::bedrock_request_rejects_other_providers(Some("openai"), Some("bedrock"), false)]
#[case::bedrock_entry_needs_a_bedrock_request(Some("bedrock"), Some("sagemaker"), false)]
@ -252,7 +260,10 @@ fn select_model_key_skips_entries_owned_by_another_provider(
json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}),
)]));
assert_eq!(
catalog.select_model_key("gemini/gemini-x", provider, None),
catalog
.select_model_info("gemini/gemini-x", provider, None)
.map(|selected| selected.key.into_owned())
.as_deref(),
expected
);
}
@ -264,7 +275,10 @@ fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() {
json!({"litellm_provider": "vertex_ai-language-models"}),
)]));
assert_eq!(
catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None),
catalog
.select_model_info("vertex_ai/gemini-x", Some("vertex_ai_beta"), None)
.map(|selected| selected.key.into_owned())
.as_deref(),
Some("vertex_ai/gemini-x")
);
}

View file

@ -12,7 +12,15 @@ use litellm_cost::model_selection::{
use rstest::rstest;
use serde_json::{Value, json};
const PROVIDERS: &[&str] = &["openai", "bedrock", "vertex_ai", "anthropic", "dashscope"];
fn pricing<'a>(
catalog: &'a ModelInfoCatalog,
request: ModelSelectionRequest<'a>,
logging_details: Option<&'a Value>,
) -> Option<(String, Value)> {
catalog
.pricing_entry_for_cost_calc(request, logging_details)
.map(|selected| (selected.key.into_owned(), selected.info.into_owned()))
}
fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelectionRequest<'a> {
ModelSelectionRequest {
@ -24,7 +32,6 @@ fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelect
provider,
router_model_id: None,
region_name: None,
known_providers: PROVIDERS,
}
}
@ -166,31 +173,64 @@ fn model_selection_helpers_ignore_invalid_hidden_values_and_stop_at_provider_seg
get_hidden_str_for_cost_calc(Some(&json!({"model": ""})), "model"),
None
);
assert!(model_contains_known_llm_provider("openai/model", PROVIDERS));
assert!(!model_contains_known_llm_provider("team/model", PROVIDERS));
let catalog = HashMap::from([(
assert!(model_contains_known_llm_provider("openai/model"));
assert!(!model_contains_known_llm_provider("team/model"));
let catalog = ModelInfoCatalog::new(HashMap::from([(
"vertex_ai/claude".to_owned(),
json!({"input_cost_per_token": 1e-6}),
)]);
)]));
assert_eq!(
strip_unregistered_leading_segments("vertex_ai/openai/claude", None, PROVIDERS, &catalog,),
strip_unregistered_leading_segments("vertex_ai/openai/claude", None, &catalog),
"vertex_ai/openai/claude"
);
}
#[rstest]
#[case(Some("xai/model"), None, Some("xai"))]
#[case(Some("custom"), None, Some("xai"))]
#[case(Some("xai/model"), Some("anthropic"), Some("anthropic"))]
#[case(Some("unknown"), None, None)]
fn provider_inference_uses_explicit_prefix_or_catalog_metadata(
#[case::provider_prefix(Some("xai/model"), None, Some("xai"))]
#[case::explicit_provider_wins(Some("xai/model"), Some("anthropic"), Some("anthropic"))]
#[case::no_model(None, None, None)]
#[case::unlisted_bare_provider_is_not_inferred(Some("custom-xai"), None, None)]
#[case::unknown_bare_model(Some("unknown"), None, None)]
#[case::bedrock_converse_routes_to_bedrock(Some("converse-model"), None, Some("bedrock"))]
#[case::vertex_family_routes_to_vertex_ai(Some("vertex-model"), None, Some("vertex_ai"))]
#[case::ai21_chat(Some("ai21-model"), None, Some("ai21_chat"))]
#[case::ai21_prefix(Some("ai21/any"), None, Some("ai21_chat"))]
#[case::cohere_chat_under_cohere_prefix(Some("cohere/chat-model"), None, Some("cohere_chat"))]
#[case::azure_hosted_mistral(Some("azure/mistral-model"), None, Some("openai"))]
#[case::anthropic_text(Some("anthropic/claude-2"), None, Some("anthropic_text"))]
#[case::openai_finetune(Some("ft:gpt-4o:org::id"), None, Some("openai"))]
#[case::replicate_version_id(Some(&*format!("owner/model:{}", "a".repeat(64))), None, Some("replicate"))]
#[case::wildcard(Some("*"), None, Some("openai"))]
fn provider_inference_follows_get_llm_provider(
#[case] model: Option<&str>,
#[case] explicit: Option<&str>,
#[case] expected: Option<&str>,
) {
let catalog = HashMap::from([("custom".to_owned(), json!({"litellm_provider": "xai"}))]);
let catalog = ModelInfoCatalog::new(HashMap::from([
("custom-xai".to_owned(), json!({"litellm_provider": "xai"})),
(
"converse-model".to_owned(),
json!({"litellm_provider": "bedrock_converse"}),
),
(
"vertex-model".to_owned(),
json!({"litellm_provider": "vertex_ai-language-models"}),
),
(
"ai21-model".to_owned(),
json!({"litellm_provider": "ai21", "mode": "chat"}),
),
(
"chat-model".to_owned(),
json!({"litellm_provider": "cohere_chat"}),
),
(
"mistral/mistral-model".to_owned(),
json!({"litellm_provider": "mistral"}),
),
]));
assert_eq!(
get_provider_for_cost_calc(model, explicit, &["xai", "anthropic"], &catalog).as_deref(),
get_provider_for_cost_calc(model, explicit, &catalog).as_deref(),
expected
);
}
@ -215,8 +255,11 @@ fn pricing_entry_prefers_registered_deployment_to_served_model() {
..request(Some("requested"), Some("openai"))
};
assert_eq!(
catalog.pricing_entry_for_cost_calc(request, None),
Some(("router-id", &json!({"input_cost_per_token": 0.03})))
pricing(&catalog, request, None),
Some((
"router-id".to_owned(),
json!({"input_cost_per_token": 0.03})
))
);
}
@ -237,18 +280,25 @@ fn pricing_entry_reads_deployment_metadata_before_published_price() {
..request(Some("requested"), Some("openai"))
};
assert_eq!(
catalog.pricing_entry_for_cost_calc(selected, Some(&logging)),
Some(("requested", &json!({"input_cost_per_token": 0.02})))
pricing(&catalog, selected, Some(&logging)),
Some((
"requested".to_owned(),
json!({"input_cost_per_token": 0.02})
))
);
assert_eq!(
catalog.pricing_entry_for_cost_calc(
pricing(
&catalog,
ModelSelectionRequest {
custom_pricing: false,
..selected
},
Some(&logging)
),
Some(("openai/served", &json!({"input_cost_per_token": 0.01})))
Some((
"openai/served".to_owned(),
json!({"input_cost_per_token": 0.01})
))
);
}
@ -260,7 +310,8 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() {
)]));
let response = json!({"model": "served"});
assert_eq!(
catalog.pricing_entry_for_cost_calc(
pricing(
&catalog,
ModelSelectionRequest {
response: Some(&response),
base_model: Some("unpriced"),
@ -268,7 +319,10 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() {
},
None,
),
Some(("openai/served", &json!({"input_cost_per_token": 0.01})))
Some((
"openai/served".to_owned(),
json!({"input_cost_per_token": 0.01})
))
);
}

View file

@ -138,7 +138,6 @@ fn finding_request<'a>(response: &'a Value) -> ZeroCostFindingRequest<'a> {
provider: Some("openai"),
router_model_id: None,
region_name: None,
known_providers: &["openai"],
},
logging_details: None,
metadata: None,