mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(cost): map Together models to their size category once, before cost_per_token
Python swaps an unpriced Together model for its size category in completion_cost, with the call type, before cost_per_token runs. Rust did it inside cost_per_token as chat, so embeddings were mapped twice and per-second pricing was read from the unpriced model. together_pricing_model now runs in the completion layer only. The billed-rate breakdown also falls back to the private cache and reasoning counters with Python's _coerce_token_count rules Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
bf6ea4d781
commit
75170ed344
7 changed files with 216 additions and 65 deletions
|
|
@ -121,8 +121,12 @@ pub fn calculate_billed_token_rates(request: BilledRatesRequest<'_>) -> Option<B
|
|||
)
|
||||
}
|
||||
|
||||
fn count(value: Option<&Value>) -> u64 {
|
||||
value.and_then(Value::as_u64).unwrap_or(0)
|
||||
fn coerce_token_count(value: Option<&Value>) -> u64 {
|
||||
match value {
|
||||
Some(Value::Bool(true)) => 1,
|
||||
Some(Value::Number(number)) => number.as_u64().unwrap_or(0),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calculate_token_type_cost_breakdown(
|
||||
|
|
@ -145,7 +149,7 @@ pub fn calculate_token_type_cost_breakdown(
|
|||
.as_ref()
|
||||
.map(|details| details.cache_hit_tokens)
|
||||
.filter(|tokens| *tokens > 0)
|
||||
.unwrap_or_else(|| count(request.usage.extra.get("_cache_read_input_tokens")));
|
||||
.unwrap_or_else(|| coerce_token_count(request.usage.extra.get("_cache_read_input_tokens")));
|
||||
let cached_audio = prompt
|
||||
.as_ref()
|
||||
.map_or(0, |details| details.cache_hit_audio_tokens);
|
||||
|
|
@ -153,14 +157,16 @@ pub fn calculate_token_type_cost_breakdown(
|
|||
.as_ref()
|
||||
.map(|details| details.cache_creation_tokens)
|
||||
.filter(|tokens| *tokens > 0)
|
||||
.unwrap_or_else(|| count(request.usage.extra.get("_cache_creation_input_tokens")));
|
||||
.unwrap_or_else(|| {
|
||||
coerce_token_count(request.usage.extra.get("_cache_creation_input_tokens"))
|
||||
});
|
||||
let reasoning = request
|
||||
.usage
|
||||
.completion_tokens_details
|
||||
.as_ref()
|
||||
.map(|_| parse_completion_tokens_details(request.usage).reasoning_tokens)
|
||||
.filter(|tokens| *tokens > 0)
|
||||
.unwrap_or_else(|| count(request.usage.extra.get("reasoning_tokens")));
|
||||
.unwrap_or_else(|| coerce_token_count(request.usage.extra.get("reasoning_tokens")));
|
||||
let cache_creation_cost = if request.custom_cost_per_token.is_some() {
|
||||
cache_creation as f64 * rates.cache_creation_input_token_cost
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ use crate::realtime_cost::{
|
|||
};
|
||||
use crate::responses_usage::ChatUsage;
|
||||
use crate::speech_cost::count_characters;
|
||||
use crate::together_cost::together_pricing_model;
|
||||
use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind};
|
||||
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||
|
||||
|
|
@ -795,15 +796,17 @@ pub fn completion_cost_from_response(
|
|||
} else {
|
||||
request_model
|
||||
};
|
||||
let pricing_model =
|
||||
together_pricing_model(catalog, model, provider, &prepared.call_type);
|
||||
let call = cost_call(
|
||||
&prepared.call_type,
|
||||
request,
|
||||
model,
|
||||
&pricing_model,
|
||||
token_request_model,
|
||||
&empty_params,
|
||||
)?;
|
||||
let cost_request = ModelCostRequest {
|
||||
model,
|
||||
model: &pricing_model,
|
||||
provider,
|
||||
region,
|
||||
usage,
|
||||
|
|
|
|||
|
|
@ -42,10 +42,7 @@ use crate::speech_cost::{
|
|||
SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost,
|
||||
select_cost_metric_for_model, transcription_usage_has_token_details,
|
||||
};
|
||||
use crate::together_cost::{
|
||||
TogetherThresholds, get_model_params_and_category, has_together_registry_pricing,
|
||||
together_ai_cost_per_token,
|
||||
};
|
||||
use crate::together_cost::together_pricing_model;
|
||||
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||
use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost};
|
||||
use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost};
|
||||
|
|
@ -119,15 +116,6 @@ pub fn cost_per_token_for_call(
|
|||
completion_characters,
|
||||
);
|
||||
}
|
||||
if provider == Some(LlmProviders::TOGETHER_AI)
|
||||
&& matches!(
|
||||
call_type.parse::<crate::call_type::CallTypes>().ok(),
|
||||
Some(crate::call_type::CallTypes::embedding)
|
||||
| Some(crate::call_type::CallTypes::aembedding)
|
||||
)
|
||||
{
|
||||
return together_ai_cost_per_token(catalog, request, call_type);
|
||||
}
|
||||
if provider == Some(LlmProviders::AZURE_AI) {
|
||||
return azure_ai_cost_per_token(catalog, request, request_model);
|
||||
}
|
||||
|
|
@ -231,25 +219,16 @@ pub fn cost_per_token(
|
|||
{
|
||||
return Ok((0.0, cost));
|
||||
}
|
||||
let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI)
|
||||
|| request.model.contains("togethercomputer")
|
||||
|| request.model.contains("together_ai"))
|
||||
&& !has_together_registry_pricing(request.model, catalog.entries());
|
||||
let together_fallback = needs_together_fallback.then(|| {
|
||||
get_model_params_and_category(request.model, "completion", TogetherThresholds::default())
|
||||
});
|
||||
let key = match together_fallback.as_deref() {
|
||||
Some(category) => catalog.select_model_key(category, None, request.region),
|
||||
None => catalog.select_model_key(request.model, request.provider, request.region),
|
||||
}
|
||||
.or_else(|| {
|
||||
(provider == Some(LlmProviders::FIREWORKS_AI))
|
||||
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
|
||||
.and_then(|category| {
|
||||
catalog.select_model_key(category, request.provider, request.region)
|
||||
})
|
||||
})
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let key = catalog
|
||||
.select_model_key(request.model, request.provider, request.region)
|
||||
.or_else(|| {
|
||||
(provider == Some(LlmProviders::FIREWORKS_AI))
|
||||
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
|
||||
.and_then(|category| {
|
||||
catalog.select_model_key(category, request.provider, request.region)
|
||||
})
|
||||
})
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
let model_info =
|
||||
apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider);
|
||||
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
|
||||
|
|
@ -537,7 +516,19 @@ pub fn completion_cost(
|
|||
catalog: &ModelInfoCatalog,
|
||||
request: CompletionCostRequest<'_>,
|
||||
) -> Result<CompletionCost, CostError> {
|
||||
let (prompt, output) = cost_per_token(catalog, request.token)?;
|
||||
let pricing_model = together_pricing_model(
|
||||
catalog,
|
||||
request.token.model,
|
||||
request.token.provider,
|
||||
"completion",
|
||||
);
|
||||
let (prompt, output) = cost_per_token(
|
||||
catalog,
|
||||
ModelCostRequest {
|
||||
model: &pricing_model,
|
||||
..request.token
|
||||
},
|
||||
)?;
|
||||
let built_in_tools = match request.built_in_tools {
|
||||
BuiltInToolCharge::Provided(cost) => cost,
|
||||
BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools(
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use crate::cost_calculator::cost_per_token;
|
||||
use crate::error::CostError;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
use crate::provider::LlmProviders;
|
||||
|
||||
static CHAT_SIZE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d+)b").unwrap());
|
||||
static EMBEDDING_SIZE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d+)m").unwrap());
|
||||
|
||||
|
|
@ -87,22 +88,22 @@ pub fn get_model_params_and_category_embeddings(
|
|||
.to_owned()
|
||||
}
|
||||
|
||||
pub fn together_ai_cost_per_token(
|
||||
pub fn together_pricing_model<'a>(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
model: &'a str,
|
||||
provider: Option<&str>,
|
||||
call_type: &str,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
if has_together_registry_pricing(request.model, catalog.entries()) {
|
||||
return cost_per_token(catalog, request);
|
||||
) -> Cow<'a, str> {
|
||||
let is_together = LlmProviders::TOGETHER_AI.matches(provider)
|
||||
|| model.contains("togethercomputer")
|
||||
|| model.contains("together_ai");
|
||||
if is_together && !has_together_registry_pricing(model, catalog.entries()) {
|
||||
Cow::Owned(get_model_params_and_category(
|
||||
model,
|
||||
call_type,
|
||||
TogetherThresholds::default(),
|
||||
))
|
||||
} else {
|
||||
Cow::Borrowed(model)
|
||||
}
|
||||
let category =
|
||||
get_model_params_and_category(request.model, call_type, TogetherThresholds::default());
|
||||
cost_per_token(
|
||||
catalog,
|
||||
ModelCostRequest {
|
||||
model: &category,
|
||||
provider: Some("together_ai"),
|
||||
..request
|
||||
},
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@ use litellm_cost::billed_token_rates::{
|
|||
};
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::custom_pricing::CustomTokenRates;
|
||||
use litellm_cost::responses_usage::{ChatUsage, CompletionTokenDetails, PromptTokenDetails};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn at(value: &str) -> Timestamp {
|
||||
value.parse().unwrap()
|
||||
|
|
@ -237,3 +238,57 @@ fn token_type_cost_breakdown_is_provider_agnostic_for_perplexity() {
|
|||
assert!((breakdown.reasoning_cost - 400.0 * 4e-6).abs() < 1e-12);
|
||||
assert!((breakdown.cache_read_cost - 0.0).abs() < 1e-12);
|
||||
}
|
||||
|
||||
fn breakdown_rates() -> Value {
|
||||
json!({
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"output_cost_per_reasoning_token": 3e-6
|
||||
})
|
||||
}
|
||||
|
||||
fn usage_with_zero_details(private: (&str, Value)) -> ChatUsage {
|
||||
ChatUsage {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 100,
|
||||
total_tokens: 200,
|
||||
prompt_tokens_details: Some(PromptTokenDetails::default()),
|
||||
completion_tokens_details: Some(CompletionTokenDetails {
|
||||
reasoning_tokens: Some(0),
|
||||
..CompletionTokenDetails::default()
|
||||
}),
|
||||
extra: [(private.0.to_owned(), private.1)].into_iter().collect(),
|
||||
..ChatUsage::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cache_read("_cache_read_input_tokens", json!(40), (0.0, 40.0 * 1e-7, 0.0))]
|
||||
#[case::cache_creation("_cache_creation_input_tokens", json!(40), (0.0, 0.0, 40.0 * 1.25e-6))]
|
||||
#[case::reasoning("reasoning_tokens", json!(40), (40.0 * 3e-6, 0.0, 0.0))]
|
||||
#[case::bool_counts_as_one("reasoning_tokens", json!(true), (3e-6, 0.0, 0.0))]
|
||||
#[case::float_is_not_an_int("reasoning_tokens", json!(40.0), (0.0, 0.0, 0.0))]
|
||||
#[case::negative_counts_nothing("_cache_read_input_tokens", json!(-5), (0.0, 0.0, 0.0))]
|
||||
fn calculate_token_type_cost_breakdown_falls_back_to_private_counters_when_details_are_zero(
|
||||
#[case] counter: &str,
|
||||
#[case] value: Value,
|
||||
#[case] expected: (f64, f64, f64),
|
||||
) {
|
||||
let model_info = breakdown_rates();
|
||||
let usage = usage_with_zero_details((counter, value));
|
||||
let breakdown = calculate_token_type_cost_breakdown(BilledRatesRequest {
|
||||
model_info: Some(&model_info),
|
||||
usage: &usage,
|
||||
provider: Some("openai"),
|
||||
service_tier: None,
|
||||
data_residency: None,
|
||||
vertex_location: None,
|
||||
at: at("2026-01-01T12:00Z"),
|
||||
custom_cost_per_token: None,
|
||||
});
|
||||
assert!((breakdown.reasoning_cost - expected.0).abs() < 1e-15);
|
||||
assert!((breakdown.cache_read_cost - expected.1).abs() < 1e-15);
|
||||
assert!((breakdown.cache_creation_cost - expected.2).abs() < 1e-15);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::collections::HashMap;
|
|||
|
||||
use litellm_cost::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::responses_usage::ChatUsage;
|
||||
use litellm_cost::together_cost::together_pricing_model;
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
@ -111,10 +112,12 @@ fn token_call_routes_together_embedding_and_azure_router() {
|
|||
),
|
||||
]));
|
||||
let usage = usage();
|
||||
let together_model =
|
||||
together_pricing_model(&catalog, "model-200m", Some("together_ai"), "aembedding");
|
||||
assert_eq!(
|
||||
cost_per_token_for_call(
|
||||
&catalog,
|
||||
request("model-200m", Some("together_ai"), &usage),
|
||||
request(&together_model, Some("together_ai"), &usage),
|
||||
token("aembedding", None),
|
||||
)
|
||||
.unwrap(),
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost
|
||||
use litellm_cost::cost_calculator::cost_per_token;
|
||||
use litellm_cost::error::CostError;
|
||||
use litellm_cost::together_cost::together_ai_cost_per_token;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
|
@ -10,7 +9,7 @@ use jiff::Timestamp;
|
|||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::together_cost::{
|
||||
TogetherThresholds, get_model_params_and_category, get_model_params_and_category_embeddings,
|
||||
has_together_registry_pricing,
|
||||
has_together_registry_pricing, together_pricing_model,
|
||||
};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
|
|
@ -33,6 +32,21 @@ fn request<'a>(
|
|||
}
|
||||
}
|
||||
|
||||
fn priced_as_completion_cost(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: ModelCostRequest<'_>,
|
||||
call_type: &str,
|
||||
) -> Result<(f64, f64), CostError> {
|
||||
let pricing_model = together_pricing_model(catalog, request.model, request.provider, call_type);
|
||||
cost_per_token(
|
||||
catalog,
|
||||
ModelCostRequest {
|
||||
model: &pricing_model,
|
||||
..request
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn usage() -> litellm_cost::responses_usage::ChatUsage {
|
||||
get_usage_object(&json!({
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}
|
||||
|
|
@ -132,7 +146,8 @@ fn catalog_uses_exact_price_or_chat_category(
|
|||
),
|
||||
]));
|
||||
let usage = usage();
|
||||
let (prompt, completion) = cost_per_token(&catalog, request("model-7b", &usage)).unwrap();
|
||||
let (prompt, completion) =
|
||||
priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion").unwrap();
|
||||
assert!((prompt - 100.0 * expected_input_rate).abs() < 1e-12);
|
||||
let output_rate = if exact_has_input_rate { 6e-6 } else { 3e-6 };
|
||||
assert!((completion - 20.0 * output_rate).abs() < 1e-12);
|
||||
|
|
@ -146,7 +161,7 @@ fn catalog_routes_embedding_to_size_category() {
|
|||
)]));
|
||||
let usage = usage();
|
||||
let (prompt, completion) =
|
||||
together_ai_cost_per_token(&catalog, request("model-200m", &usage), "aembedding").unwrap();
|
||||
priced_as_completion_cost(&catalog, request("model-200m", &usage), "aembedding").unwrap();
|
||||
assert!((prompt - 100.0 * 4e-8).abs() < 1e-12);
|
||||
assert_eq!(completion, 0.0);
|
||||
}
|
||||
|
|
@ -159,7 +174,84 @@ fn catalog_does_not_price_metadata_only_exact_row_when_category_is_missing() {
|
|||
)]));
|
||||
let usage = usage();
|
||||
assert_eq!(
|
||||
cost_per_token(&catalog, request("model-7b", &usage)),
|
||||
priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion"),
|
||||
Err(CostError::ModelNotFound)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn an_embedding_without_a_megabyte_size_is_not_remapped_as_chat() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"together-ai-4.1b-8b".to_owned(),
|
||||
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}),
|
||||
)]));
|
||||
let usage = usage();
|
||||
assert_eq!(
|
||||
priced_as_completion_cost(&catalog, request("foo-7b-embed", &usage), "embedding"),
|
||||
Err(CostError::ModelNotFound)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn per_second_pricing_is_read_from_the_size_category_not_the_unpriced_model() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([
|
||||
(
|
||||
"together_ai/foo-7b".to_owned(),
|
||||
json!({"input_cost_per_second": 0.01}),
|
||||
),
|
||||
(
|
||||
"together-ai-4.1b-8b".to_owned(),
|
||||
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}),
|
||||
),
|
||||
]));
|
||||
let usage = usage();
|
||||
let (prompt, completion) = priced_as_completion_cost(
|
||||
&catalog,
|
||||
ModelCostRequest {
|
||||
response_time_ms: Some(1000.0),
|
||||
..request("foo-7b", &usage)
|
||||
},
|
||||
"completion",
|
||||
)
|
||||
.unwrap();
|
||||
assert!((prompt - 100.0 * 2e-6).abs() < 1e-12);
|
||||
assert!((completion - 20.0 * 3e-6).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::registry_priced_model_keeps_its_name(Some("together_ai"), "model-7b", true, "model-7b")]
|
||||
#[case::unpriced_together_model_maps_to_size(
|
||||
Some("together_ai"),
|
||||
"model-7b",
|
||||
false,
|
||||
"together-ai-4.1b-8b"
|
||||
)]
|
||||
#[case::together_name_without_provider(
|
||||
None,
|
||||
"togethercomputer/model-7b",
|
||||
false,
|
||||
"together-ai-4.1b-8b"
|
||||
)]
|
||||
#[case::other_provider_keeps_its_name(Some("openai"), "model-7b", false, "model-7b")]
|
||||
fn together_pricing_model_maps_only_unpriced_together_models(
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] model: &str,
|
||||
#[case] registry_priced: bool,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let catalog = ModelInfoCatalog::new(
|
||||
registry_priced
|
||||
.then(|| {
|
||||
(
|
||||
format!("together_ai/{model}"),
|
||||
json!({"input_cost_per_token": 1e-6}),
|
||||
)
|
||||
})
|
||||
.into_iter()
|
||||
.collect(),
|
||||
);
|
||||
assert_eq!(
|
||||
together_pricing_model(&catalog, model, provider, "completion"),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue