fix(cost): map Together models to their size category once, before cost_per_token

Python swaps an unpriced Together model for its size category in
completion_cost, with the call type, before cost_per_token runs. Rust did it
inside cost_per_token as chat, so embeddings were mapped twice and per-second
pricing was read from the unpriced model. together_pricing_model now runs in
the completion layer only. The billed-rate breakdown also falls back to the
private cache and reasoning counters with Python's _coerce_token_count rules

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-23 19:31:49 -07:00
parent bf6ea4d781
commit 75170ed344
7 changed files with 216 additions and 65 deletions

View file

@ -121,8 +121,12 @@ pub fn calculate_billed_token_rates(request: BilledRatesRequest<'_>) -> Option<B
)
}
fn count(value: Option<&Value>) -> u64 {
value.and_then(Value::as_u64).unwrap_or(0)
fn coerce_token_count(value: Option<&Value>) -> u64 {
match value {
Some(Value::Bool(true)) => 1,
Some(Value::Number(number)) => number.as_u64().unwrap_or(0),
_ => 0,
}
}
pub fn calculate_token_type_cost_breakdown(
@ -145,7 +149,7 @@ pub fn calculate_token_type_cost_breakdown(
.as_ref()
.map(|details| details.cache_hit_tokens)
.filter(|tokens| *tokens > 0)
.unwrap_or_else(|| count(request.usage.extra.get("_cache_read_input_tokens")));
.unwrap_or_else(|| coerce_token_count(request.usage.extra.get("_cache_read_input_tokens")));
let cached_audio = prompt
.as_ref()
.map_or(0, |details| details.cache_hit_audio_tokens);
@ -153,14 +157,16 @@ pub fn calculate_token_type_cost_breakdown(
.as_ref()
.map(|details| details.cache_creation_tokens)
.filter(|tokens| *tokens > 0)
.unwrap_or_else(|| count(request.usage.extra.get("_cache_creation_input_tokens")));
.unwrap_or_else(|| {
coerce_token_count(request.usage.extra.get("_cache_creation_input_tokens"))
});
let reasoning = request
.usage
.completion_tokens_details
.as_ref()
.map(|_| parse_completion_tokens_details(request.usage).reasoning_tokens)
.filter(|tokens| *tokens > 0)
.unwrap_or_else(|| count(request.usage.extra.get("reasoning_tokens")));
.unwrap_or_else(|| coerce_token_count(request.usage.extra.get("reasoning_tokens")));
let cache_creation_cost = if request.custom_cost_per_token.is_some() {
cache_creation as f64 * rates.cache_creation_input_token_cost
} else {

View file

@ -31,6 +31,7 @@ use crate::realtime_cost::{
};
use crate::responses_usage::ChatUsage;
use crate::speech_cost::count_characters;
use crate::together_cost::together_pricing_model;
use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind};
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
@ -795,15 +796,17 @@ pub fn completion_cost_from_response(
} else {
request_model
};
let pricing_model =
together_pricing_model(catalog, model, provider, &prepared.call_type);
let call = cost_call(
&prepared.call_type,
request,
model,
&pricing_model,
token_request_model,
&empty_params,
)?;
let cost_request = ModelCostRequest {
model,
model: &pricing_model,
provider,
region,
usage,

View file

@ -42,10 +42,7 @@ use crate::speech_cost::{
SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost,
select_cost_metric_for_model, transcription_usage_has_token_details,
};
use crate::together_cost::{
TogetherThresholds, get_model_params_and_category, has_together_registry_pricing,
together_ai_cost_per_token,
};
use crate::together_cost::together_pricing_model;
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost};
use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost};
@ -119,15 +116,6 @@ pub fn cost_per_token_for_call(
completion_characters,
);
}
if provider == Some(LlmProviders::TOGETHER_AI)
&& matches!(
call_type.parse::<crate::call_type::CallTypes>().ok(),
Some(crate::call_type::CallTypes::embedding)
| Some(crate::call_type::CallTypes::aembedding)
)
{
return together_ai_cost_per_token(catalog, request, call_type);
}
if provider == Some(LlmProviders::AZURE_AI) {
return azure_ai_cost_per_token(catalog, request, request_model);
}
@ -231,25 +219,16 @@ pub fn cost_per_token(
{
return Ok((0.0, cost));
}
let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI)
|| request.model.contains("togethercomputer")
|| request.model.contains("together_ai"))
&& !has_together_registry_pricing(request.model, catalog.entries());
let together_fallback = needs_together_fallback.then(|| {
get_model_params_and_category(request.model, "completion", TogetherThresholds::default())
});
let key = match together_fallback.as_deref() {
Some(category) => catalog.select_model_key(category, None, request.region),
None => catalog.select_model_key(request.model, request.provider, request.region),
}
.or_else(|| {
(provider == Some(LlmProviders::FIREWORKS_AI))
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
.and_then(|category| {
catalog.select_model_key(category, request.provider, request.region)
})
})
.ok_or(CostError::ModelNotFound)?;
let key = catalog
.select_model_key(request.model, request.provider, request.region)
.or_else(|| {
(provider == Some(LlmProviders::FIREWORKS_AI))
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
.and_then(|category| {
catalog.select_model_key(category, request.provider, request.region)
})
})
.ok_or(CostError::ModelNotFound)?;
let model_info =
apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider);
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
@ -537,7 +516,19 @@ pub fn completion_cost(
catalog: &ModelInfoCatalog,
request: CompletionCostRequest<'_>,
) -> Result<CompletionCost, CostError> {
let (prompt, output) = cost_per_token(catalog, request.token)?;
let pricing_model = together_pricing_model(
catalog,
request.token.model,
request.token.provider,
"completion",
);
let (prompt, output) = cost_per_token(
catalog,
ModelCostRequest {
model: &pricing_model,
..request.token
},
)?;
let built_in_tools = match request.built_in_tools {
BuiltInToolCharge::Provided(cost) => cost,
BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools(

View file

@ -1,12 +1,13 @@
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
use crate::cost_calculator::cost_per_token;
use crate::error::CostError;
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::LazyLock;
use regex::Regex;
use serde_json::Value;
use crate::catalog::ModelInfoCatalog;
use crate::provider::LlmProviders;
static CHAT_SIZE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d+)b").unwrap());
static EMBEDDING_SIZE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d+)m").unwrap());
@ -87,22 +88,22 @@ pub fn get_model_params_and_category_embeddings(
.to_owned()
}
pub fn together_ai_cost_per_token(
pub fn together_pricing_model<'a>(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
model: &'a str,
provider: Option<&str>,
call_type: &str,
) -> Result<(f64, f64), CostError> {
if has_together_registry_pricing(request.model, catalog.entries()) {
return cost_per_token(catalog, request);
) -> Cow<'a, str> {
let is_together = LlmProviders::TOGETHER_AI.matches(provider)
|| model.contains("togethercomputer")
|| model.contains("together_ai");
if is_together && !has_together_registry_pricing(model, catalog.entries()) {
Cow::Owned(get_model_params_and_category(
model,
call_type,
TogetherThresholds::default(),
))
} else {
Cow::Borrowed(model)
}
let category =
get_model_params_and_category(request.model, call_type, TogetherThresholds::default());
cost_per_token(
catalog,
ModelCostRequest {
model: &category,
provider: Some("together_ai"),
..request
},
)
}

View file

@ -11,9 +11,10 @@ use litellm_cost::billed_token_rates::{
};
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::custom_pricing::CustomTokenRates;
use litellm_cost::responses_usage::{ChatUsage, CompletionTokenDetails, PromptTokenDetails};
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::json;
use serde_json::{Value, json};
fn at(value: &str) -> Timestamp {
value.parse().unwrap()
@ -237,3 +238,57 @@ fn token_type_cost_breakdown_is_provider_agnostic_for_perplexity() {
assert!((breakdown.reasoning_cost - 400.0 * 4e-6).abs() < 1e-12);
assert!((breakdown.cache_read_cost - 0.0).abs() < 1e-12);
}
fn breakdown_rates() -> Value {
json!({
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"cache_read_input_token_cost": 1e-7,
"cache_creation_input_token_cost": 1.25e-6,
"output_cost_per_reasoning_token": 3e-6
})
}
fn usage_with_zero_details(private: (&str, Value)) -> ChatUsage {
ChatUsage {
prompt_tokens: 100,
completion_tokens: 100,
total_tokens: 200,
prompt_tokens_details: Some(PromptTokenDetails::default()),
completion_tokens_details: Some(CompletionTokenDetails {
reasoning_tokens: Some(0),
..CompletionTokenDetails::default()
}),
extra: [(private.0.to_owned(), private.1)].into_iter().collect(),
..ChatUsage::default()
}
}
#[rstest]
#[case::cache_read("_cache_read_input_tokens", json!(40), (0.0, 40.0 * 1e-7, 0.0))]
#[case::cache_creation("_cache_creation_input_tokens", json!(40), (0.0, 0.0, 40.0 * 1.25e-6))]
#[case::reasoning("reasoning_tokens", json!(40), (40.0 * 3e-6, 0.0, 0.0))]
#[case::bool_counts_as_one("reasoning_tokens", json!(true), (3e-6, 0.0, 0.0))]
#[case::float_is_not_an_int("reasoning_tokens", json!(40.0), (0.0, 0.0, 0.0))]
#[case::negative_counts_nothing("_cache_read_input_tokens", json!(-5), (0.0, 0.0, 0.0))]
fn calculate_token_type_cost_breakdown_falls_back_to_private_counters_when_details_are_zero(
#[case] counter: &str,
#[case] value: Value,
#[case] expected: (f64, f64, f64),
) {
let model_info = breakdown_rates();
let usage = usage_with_zero_details((counter, value));
let breakdown = calculate_token_type_cost_breakdown(BilledRatesRequest {
model_info: Some(&model_info),
usage: &usage,
provider: Some("openai"),
service_tier: None,
data_residency: None,
vertex_location: None,
at: at("2026-01-01T12:00Z"),
custom_cost_per_token: None,
});
assert!((breakdown.reasoning_cost - expected.0).abs() < 1e-15);
assert!((breakdown.cache_read_cost - expected.1).abs() < 1e-15);
assert!((breakdown.cache_creation_cost - expected.2).abs() < 1e-15);
}

View file

@ -7,6 +7,7 @@ use std::collections::HashMap;
use litellm_cost::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog};
use litellm_cost::responses_usage::ChatUsage;
use litellm_cost::together_cost::together_pricing_model;
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::json;
@ -111,10 +112,12 @@ fn token_call_routes_together_embedding_and_azure_router() {
),
]));
let usage = usage();
let together_model =
together_pricing_model(&catalog, "model-200m", Some("together_ai"), "aembedding");
assert_eq!(
cost_per_token_for_call(
&catalog,
request("model-200m", Some("together_ai"), &usage),
request(&together_model, Some("together_ai"), &usage),
token("aembedding", None),
)
.unwrap(),

View file

@ -2,7 +2,6 @@
// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost
use litellm_cost::cost_calculator::cost_per_token;
use litellm_cost::error::CostError;
use litellm_cost::together_cost::together_ai_cost_per_token;
use std::collections::HashMap;
@ -10,7 +9,7 @@ use jiff::Timestamp;
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::together_cost::{
TogetherThresholds, get_model_params_and_category, get_model_params_and_category_embeddings,
has_together_registry_pricing,
has_together_registry_pricing, together_pricing_model,
};
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
@ -33,6 +32,21 @@ fn request<'a>(
}
}
fn priced_as_completion_cost(
catalog: &ModelInfoCatalog,
request: ModelCostRequest<'_>,
call_type: &str,
) -> Result<(f64, f64), CostError> {
let pricing_model = together_pricing_model(catalog, request.model, request.provider, call_type);
cost_per_token(
catalog,
ModelCostRequest {
model: &pricing_model,
..request
},
)
}
fn usage() -> litellm_cost::responses_usage::ChatUsage {
get_usage_object(&json!({
"usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}
@ -132,7 +146,8 @@ fn catalog_uses_exact_price_or_chat_category(
),
]));
let usage = usage();
let (prompt, completion) = cost_per_token(&catalog, request("model-7b", &usage)).unwrap();
let (prompt, completion) =
priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion").unwrap();
assert!((prompt - 100.0 * expected_input_rate).abs() < 1e-12);
let output_rate = if exact_has_input_rate { 6e-6 } else { 3e-6 };
assert!((completion - 20.0 * output_rate).abs() < 1e-12);
@ -146,7 +161,7 @@ fn catalog_routes_embedding_to_size_category() {
)]));
let usage = usage();
let (prompt, completion) =
together_ai_cost_per_token(&catalog, request("model-200m", &usage), "aembedding").unwrap();
priced_as_completion_cost(&catalog, request("model-200m", &usage), "aembedding").unwrap();
assert!((prompt - 100.0 * 4e-8).abs() < 1e-12);
assert_eq!(completion, 0.0);
}
@ -159,7 +174,84 @@ fn catalog_does_not_price_metadata_only_exact_row_when_category_is_missing() {
)]));
let usage = usage();
assert_eq!(
cost_per_token(&catalog, request("model-7b", &usage)),
priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion"),
Err(CostError::ModelNotFound)
);
}
#[rstest]
fn an_embedding_without_a_megabyte_size_is_not_remapped_as_chat() {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"together-ai-4.1b-8b".to_owned(),
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}),
)]));
let usage = usage();
assert_eq!(
priced_as_completion_cost(&catalog, request("foo-7b-embed", &usage), "embedding"),
Err(CostError::ModelNotFound)
);
}
#[rstest]
fn per_second_pricing_is_read_from_the_size_category_not_the_unpriced_model() {
let catalog = ModelInfoCatalog::new(HashMap::from([
(
"together_ai/foo-7b".to_owned(),
json!({"input_cost_per_second": 0.01}),
),
(
"together-ai-4.1b-8b".to_owned(),
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}),
),
]));
let usage = usage();
let (prompt, completion) = priced_as_completion_cost(
&catalog,
ModelCostRequest {
response_time_ms: Some(1000.0),
..request("foo-7b", &usage)
},
"completion",
)
.unwrap();
assert!((prompt - 100.0 * 2e-6).abs() < 1e-12);
assert!((completion - 20.0 * 3e-6).abs() < 1e-12);
}
#[rstest]
#[case::registry_priced_model_keeps_its_name(Some("together_ai"), "model-7b", true, "model-7b")]
#[case::unpriced_together_model_maps_to_size(
Some("together_ai"),
"model-7b",
false,
"together-ai-4.1b-8b"
)]
#[case::together_name_without_provider(
None,
"togethercomputer/model-7b",
false,
"together-ai-4.1b-8b"
)]
#[case::other_provider_keeps_its_name(Some("openai"), "model-7b", false, "model-7b")]
fn together_pricing_model_maps_only_unpriced_together_models(
#[case] provider: Option<&str>,
#[case] model: &str,
#[case] registry_priced: bool,
#[case] expected: &str,
) {
let catalog = ModelInfoCatalog::new(
registry_priced
.then(|| {
(
format!("together_ai/{model}"),
json!({"input_cost_per_token": 1e-6}),
)
})
.into_iter()
.collect(),
);
assert_eq!(
together_pricing_model(&catalog, model, provider, "completion"),
expected
);
}