diff --git a/litellm-rust/crates/cost/src/billed_token_rates.rs b/litellm-rust/crates/cost/src/billed_token_rates.rs index 456a093b439..10a4da4907a 100644 --- a/litellm-rust/crates/cost/src/billed_token_rates.rs +++ b/litellm-rust/crates/cost/src/billed_token_rates.rs @@ -121,8 +121,12 @@ pub fn calculate_billed_token_rates(request: BilledRatesRequest<'_>) -> Option) -> u64 { - value.and_then(Value::as_u64).unwrap_or(0) +fn coerce_token_count(value: Option<&Value>) -> u64 { + match value { + Some(Value::Bool(true)) => 1, + Some(Value::Number(number)) => number.as_u64().unwrap_or(0), + _ => 0, + } } pub fn calculate_token_type_cost_breakdown( @@ -145,7 +149,7 @@ pub fn calculate_token_type_cost_breakdown( .as_ref() .map(|details| details.cache_hit_tokens) .filter(|tokens| *tokens > 0) - .unwrap_or_else(|| count(request.usage.extra.get("_cache_read_input_tokens"))); + .unwrap_or_else(|| coerce_token_count(request.usage.extra.get("_cache_read_input_tokens"))); let cached_audio = prompt .as_ref() .map_or(0, |details| details.cache_hit_audio_tokens); @@ -153,14 +157,16 @@ pub fn calculate_token_type_cost_breakdown( .as_ref() .map(|details| details.cache_creation_tokens) .filter(|tokens| *tokens > 0) - .unwrap_or_else(|| count(request.usage.extra.get("_cache_creation_input_tokens"))); + .unwrap_or_else(|| { + coerce_token_count(request.usage.extra.get("_cache_creation_input_tokens")) + }); let reasoning = request .usage .completion_tokens_details .as_ref() .map(|_| parse_completion_tokens_details(request.usage).reasoning_tokens) .filter(|tokens| *tokens > 0) - .unwrap_or_else(|| count(request.usage.extra.get("reasoning_tokens"))); + .unwrap_or_else(|| coerce_token_count(request.usage.extra.get("reasoning_tokens"))); let cache_creation_cost = if request.custom_cost_per_token.is_some() { cache_creation as f64 * rates.cache_creation_input_token_cost } else { diff --git a/litellm-rust/crates/cost/src/completion_response.rs b/litellm-rust/crates/cost/src/completion_response.rs index 50442d008a6..3f55e6c457c 100644 --- a/litellm-rust/crates/cost/src/completion_response.rs +++ b/litellm-rust/crates/cost/src/completion_response.rs @@ -31,6 +31,7 @@ use crate::realtime_cost::{ }; use crate::responses_usage::ChatUsage; use crate::speech_cost::count_characters; +use crate::together_cost::together_pricing_model; use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind}; use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; @@ -795,15 +796,17 @@ pub fn completion_cost_from_response( } else { request_model }; + let pricing_model = + together_pricing_model(catalog, model, provider, &prepared.call_type); let call = cost_call( &prepared.call_type, request, - model, + &pricing_model, token_request_model, &empty_params, )?; let cost_request = ModelCostRequest { - model, + model: &pricing_model, provider, region, usage, diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index 20e35c0173d..d704d4da297 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -42,10 +42,7 @@ use crate::speech_cost::{ SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost, select_cost_metric_for_model, transcription_usage_has_token_details, }; -use crate::together_cost::{ - TogetherThresholds, get_model_params_and_category, has_together_registry_pricing, - together_ai_cost_per_token, -}; +use crate::together_cost::together_pricing_model; use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost}; use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost}; @@ -119,15 +116,6 @@ pub fn cost_per_token_for_call( completion_characters, ); } - if provider == Some(LlmProviders::TOGETHER_AI) - && matches!( - call_type.parse::().ok(), - Some(crate::call_type::CallTypes::embedding) - | Some(crate::call_type::CallTypes::aembedding) - ) - { - return together_ai_cost_per_token(catalog, request, call_type); - } if provider == Some(LlmProviders::AZURE_AI) { return azure_ai_cost_per_token(catalog, request, request_model); } @@ -231,25 +219,16 @@ pub fn cost_per_token( { return Ok((0.0, cost)); } - let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI) - || request.model.contains("togethercomputer") - || request.model.contains("together_ai")) - && !has_together_registry_pricing(request.model, catalog.entries()); - let together_fallback = needs_together_fallback.then(|| { - get_model_params_and_category(request.model, "completion", TogetherThresholds::default()) - }); - let key = match together_fallback.as_deref() { - Some(category) => catalog.select_model_key(category, None, request.region), - None => catalog.select_model_key(request.model, request.provider, request.region), - } - .or_else(|| { - (provider == Some(LlmProviders::FIREWORKS_AI)) - .then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default())) - .and_then(|category| { - catalog.select_model_key(category, request.provider, request.region) - }) - }) - .ok_or(CostError::ModelNotFound)?; + let key = catalog + .select_model_key(request.model, request.provider, request.region) + .or_else(|| { + (provider == Some(LlmProviders::FIREWORKS_AI)) + .then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default())) + .and_then(|category| { + catalog.select_model_key(category, request.provider, request.region) + }) + }) + .ok_or(CostError::ModelNotFound)?; let model_info = apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider); if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) { @@ -537,7 +516,19 @@ pub fn completion_cost( catalog: &ModelInfoCatalog, request: CompletionCostRequest<'_>, ) -> Result { - let (prompt, output) = cost_per_token(catalog, request.token)?; + let pricing_model = together_pricing_model( + catalog, + request.token.model, + request.token.provider, + "completion", + ); + let (prompt, output) = cost_per_token( + catalog, + ModelCostRequest { + model: &pricing_model, + ..request.token + }, + )?; let built_in_tools = match request.built_in_tools { BuiltInToolCharge::Provided(cost) => cost, BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools( diff --git a/litellm-rust/crates/cost/src/together_cost.rs b/litellm-rust/crates/cost/src/together_cost.rs index 4f574741ae5..029b80ed5c0 100644 --- a/litellm-rust/crates/cost/src/together_cost.rs +++ b/litellm-rust/crates/cost/src/together_cost.rs @@ -1,12 +1,13 @@ -use crate::catalog::{ModelCostRequest, ModelInfoCatalog}; -use crate::cost_calculator::cost_per_token; -use crate::error::CostError; +use std::borrow::Cow; use std::collections::HashMap; use std::sync::LazyLock; use regex::Regex; use serde_json::Value; +use crate::catalog::ModelInfoCatalog; +use crate::provider::LlmProviders; + static CHAT_SIZE: LazyLock = LazyLock::new(|| Regex::new(r"(\d+)b").unwrap()); static EMBEDDING_SIZE: LazyLock = LazyLock::new(|| Regex::new(r"(\d+)m").unwrap()); @@ -87,22 +88,22 @@ pub fn get_model_params_and_category_embeddings( .to_owned() } -pub fn together_ai_cost_per_token( +pub fn together_pricing_model<'a>( catalog: &ModelInfoCatalog, - request: ModelCostRequest<'_>, + model: &'a str, + provider: Option<&str>, call_type: &str, -) -> Result<(f64, f64), CostError> { - if has_together_registry_pricing(request.model, catalog.entries()) { - return cost_per_token(catalog, request); +) -> Cow<'a, str> { + let is_together = LlmProviders::TOGETHER_AI.matches(provider) + || model.contains("togethercomputer") + || model.contains("together_ai"); + if is_together && !has_together_registry_pricing(model, catalog.entries()) { + Cow::Owned(get_model_params_and_category( + model, + call_type, + TogetherThresholds::default(), + )) + } else { + Cow::Borrowed(model) } - let category = - get_model_params_and_category(request.model, call_type, TogetherThresholds::default()); - cost_per_token( - catalog, - ModelCostRequest { - model: &category, - provider: Some("together_ai"), - ..request - }, - ) } diff --git a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs index da82819976d..524b6766c1b 100644 --- a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs +++ b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs @@ -11,9 +11,10 @@ use litellm_cost::billed_token_rates::{ }; use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; use litellm_cost::custom_pricing::CustomTokenRates; +use litellm_cost::responses_usage::{ChatUsage, CompletionTokenDetails, PromptTokenDetails}; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; -use serde_json::json; +use serde_json::{Value, json}; fn at(value: &str) -> Timestamp { value.parse().unwrap() @@ -237,3 +238,57 @@ fn token_type_cost_breakdown_is_provider_agnostic_for_perplexity() { assert!((breakdown.reasoning_cost - 400.0 * 4e-6).abs() < 1e-12); assert!((breakdown.cache_read_cost - 0.0).abs() < 1e-12); } + +fn breakdown_rates() -> Value { + json!({ + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 1e-7, + "cache_creation_input_token_cost": 1.25e-6, + "output_cost_per_reasoning_token": 3e-6 + }) +} + +fn usage_with_zero_details(private: (&str, Value)) -> ChatUsage { + ChatUsage { + prompt_tokens: 100, + completion_tokens: 100, + total_tokens: 200, + prompt_tokens_details: Some(PromptTokenDetails::default()), + completion_tokens_details: Some(CompletionTokenDetails { + reasoning_tokens: Some(0), + ..CompletionTokenDetails::default() + }), + extra: [(private.0.to_owned(), private.1)].into_iter().collect(), + ..ChatUsage::default() + } +} + +#[rstest] +#[case::cache_read("_cache_read_input_tokens", json!(40), (0.0, 40.0 * 1e-7, 0.0))] +#[case::cache_creation("_cache_creation_input_tokens", json!(40), (0.0, 0.0, 40.0 * 1.25e-6))] +#[case::reasoning("reasoning_tokens", json!(40), (40.0 * 3e-6, 0.0, 0.0))] +#[case::bool_counts_as_one("reasoning_tokens", json!(true), (3e-6, 0.0, 0.0))] +#[case::float_is_not_an_int("reasoning_tokens", json!(40.0), (0.0, 0.0, 0.0))] +#[case::negative_counts_nothing("_cache_read_input_tokens", json!(-5), (0.0, 0.0, 0.0))] +fn calculate_token_type_cost_breakdown_falls_back_to_private_counters_when_details_are_zero( + #[case] counter: &str, + #[case] value: Value, + #[case] expected: (f64, f64, f64), +) { + let model_info = breakdown_rates(); + let usage = usage_with_zero_details((counter, value)); + let breakdown = calculate_token_type_cost_breakdown(BilledRatesRequest { + model_info: Some(&model_info), + usage: &usage, + provider: Some("openai"), + service_tier: None, + data_residency: None, + vertex_location: None, + at: at("2026-01-01T12:00Z"), + custom_cost_per_token: None, + }); + assert!((breakdown.reasoning_cost - expected.0).abs() < 1e-15); + assert!((breakdown.cache_read_cost - expected.1).abs() < 1e-15); + assert!((breakdown.cache_creation_cost - expected.2).abs() < 1e-15); +} diff --git a/litellm-rust/crates/cost/tests/python_call_dispatch.rs b/litellm-rust/crates/cost/tests/python_call_dispatch.rs index f9cfc668c31..915fe705390 100644 --- a/litellm-rust/crates/cost/tests/python_call_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_call_dispatch.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use litellm_cost::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog}; use litellm_cost::responses_usage::ChatUsage; +use litellm_cost::together_cost::together_pricing_model; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::json; @@ -111,10 +112,12 @@ fn token_call_routes_together_embedding_and_azure_router() { ), ])); let usage = usage(); + let together_model = + together_pricing_model(&catalog, "model-200m", Some("together_ai"), "aembedding"); assert_eq!( cost_per_token_for_call( &catalog, - request("model-200m", Some("together_ai"), &usage), + request(&together_model, Some("together_ai"), &usage), token("aembedding", None), ) .unwrap(), diff --git a/litellm-rust/crates/cost/tests/python_together_cost.rs b/litellm-rust/crates/cost/tests/python_together_cost.rs index 2c3c78ec2e3..ef6662fef99 100644 --- a/litellm-rust/crates/cost/tests/python_together_cost.rs +++ b/litellm-rust/crates/cost/tests/python_together_cost.rs @@ -2,7 +2,6 @@ // mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost use litellm_cost::cost_calculator::cost_per_token; use litellm_cost::error::CostError; -use litellm_cost::together_cost::together_ai_cost_per_token; use std::collections::HashMap; @@ -10,7 +9,7 @@ use jiff::Timestamp; use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; use litellm_cost::together_cost::{ TogetherThresholds, get_model_params_and_category, get_model_params_and_category_embeddings, - has_together_registry_pricing, + has_together_registry_pricing, together_pricing_model, }; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; @@ -33,6 +32,21 @@ fn request<'a>( } } +fn priced_as_completion_cost( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + call_type: &str, +) -> Result<(f64, f64), CostError> { + let pricing_model = together_pricing_model(catalog, request.model, request.provider, call_type); + cost_per_token( + catalog, + ModelCostRequest { + model: &pricing_model, + ..request + }, + ) +} + fn usage() -> litellm_cost::responses_usage::ChatUsage { get_usage_object(&json!({ "usage": {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120} @@ -132,7 +146,8 @@ fn catalog_uses_exact_price_or_chat_category( ), ])); let usage = usage(); - let (prompt, completion) = cost_per_token(&catalog, request("model-7b", &usage)).unwrap(); + let (prompt, completion) = + priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion").unwrap(); assert!((prompt - 100.0 * expected_input_rate).abs() < 1e-12); let output_rate = if exact_has_input_rate { 6e-6 } else { 3e-6 }; assert!((completion - 20.0 * output_rate).abs() < 1e-12); @@ -146,7 +161,7 @@ fn catalog_routes_embedding_to_size_category() { )])); let usage = usage(); let (prompt, completion) = - together_ai_cost_per_token(&catalog, request("model-200m", &usage), "aembedding").unwrap(); + priced_as_completion_cost(&catalog, request("model-200m", &usage), "aembedding").unwrap(); assert!((prompt - 100.0 * 4e-8).abs() < 1e-12); assert_eq!(completion, 0.0); } @@ -159,7 +174,84 @@ fn catalog_does_not_price_metadata_only_exact_row_when_category_is_missing() { )])); let usage = usage(); assert_eq!( - cost_per_token(&catalog, request("model-7b", &usage)), + priced_as_completion_cost(&catalog, request("model-7b", &usage), "completion"), Err(CostError::ModelNotFound) ); } + +#[rstest] +fn an_embedding_without_a_megabyte_size_is_not_remapped_as_chat() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "together-ai-4.1b-8b".to_owned(), + json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}), + )])); + let usage = usage(); + assert_eq!( + priced_as_completion_cost(&catalog, request("foo-7b-embed", &usage), "embedding"), + Err(CostError::ModelNotFound) + ); +} + +#[rstest] +fn per_second_pricing_is_read_from_the_size_category_not_the_unpriced_model() { + let catalog = ModelInfoCatalog::new(HashMap::from([ + ( + "together_ai/foo-7b".to_owned(), + json!({"input_cost_per_second": 0.01}), + ), + ( + "together-ai-4.1b-8b".to_owned(), + json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 3e-6}), + ), + ])); + let usage = usage(); + let (prompt, completion) = priced_as_completion_cost( + &catalog, + ModelCostRequest { + response_time_ms: Some(1000.0), + ..request("foo-7b", &usage) + }, + "completion", + ) + .unwrap(); + assert!((prompt - 100.0 * 2e-6).abs() < 1e-12); + assert!((completion - 20.0 * 3e-6).abs() < 1e-12); +} + +#[rstest] +#[case::registry_priced_model_keeps_its_name(Some("together_ai"), "model-7b", true, "model-7b")] +#[case::unpriced_together_model_maps_to_size( + Some("together_ai"), + "model-7b", + false, + "together-ai-4.1b-8b" +)] +#[case::together_name_without_provider( + None, + "togethercomputer/model-7b", + false, + "together-ai-4.1b-8b" +)] +#[case::other_provider_keeps_its_name(Some("openai"), "model-7b", false, "model-7b")] +fn together_pricing_model_maps_only_unpriced_together_models( + #[case] provider: Option<&str>, + #[case] model: &str, + #[case] registry_priced: bool, + #[case] expected: &str, +) { + let catalog = ModelInfoCatalog::new( + registry_priced + .then(|| { + ( + format!("together_ai/{model}"), + json!({"input_cost_per_token": 1e-6}), + ) + }) + .into_iter() + .collect(), + ); + assert_eq!( + together_pricing_model(&catalog, model, provider, "completion"), + expected + ); +}