diff --git a/litellm-rust/crates/cost/src/base_rate_selection.rs b/litellm-rust/crates/cost/src/base_rate_selection.rs index 9d3fd82b3cd..17148344084 100644 --- a/litellm-rust/crates/cost/src/base_rate_selection.rs +++ b/litellm-rust/crates/cost/src/base_rate_selection.rs @@ -95,8 +95,17 @@ fn tiered_base_rates(model_info: &Value, prompt_tokens: u64) -> Option Option<&str> { + let above = key.split_once("_above_")?.1; + Some( + above + .split_once("_tokens") + .map_or(above, |(threshold, _)| threshold), + ) +} + fn threshold_number(key: &str) -> Option { - let threshold = key.split_once("_above_")?.1.split_once("_tokens")?.0; + let threshold = threshold_text(key)?; let number = threshold.replace('k', "").parse::().ok()?; Some(number * if threshold.contains('k') { 1000.0 } else { 1.0 }) } @@ -173,19 +182,17 @@ fn select_base_rates( ); let selected = crossed_threshold(model_info, prompt_tokens, threshold_inclusive).map(|(key, _)| { - let threshold = key - .split_once("_above_") - .unwrap() - .1 - .split_once("_tokens") - .unwrap() - .0; + let threshold = threshold_text(key).unwrap_or_default(); let rate_key = |prefix: &str| { tier_key(&format!("{prefix}_above_{threshold}_tokens"), service_tier) }; + let input_key = if service_tier.is_some_and(|tier| !tier.is_empty()) { + rate_key("input_cost_per_token") + } else { + key.to_owned() + }; ( - get_cost_per_unit(model_info, &rate_key("input_cost_per_token"), Some(input)) - .unwrap_or(input), + get_cost_per_unit(model_info, &input_key, Some(input)).unwrap_or(input), get_cost_per_unit(model_info, &rate_key("output_cost_per_token"), Some(output)) .unwrap_or(output), get_cost_per_unit( diff --git a/litellm-rust/crates/cost/tests/python_base_rate_selection.rs b/litellm-rust/crates/cost/tests/python_base_rate_selection.rs index 07ea046de87..5d961177f6d 100644 --- a/litellm-rust/crates/cost/tests/python_base_rate_selection.rs +++ b/litellm-rust/crates/cost/tests/python_base_rate_selection.rs @@ -180,3 +180,29 @@ fn inclusive_threshold_providers_stay_pinned_to_the_python_set() { assert!(!uses_inclusive_token_thresholds(provider)); } } + +#[rstest] +#[case::threshold_key_without_tokens_suffix(json!({"input_cost_per_token_above_50": 3e-6}), None, 3e-6, 4e-6)] +#[case::tiered_request_builds_the_tokens_key(json!({"input_cost_per_token_above_50": 3e-6, "input_cost_per_token_above_50_tokens_priority": 5e-6}), Some("priority"), 5e-6, 4e-6)] +#[case::unparseable_threshold_is_ignored(json!({"input_cost_per_token_above_abc_tokens": 3e-6}), None, 2e-6, 4e-6)] +fn get_token_base_cost_reads_threshold_keys_like_python_split( + #[case] extra: Value, + #[case] service_tier: Option<&str>, + #[case] expected_input: f64, + #[case] expected_output: f64, +) { + let model_info = Value::Object( + json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6}) + .as_object() + .unwrap() + .clone() + .into_iter() + .chain(extra.as_object().unwrap().clone()) + .collect(), + ); + let rates = get_token_base_cost_without_off_peak(&model_info, 100, service_tier, false); + assert_eq!( + (rates.input, rates.output), + (expected_input, expected_output) + ); +}