diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index ed0cf7ee4da..5301cbabd47 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -54,6 +54,7 @@ pub mod tool_cost_dispatch; pub mod transcription_usage; pub mod usage_dispatch; pub mod vertex_cost; +pub mod wire; pub mod xai_cost; pub mod zero_cost_diagnostic; diff --git a/litellm-rust/crates/cost/src/wire.rs b/litellm-rust/crates/cost/src/wire.rs new file mode 100644 index 00000000000..dda30be28df --- /dev/null +++ b/litellm-rust/crates/cost/src/wire.rs @@ -0,0 +1,630 @@ +//! The only module allowed to handle serde_json::Value: it deserializes the +//! wire contract Python sends and exposes the embedded pricing catalog. The +//! JSON shapes are pinned by the wire_usage and wire_response surfaces of +//! tests/generate_python_fixtures.py. + +use std::collections::BTreeMap; +use std::sync::LazyLock; + +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::responses_usage::{ + CacheCreationTokenDetails, CompletionTokenDetails, PromptTokenDetails, +}; + +static EMBEDDED_CATALOG_JSON: &str = + include_str!("../../../../model_prices_and_context_window.json"); + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct RawCatalogEntry(Map); + +impl RawCatalogEntry { + pub fn fields(&self) -> &Map { + &self.0 + } + + fn merge(&mut self, overlay: RawCatalogEntry) { + for (key, value) in overlay.0 { + self.0.insert(key, value); + } + } +} + +pub static EMBEDDED_CATALOG: LazyLock> = LazyLock::new(|| { + let root: Map = + serde_json::from_str(EMBEDDED_CATALOG_JSON).expect("embedded catalog JSON parses"); + root.into_iter() + .filter(|(key, _)| key != "fallback_generalizations") + .filter_map(|(key, value)| match value { + Value::Object(entry) => Some((key, RawCatalogEntry(entry))), + _ => None, + }) + .collect() +}); + +pub fn overlay_catalog( + base: &BTreeMap, + overlays: impl IntoIterator, +) -> BTreeMap { + let mut catalog: BTreeMap<_, _> = base + .iter() + .map(|(key, entry)| (key.clone(), entry.clone())) + .collect(); + for (key, overlay) in overlays { + match catalog.get_mut(&key) { + Some(existing) => existing.merge(overlay), + None => { + catalog.insert(key, overlay); + } + } + } + catalog +} + +pub fn catalog_with_overlays( + overlays: impl IntoIterator, +) -> BTreeMap { + overlay_catalog(&EMBEDDED_CATALOG, overlays) +} + +fn count_value(value: &Value) -> Option { + match value { + Value::Null => Some(0), + Value::Number(number) => number.as_u64().or_else(|| { + number + .as_f64() + .filter(|value| value.is_finite() && *value >= 0.0 && *value < u64::MAX as f64) + .map(|value| value as u64) + }), + Value::Bool(flag) => Some(u64::from(*flag)), + Value::String(text) => text.trim().parse().ok(), + _ => None, + } +} + +fn lenient_count<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + count_value(&value).ok_or_else(|| serde::de::Error::custom("invalid token count")) +} + +fn lenient_count_option<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + match value { + None | Some(Value::Null) => Ok(None), + Some(value) => count_value(&value) + .map(Some) + .ok_or_else(|| serde::de::Error::custom("invalid token count")), + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum WireNumber { + Number(f64), + Text(String), + Flag(bool), +} + +impl WireNumber { + pub fn cost(self) -> Option { + match self { + Self::Number(value) => Some(value), + Self::Text(text) => text.trim().parse().ok(), + Self::Flag(flag) => Some(f64::from(flag)), + } + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum ReportedCostInput { + Number(f64), + Detailed { total_cost: Option }, +} + +impl ReportedCostInput { + pub fn cost(self) -> Option { + match self { + Self::Number(value) => Some(value), + Self::Detailed { total_cost } => total_cost, + } + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ServerToolUse { + #[serde(default, deserialize_with = "lenient_count_option")] + pub web_search_requests: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub tool_search_requests: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub browser_open_requests: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ServerSideToolUsageDetails { + #[serde(default, deserialize_with = "lenient_count_option")] + pub web_search_calls: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct ChatUsageInput { + #[serde(default, deserialize_with = "lenient_count")] + pub prompt_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub completion_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_tokens: u64, + pub cost: Option, + pub prompt_tokens_details: Option, + pub completion_tokens_details: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub reasoning_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_read_input_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_creation_input_tokens: Option, + pub server_tool_use: Option, + pub speed: Option, + pub inference_geo: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub _cache_read_input_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub _cache_creation_input_tokens: Option, + pub server_side_tool_usage_details: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub citation_tokens: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ResponsesUsageInput { + #[serde(default, deserialize_with = "lenient_count")] + pub input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub output_tokens: u64, + #[serde(default)] + pub input_tokens_details: Option, + #[serde(default)] + pub input_token_details: Option, + #[serde(default)] + pub output_tokens_details: Option, + #[serde(default)] + pub output_token_details: Option, + pub cost: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ThinkingTokenDetails { + #[serde(default, deserialize_with = "lenient_count_option")] + pub thinking_tokens: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct AnthropicIteration { + #[serde(default, deserialize_with = "lenient_count")] + pub input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub output_tokens: u64, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_read_input_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_creation_input_tokens: Option, + pub cache_creation: Option, + pub output_tokens_details: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct AnthropicUsageInput { + #[serde(default, deserialize_with = "lenient_count")] + pub input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub output_tokens: u64, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_read_input_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub cache_creation_input_tokens: Option, + pub cache_creation: Option, + pub output_tokens_details: Option, + pub iterations: Option>, + pub server_tool_use: Option, + pub speed: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ModalityTokens { + #[serde(default, deserialize_with = "lenient_count")] + pub tokens: u64, + pub modality: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct GroundingToolCount { + #[serde(rename = "type")] + pub kind: Option, + #[serde(default, deserialize_with = "lenient_count")] + pub count: u64, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct InteractionsUsageInput { + #[serde(default, deserialize_with = "lenient_count")] + pub total_input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_output_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_tool_use_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_cached_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_reasoning_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_thought_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_tokens: u64, + #[serde(default)] + pub input_tokens_by_modality: Vec, + #[serde(default)] + pub tool_use_tokens_by_modality: Vec, + #[serde(default)] + pub cached_tokens_by_modality: Vec, + #[serde(default)] + pub output_tokens_by_modality: Vec, + #[serde(default)] + pub grounding_tool_count: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq)] +pub struct TranscriptionTokenDetails { + #[serde(default, deserialize_with = "lenient_count")] + pub text_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub audio_tokens: u64, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum TranscriptionUsageInput { + Duration { + seconds: f64, + }, + Tokens { + #[serde(default, deserialize_with = "lenient_count")] + input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + output_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + total_tokens: u64, + #[serde(default)] + input_token_details: TranscriptionTokenDetails, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(tag = "format", rename_all = "snake_case")] +pub enum UsageInput { + Chat { + #[serde(flatten)] + usage: ChatUsageInput, + }, + Responses { + #[serde(flatten)] + usage: ResponsesUsageInput, + }, + Anthropic { + #[serde(flatten)] + usage: AnthropicUsageInput, + }, + Interactions { + #[serde(flatten)] + usage: InteractionsUsageInput, + }, + Transcription { + #[serde(flatten)] + usage: TranscriptionUsageInput, + }, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct BilledUnits { + #[serde(default, deserialize_with = "lenient_count_option")] + pub total_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub search_units: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ResponseMeta { + pub billed_units: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct RealtimeResponse { + pub usage: Option, + pub service_tier: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct RealtimeResultInput { + pub response: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ResponseInput { + pub model: Option, + pub usage: Option, + pub created: Option, + pub ended: Option, + #[serde(rename = "_response_ms")] + pub response_ms: Option, + pub provider_reported_cost_usd: Option, + pub duration: Option, + pub meta: Option, + #[serde(default)] + pub results: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ProviderSpecificFields { + pub traffic_type: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct HiddenParamsInput { + pub custom_llm_provider: Option, + pub region_name: Option, + pub model: Option, + pub litellm_model_name: Option, + pub additional_headers: Option>, + pub audio_transcription_duration: Option, + pub provider_specific_fields: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct QueryCount(pub u64); + +impl QueryCount { + pub fn get(self) -> u64 { + self.0 + } +} + +impl<'de> Deserialize<'de> for QueryCount { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let count = value.as_array().map_or(1, Vec::len) as u64; + Ok(Self(count)) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct OptionalParamsInput { + pub service_tier: Option, + pub query: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ImageInputTokenDetails { + #[serde(default, deserialize_with = "lenient_count_option")] + pub text_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub image_tokens: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ImageOutputTokenDetails { + #[serde(default, deserialize_with = "lenient_count_option")] + pub text_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub image_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub audio_tokens: Option, + #[serde(default, deserialize_with = "lenient_count_option")] + pub reasoning_tokens: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ImageUsageInput { + #[serde(default, deserialize_with = "lenient_count")] + pub input_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub output_tokens: u64, + #[serde(default, deserialize_with = "lenient_count")] + pub total_tokens: u64, + #[serde(default)] + pub input_tokens_details: Option, + #[serde(default)] + pub completion_tokens_details: Option, + #[serde(default)] + pub output_tokens_details: Option, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct DataLen(pub u64); + +impl DataLen { + pub fn get(self) -> u64 { + self.0 + } +} + +impl<'de> Deserialize<'de> for DataLen { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + let count = value.as_array().map_or(0, Vec::len) as u64; + Ok(Self(count)) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +pub struct ImageResponseInput { + pub usage: Option, + #[serde(default)] + pub data: DataLen, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(fields: &[(&str, Value)]) -> RawCatalogEntry { + RawCatalogEntry(Map::from_iter( + fields + .iter() + .map(|(key, value)| (key.to_string(), value.clone())), + )) + } + + #[test] + fn overlay_merges_fields_into_existing_entries_and_adds_new_ones() { + let base = BTreeMap::from([( + "synthetic-base".to_string(), + entry(&[ + ("input_cost_per_token", serde_json::json!(3e-6)), + ("output_cost_per_token", serde_json::json!(15e-6)), + ]), + )]); + let overlaid = overlay_catalog( + &base, + [ + ( + "synthetic-base".to_string(), + entry(&[("output_cost_per_token", serde_json::json!(9e-6))]), + ), + ( + "synthetic-new".to_string(), + entry(&[("input_cost_per_token", serde_json::json!(1e-6))]), + ), + ], + ); + assert_eq!(overlaid.len(), 2); + let merged = &overlaid["synthetic-base"]; + assert_eq!( + merged.fields().get("input_cost_per_token"), + Some(&serde_json::json!(3e-6)) + ); + assert_eq!( + merged.fields().get("output_cost_per_token"), + Some(&serde_json::json!(9e-6)) + ); + assert!( + overlaid["synthetic-new"] + .fields() + .contains_key("input_cost_per_token") + ); + } + + #[test] + fn embedded_catalog_excludes_generalizations_meta_key() { + let catalog = LazyLock::force(&EMBEDDED_CATALOG); + assert!(!catalog.is_empty()); + assert!(!catalog.contains_key("fallback_generalizations")); + } + + #[test] + fn chat_usage_reads_lenient_counts_and_cost_projection() { + let usage: ChatUsageInput = serde_json::from_value(serde_json::json!({ + "prompt_tokens": true, + "completion_tokens": " 42 ", + "total_tokens": 58.0, + "cost": {"total_cost": 0.5}, + "cache_read_input_tokens": 7, + })) + .unwrap(); + assert_eq!(usage.prompt_tokens, 1); + assert_eq!(usage.completion_tokens, 42); + assert_eq!(usage.total_tokens, 58); + assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.5))); + assert_eq!(usage.cache_read_input_tokens, Some(7)); + } + + #[test] + fn chat_usage_rejects_negative_and_garbage_counts() { + assert!( + serde_json::from_value::(serde_json::json!({ + "prompt_tokens": -1 + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "prompt_tokens": "not a number" + })) + .is_err() + ); + } + + #[test] + fn usage_input_dispatches_on_format_tag_not_key_shapes() { + let chat: UsageInput = serde_json::from_value(serde_json::json!({ + "format": "chat", + "prompt_tokens": 10, + "completion_tokens": 5, + "input_tokens": 999, + "output_tokens": 99, + })) + .unwrap(); + let UsageInput::Chat { usage } = chat else { + panic!("chat tag must select the chat payload"); + }; + assert_eq!(usage.prompt_tokens, 10); + let transcription: UsageInput = serde_json::from_value(serde_json::json!({ + "format": "transcription", + "type": "tokens", + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "input_token_details": {"text_tokens": 60, "audio_tokens": 40}, + })) + .unwrap(); + let UsageInput::Transcription { usage } = transcription else { + panic!("transcription tag must select the transcription payload"); + }; + let TranscriptionUsageInput::Tokens { + input_tokens, + output_tokens, + .. + } = usage + else { + panic!("transcription type tag must select the tokens variant"); + }; + assert_eq!((input_tokens, output_tokens), (100, 200)); + } + + #[test] + fn hidden_params_read_wire_number_header_costs() { + let hidden: HiddenParamsInput = serde_json::from_value(serde_json::json!({ + "custom_llm_provider": "openai", + "additional_headers": {"llm_provider-x-litellm-response-cost": " 0.0123 "}, + })) + .unwrap(); + let header = hidden + .additional_headers + .as_ref() + .and_then(|headers| headers.get("llm_provider-x-litellm-response-cost")) + .cloned(); + assert_eq!(header.and_then(WireNumber::cost), Some(0.0123)); + } + + #[test] + fn query_count_counts_arrays_and_defaults_non_arrays_to_one() { + let params: OptionalParamsInput = + serde_json::from_value(serde_json::json!({"query": ["a", "b", "c"]})).unwrap(); + assert_eq!(params.query.map(QueryCount::get), Some(3)); + let single: OptionalParamsInput = + serde_json::from_value(serde_json::json!({"query": "galaxy"})).unwrap(); + assert_eq!(single.query.map(QueryCount::get), Some(1)); + } +} diff --git a/litellm-rust/crates/cost/tests/generate_python_fixtures.py b/litellm-rust/crates/cost/tests/generate_python_fixtures.py index 124c21c1cc6..013081e04a5 100644 --- a/litellm-rust/crates/cost/tests/generate_python_fixtures.py +++ b/litellm-rust/crates/cost/tests/generate_python_fixtures.py @@ -22,11 +22,255 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( ) from litellm.responses.utils import ResponseAPILoggingUtils from litellm.utils import register_model -from litellm.types.utils import Usage +from litellm.types.utils import Choices, Message, ModelResponse, Usage +from openai.types.images_response import ImagesResponse UTC = timezone.utc +def private_cache_attrs(usage: Usage) -> dict[str, int]: + return { + "_cache_read_input_tokens": usage._cache_read_input_tokens, + "_cache_creation_input_tokens": usage._cache_creation_input_tokens, + } + + +def chat_usage_payload(usage: Usage) -> dict[str, Any]: + return {"format": "chat", **usage.model_dump(), **private_cache_attrs(usage)} + + +def tagged_response_payload(response: ModelResponse) -> dict[str, Any]: + dump = response.model_dump() + if dump.get("usage") is not None: + dump["usage"] = {"format": "chat", **dump["usage"], **private_cache_attrs(response.usage)} + return dump + + +WIRE_USAGE_CASES: tuple[tuple[str, str, dict[str, Any]], ...] = ( + ( + "chat_full_projection", + "chat", + chat_usage_payload( + Usage( + prompt_tokens=1_000, + completion_tokens=500, + reasoning_tokens=120, + prompt_tokens_details={ + "cached_tokens": 800, + "cache_write_tokens": 300, + "web_search_requests": 2, + "cache_creation_token_details": { + "ephemeral_5m_input_tokens": 200, + "ephemeral_1h_input_tokens": 100, + }, + }, + completion_tokens_details={"reasoning_tokens": 120, "text_tokens": 380}, + server_tool_use={"web_search_requests": 3}, + cost=0.75, + cache_read_input_tokens=800, + **{ + "inference_geo": "us", + "speed": "fast", + "citation_tokens": 1_500, + "server_side_tool_usage_details": {"web_search_calls": 4}, + }, + ) + ), + ), + ( + "chat_minimal", + "chat", + chat_usage_payload(Usage(prompt_tokens=10, completion_tokens=5)), + ), + ( + "chat_gateway_merged_input_token_keys", + "chat", + chat_usage_payload(Usage(prompt_tokens=100, completion_tokens=50, input_tokens=100, output_tokens=50)), + ), + ( + "responses_singular_details", + "responses", + { + "input_tokens": 1_200, + "output_tokens": 300, + "input_token_details": {"cached_tokens": 800, "web_search_requests": 2}, + "output_token_details": {"reasoning_tokens": 100}, + }, + ), + ( + "responses_plural_details", + "responses", + { + "input_tokens": 900, + "output_tokens": 250, + "input_tokens_details": {"text_tokens": 700, "cached_tokens": 200}, + "output_tokens_details": {"text_tokens": 150, "audio_tokens": 100}, + "cost": {"total_cost": 0.25}, + }, + ), + ( + "anthropic_full", + "anthropic", + { + "input_tokens": 5_000, + "output_tokens": 200, + "cache_read_input_tokens": 3_000, + "cache_creation_input_tokens": 2_000, + "cache_creation": {"ephemeral_5m_input_tokens": 1_500, "ephemeral_1h_input_tokens": 500}, + "server_tool_use": {"web_search_requests": 2}, + "speed": "fast", + }, + ), + ( + "anthropic_iterations", + "anthropic", + { + "input_tokens": 600, + "output_tokens": 300, + "iterations": [ + { + "input_tokens": 100, + "output_tokens": 150, + "cache_read_input_tokens": 20, + "cache_creation_input_tokens": 10, + "cache_creation": {"ephemeral_5m_input_tokens": 10}, + "output_tokens_details": {"thinking_tokens": 90}, + }, + { + "input_tokens": 200, + "output_tokens": 150, + "output_tokens_details": {"thinking_tokens": 60}, + }, + ], + }, + ), + ( + "interactions_modality_split", + "interactions", + { + "total_input_tokens": 1_000, + "total_output_tokens": 300, + "total_tool_use_tokens": 50, + "total_cached_tokens": 400, + "total_reasoning_tokens": 120, + "total_tokens": 1_470, + "input_tokens_by_modality": [ + {"tokens": 600, "modality": "TEXT"}, + {"tokens": 50, "modality": "AUDIO"}, + ], + "cached_tokens_by_modality": [{"tokens": 400, "modality": "TEXT"}], + "output_tokens_by_modality": [{"tokens": 180, "modality": "TEXT"}], + "grounding_tool_count": [{"type": "google_search", "count": 3}], + }, + ), + ( + "transcription_tokens", + "transcription", + { + "type": "tokens", + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "input_token_details": {"text_tokens": 60, "audio_tokens": 40}, + }, + ), + ( + "transcription_duration", + "transcription", + {"type": "duration", "seconds": 12.5}, + ), +) + + +def wire_usage_surface() -> list[dict[str, Any]]: + return [{"id": name, "payload": {"format": declared, **payload}} for name, declared, payload in WIRE_USAGE_CASES] + + +def wire_response_surface() -> list[dict[str, Any]]: + usage = Usage( + prompt_tokens=2_000, + completion_tokens=400, + prompt_tokens_details={"cached_tokens": 1_500, "cache_write_tokens": 100}, + server_tool_use={"web_search_requests": 1}, + cost=0.5, + ) + usage.inference_geo = "us" + chat_response = ModelResponse( + id="chatcmpl-wire", + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="ok", role="assistant"), + ) + ], + created=1_774_000_000, + model="synthetic-chat", + usage=usage, + _response_ms=1_234.5, + ) + chat_response.ended = 1_774_000_012.5 + image_response = ImagesResponse( + created=1_774_000_000, + data=[ + {"url": "https://synthetic/1", "revised_prompt": None}, + {"url": "https://synthetic/2", "revised_prompt": None}, + ], + ) + return [ + { + "id": "chat_response_full", + "response": tagged_response_payload(chat_response), + "hidden_params": { + "custom_llm_provider": "openai", + "region_name": "us-west-2", + "model": "synthetic-chat", + "litellm_model_name": "synthetic-chat-alias", + "additional_headers": {"llm_provider-x-litellm-response-cost": "0.0123"}, + "provider_specific_fields": {"traffic_type": "batch"}, + }, + "optional_params": {"service_tier": "priority", "query": ["synthetic-a", "synthetic-b"]}, + }, + { + "id": "realtime_results", + "response": { + "results": [ + { + "type": "response.completed", + "response": { + "usage": { + "format": "responses", + "input_tokens": 500, + "output_tokens": 120, + "input_token_details": {"cached_tokens": 100}, + }, + "service_tier": "priority", + }, + } + ] + }, + }, + { + "id": "image_response_usage", + "response": { + **image_response.model_dump(), + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "input_tokens_details": {"text_tokens": 40, "image_tokens": 60}, + "output_tokens_details": { + "text_tokens": 10, + "image_tokens": 40, + "reasoning_tokens": 0, + "audio_tokens": 0, + }, + }, + }, + }, + ] + + def details( cached: int | None = None, cache_write: int | None = None, @@ -824,6 +1068,8 @@ def main() -> None: "responses_usage": responses_usage_surface(), "caching_savings": caching_savings_surface(), "anthropic_usage": anthropic_usage_surface(), + "wire_usage": wire_usage_surface(), + "wire_response": wire_response_surface(), } target = Path(__file__).parent / "python_fixtures.json" target.write_text(json.dumps(fixture, indent=1) + "\n") diff --git a/litellm-rust/crates/cost/tests/python_fixtures.json b/litellm-rust/crates/cost/tests/python_fixtures.json index 4268cf4219c..7e4e6e5df12 100644 --- a/litellm-rust/crates/cost/tests/python_fixtures.json +++ b/litellm-rust/crates/cost/tests/python_fixtures.json @@ -1,5 +1,5 @@ { - "python_commit": "b401bc9be2509e46fc0126087db2542b129ad56c", + "python_commit": "492ac2e9c43ed8fc9ae18650f6d41987ff7ae0fc", "generic": [ { "id": "audio_mixed_modalities", @@ -1774,5 +1774,353 @@ "cache_read_input_tokens": 0 } } + ], + "wire_usage": [ + { + "id": "chat_full_projection", + "payload": { + "format": "chat", + "completion_tokens": 500, + "prompt_tokens": 1000, + "total_tokens": 0, + "completion_tokens_details": { + "accepted_prediction_tokens": null, + "audio_tokens": null, + "reasoning_tokens": 120, + "rejected_prediction_tokens": null, + "text_tokens": 380, + "image_tokens": null, + "video_tokens": null + }, + "prompt_tokens_details": { + "audio_tokens": null, + "cached_tokens": 800, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null, + "web_search_requests": 2, + "cache_write_tokens": 300, + "cache_creation_tokens": 300, + "cache_creation_token_details": { + "ephemeral_5m_input_tokens": 200, + "ephemeral_1h_input_tokens": 100 + } + }, + "server_tool_use": { + "web_search_requests": 3, + "tool_search_requests": null, + "browser_open_requests": null + }, + "cost": 0.75, + "cache_read_input_tokens": 800, + "inference_geo": "us", + "speed": "fast", + "citation_tokens": 1500, + "server_side_tool_usage_details": { + "web_search_calls": 4 + }, + "_cache_read_input_tokens": 800, + "_cache_creation_input_tokens": 0 + } + }, + { + "id": "chat_minimal", + "payload": { + "format": "chat", + "completion_tokens": 5, + "prompt_tokens": 10, + "total_tokens": 0, + "completion_tokens_details": null, + "prompt_tokens_details": null, + "_cache_read_input_tokens": 0, + "_cache_creation_input_tokens": 0 + } + }, + { + "id": "chat_gateway_merged_input_token_keys", + "payload": { + "format": "chat", + "completion_tokens": 50, + "prompt_tokens": 100, + "total_tokens": 0, + "completion_tokens_details": null, + "prompt_tokens_details": null, + "input_tokens": 100, + "output_tokens": 50, + "_cache_read_input_tokens": 0, + "_cache_creation_input_tokens": 0 + } + }, + { + "id": "responses_singular_details", + "payload": { + "format": "responses", + "input_tokens": 1200, + "output_tokens": 300, + "input_token_details": { + "cached_tokens": 800, + "web_search_requests": 2 + }, + "output_token_details": { + "reasoning_tokens": 100 + } + } + }, + { + "id": "responses_plural_details", + "payload": { + "format": "responses", + "input_tokens": 900, + "output_tokens": 250, + "input_tokens_details": { + "text_tokens": 700, + "cached_tokens": 200 + }, + "output_tokens_details": { + "text_tokens": 150, + "audio_tokens": 100 + }, + "cost": { + "total_cost": 0.25 + } + } + }, + { + "id": "anthropic_full", + "payload": { + "format": "anthropic", + "input_tokens": 5000, + "output_tokens": 200, + "cache_read_input_tokens": 3000, + "cache_creation_input_tokens": 2000, + "cache_creation": { + "ephemeral_5m_input_tokens": 1500, + "ephemeral_1h_input_tokens": 500 + }, + "server_tool_use": { + "web_search_requests": 2 + }, + "speed": "fast" + } + }, + { + "id": "anthropic_iterations", + "payload": { + "format": "anthropic", + "input_tokens": 600, + "output_tokens": 300, + "iterations": [ + { + "input_tokens": 100, + "output_tokens": 150, + "cache_read_input_tokens": 20, + "cache_creation_input_tokens": 10, + "cache_creation": { + "ephemeral_5m_input_tokens": 10 + }, + "output_tokens_details": { + "thinking_tokens": 90 + } + }, + { + "input_tokens": 200, + "output_tokens": 150, + "output_tokens_details": { + "thinking_tokens": 60 + } + } + ] + } + }, + { + "id": "interactions_modality_split", + "payload": { + "format": "interactions", + "total_input_tokens": 1000, + "total_output_tokens": 300, + "total_tool_use_tokens": 50, + "total_cached_tokens": 400, + "total_reasoning_tokens": 120, + "total_tokens": 1470, + "input_tokens_by_modality": [ + { + "tokens": 600, + "modality": "TEXT" + }, + { + "tokens": 50, + "modality": "AUDIO" + } + ], + "cached_tokens_by_modality": [ + { + "tokens": 400, + "modality": "TEXT" + } + ], + "output_tokens_by_modality": [ + { + "tokens": 180, + "modality": "TEXT" + } + ], + "grounding_tool_count": [ + { + "type": "google_search", + "count": 3 + } + ] + } + }, + { + "id": "transcription_tokens", + "payload": { + "format": "transcription", + "type": "tokens", + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "input_token_details": { + "text_tokens": 60, + "audio_tokens": 40 + } + } + }, + { + "id": "transcription_duration", + "payload": { + "format": "transcription", + "type": "duration", + "seconds": 12.5 + } + } + ], + "wire_response": [ + { + "id": "chat_response_full", + "response": { + "id": "chatcmpl-wire", + "created": 1774000000, + "model": "synthetic-chat", + "object": "chat.completion", + "system_fingerprint": null, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "ok", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "usage": { + "format": "chat", + "completion_tokens": 400, + "prompt_tokens": 2000, + "total_tokens": 0, + "completion_tokens_details": null, + "prompt_tokens_details": { + "audio_tokens": null, + "cached_tokens": 1500, + "text_tokens": null, + "image_tokens": null, + "video_tokens": null, + "cache_write_tokens": 100, + "cache_creation_tokens": 100 + }, + "server_tool_use": { + "web_search_requests": 1, + "tool_search_requests": null, + "browser_open_requests": null + }, + "cost": 0.5, + "inference_geo": "us", + "_cache_read_input_tokens": 0, + "_cache_creation_input_tokens": 0 + }, + "_response_ms": 1234.5, + "ended": 1774000012.5 + }, + "hidden_params": { + "custom_llm_provider": "openai", + "region_name": "us-west-2", + "model": "synthetic-chat", + "litellm_model_name": "synthetic-chat-alias", + "additional_headers": { + "llm_provider-x-litellm-response-cost": "0.0123" + }, + "provider_specific_fields": { + "traffic_type": "batch" + } + }, + "optional_params": { + "service_tier": "priority", + "query": [ + "synthetic-a", + "synthetic-b" + ] + } + }, + { + "id": "realtime_results", + "response": { + "results": [ + { + "type": "response.completed", + "response": { + "usage": { + "format": "responses", + "input_tokens": 500, + "output_tokens": 120, + "input_token_details": { + "cached_tokens": 100 + } + }, + "service_tier": "priority" + } + } + ] + } + }, + { + "id": "image_response_usage", + "response": { + "created": 1774000000, + "background": null, + "data": [ + { + "b64_json": null, + "revised_prompt": null, + "url": "https://synthetic/1" + }, + { + "b64_json": null, + "revised_prompt": null, + "url": "https://synthetic/2" + } + ], + "output_format": null, + "quality": null, + "size": null, + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + "input_tokens_details": { + "text_tokens": 40, + "image_tokens": 60 + }, + "output_tokens_details": { + "text_tokens": 10, + "image_tokens": 40, + "reasoning_tokens": 0, + "audio_tokens": 0 + } + } + } + } ] } diff --git a/litellm-rust/crates/cost/tests/python_wire_contract.rs b/litellm-rust/crates/cost/tests/python_wire_contract.rs new file mode 100644 index 00000000000..2103c4a66c9 --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_wire_contract.rs @@ -0,0 +1,309 @@ +// mirrors: crates/cost/tests/generate_python_fixtures.py (wire_usage + wire_response surfaces) + +use serde_json::Value; + +use litellm_cost::wire::{ + ChatUsageInput, DataLen, HiddenParamsInput, ImageResponseInput, OptionalParamsInput, + QueryCount, ReportedCostInput, ResponseInput, TranscriptionUsageInput, UsageInput, WireNumber, +}; + +fn fixture() -> Value { + serde_json::from_str(include_str!("python_fixtures.json")).expect("fixture parses") +} + +fn usage_input(payload: &Value) -> UsageInput { + serde_json::from_value(payload.clone()).expect("wire usage payload deserializes") +} + +#[test] +fn wire_usage_payloads_match_their_declared_format() { + for row in fixture()["wire_usage"].as_array().expect("wire_usage rows") { + let payload = &row["payload"]; + let input = usage_input(payload); + let declared = payload["format"].as_str().expect("format declared"); + let actual = match input { + UsageInput::Chat { .. } => "chat", + UsageInput::Responses { .. } => "responses", + UsageInput::Anthropic { .. } => "anthropic", + UsageInput::Interactions { .. } => "interactions", + UsageInput::Transcription { .. } => "transcription", + }; + assert_eq!(actual, declared, "row {}", row["id"]); + } +} + +#[test] +fn chat_wire_usage_carries_every_field_the_cost_path_reads() { + let input = usage_input(&fixture()["wire_usage"][0]["payload"]); + let UsageInput::Chat { usage } = input else { + panic!("chat tag selects chat payload"); + }; + assert_eq!(usage.prompt_tokens, 1_000); + assert_eq!(usage.completion_tokens, 500); + assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.75))); + assert_eq!(usage.cache_read_input_tokens, Some(800)); + let details = usage.prompt_tokens_details.expect("prompt details present"); + assert_eq!(details.cached_tokens, 800); + assert_eq!(details.cache_write_tokens, Some(300)); + assert_eq!(details.web_search_requests, Some(2)); + let creation = details + .cache_creation_token_details + .expect("cache creation details present"); + assert_eq!(creation.ephemeral_5m_input_tokens, Some(200)); + assert_eq!(creation.ephemeral_1h_input_tokens, Some(100)); + let completion = usage + .completion_tokens_details + .expect("completion details present"); + assert_eq!(completion.reasoning_tokens, Some(120)); + assert_eq!(completion.text_tokens, Some(380)); + let server_tool_use = usage.server_tool_use.expect("server tool use present"); + assert_eq!(server_tool_use.web_search_requests, Some(3)); + assert_eq!(usage.speed.as_deref(), Some("fast")); + assert_eq!(usage.inference_geo.as_deref(), Some("us")); + assert_eq!(usage._cache_read_input_tokens, Some(800)); + let server_side = usage + .server_side_tool_usage_details + .expect("server side details present"); + assert_eq!(server_side.web_search_calls, Some(4)); + assert_eq!(usage.citation_tokens, Some(1_500)); +} + +#[test] +fn chat_wire_usage_with_gateway_merged_token_keys_stays_chat() { + let input = usage_input(&fixture()["wire_usage"][2]["payload"]); + let UsageInput::Chat { usage } = input else { + panic!("gateway merged usage must stay chat"); + }; + assert_eq!(usage.prompt_tokens, 100); + assert_eq!(usage.completion_tokens, 50); +} + +#[test] +fn responses_wire_usage_reads_both_detail_spellings() { + let singular = usage_input(&fixture()["wire_usage"][3]["payload"]); + let UsageInput::Responses { usage } = singular else { + panic!("responses tag selects responses payload"); + }; + assert_eq!(usage.input_tokens, 1_200); + assert_eq!(usage.output_tokens, 300); + assert_eq!( + usage.input_token_details.as_ref().map(|d| d.cached_tokens), + Some(800) + ); + let plural = usage_input(&fixture()["wire_usage"][4]["payload"]); + let UsageInput::Responses { usage } = plural else { + panic!("responses tag selects responses payload"); + }; + assert_eq!( + usage.input_tokens_details.as_ref().map(|d| d.cached_tokens), + Some(200) + ); + assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.25))); +} + +#[test] +fn anthropic_wire_usage_types_iterations_and_cache_breakdown() { + let full = usage_input(&fixture()["wire_usage"][5]["payload"]); + let UsageInput::Anthropic { usage } = full else { + panic!("anthropic tag selects anthropic payload"); + }; + assert_eq!(usage.input_tokens, 5_000); + assert_eq!(usage.cache_read_input_tokens, Some(3_000)); + let creation = usage + .cache_creation + .as_ref() + .expect("cache creation present"); + assert_eq!(creation.ephemeral_5m_input_tokens, Some(1_500)); + assert_eq!(creation.ephemeral_1h_input_tokens, Some(500)); + assert_eq!( + usage + .server_tool_use + .as_ref() + .and_then(|s| s.web_search_requests), + Some(2) + ); + assert_eq!(usage.speed.as_deref(), Some("fast")); + let iterations = usage_input(&fixture()["wire_usage"][6]["payload"]); + let UsageInput::Anthropic { usage } = iterations else { + panic!("anthropic tag selects anthropic payload"); + }; + let iterations = usage.iterations.expect("iterations present"); + assert_eq!(iterations.len(), 2); + assert_eq!(iterations[0].cache_read_input_tokens, Some(20)); + assert_eq!( + iterations[0] + .output_tokens_details + .as_ref() + .and_then(|d| d.thinking_tokens), + Some(90) + ); + assert_eq!( + iterations[0] + .cache_creation + .as_ref() + .and_then(|d| d.ephemeral_5m_input_tokens), + Some(10) + ); +} + +#[test] +fn interactions_wire_usage_types_modality_and_grounding_counts() { + let input = usage_input(&fixture()["wire_usage"][7]["payload"]); + let UsageInput::Interactions { usage } = input else { + panic!("interactions tag selects interactions payload"); + }; + assert_eq!(usage.total_input_tokens, 1_000); + assert_eq!(usage.total_tool_use_tokens, 50); + assert_eq!(usage.total_cached_tokens, 400); + assert_eq!(usage.total_reasoning_tokens, 120); + assert_eq!(usage.input_tokens_by_modality.len(), 2); + assert_eq!( + usage.input_tokens_by_modality[0].modality.as_deref(), + Some("TEXT") + ); + assert_eq!(usage.grounding_tool_count[0].count, 3); + assert_eq!( + usage.grounding_tool_count[0].kind.as_deref(), + Some("google_search") + ); +} + +#[test] +fn transcription_wire_usage_types_duration_and_tokens_variants() { + let tokens = usage_input(&fixture()["wire_usage"][8]["payload"]); + let UsageInput::Transcription { usage } = tokens else { + panic!("transcription tag selects transcription payload"); + }; + let TranscriptionUsageInput::Tokens { + input_tokens, + output_tokens, + total_tokens, + input_token_details, + } = usage + else { + panic!("type tag selects the tokens variant"); + }; + assert_eq!((input_tokens, output_tokens, total_tokens), (100, 200, 300)); + assert_eq!(input_token_details.text_tokens, 60); + assert_eq!(input_token_details.audio_tokens, 40); + let duration = usage_input(&fixture()["wire_usage"][9]["payload"]); + let UsageInput::Transcription { usage } = duration else { + panic!("transcription tag selects transcription payload"); + }; + let TranscriptionUsageInput::Duration { seconds } = usage else { + panic!("type tag selects the duration variant"); + }; + assert_eq!(seconds, 12.5); +} + +fn wire_response_rows() -> Vec { + let fixture = fixture(); + fixture["wire_response"] + .as_array() + .expect("wire_response rows") + .clone() +} + +#[test] +fn wire_response_rows_deserialize_into_typed_structs() { + let rows = wire_response_rows(); + let chat = serde_json::from_value::(rows[0]["response"].clone()) + .expect("chat response deserializes"); + assert_eq!(chat.model.as_deref(), Some("synthetic-chat")); + assert_eq!(chat.created, Some(1_774_000_000.0)); + assert_eq!(chat.ended, Some(1_774_000_012.5)); + assert_eq!(chat.response_ms, Some(1_234.5)); + let UsageInput::Chat { usage } = chat.usage.expect("usage present") else { + panic!("nested usage stays chat"); + }; + assert_eq!(usage.prompt_tokens, 2_000); + assert_eq!( + usage.server_tool_use.and_then(|s| s.web_search_requests), + Some(1) + ); + assert_eq!(usage.inference_geo.as_deref(), Some("us")); + + let hidden = serde_json::from_value::(rows[0]["hidden_params"].clone()) + .expect("hidden params deserialize"); + assert_eq!(hidden.custom_llm_provider.as_deref(), Some("openai")); + assert_eq!(hidden.region_name.as_deref(), Some("us-west-2")); + assert_eq!( + hidden.litellm_model_name.as_deref(), + Some("synthetic-chat-alias") + ); + let header = hidden + .additional_headers + .as_ref() + .and_then(|headers| headers.get("llm_provider-x-litellm-response-cost")) + .cloned(); + assert_eq!(header.and_then(WireNumber::cost), Some(0.0123)); + assert_eq!( + hidden + .provider_specific_fields + .and_then(|fields| fields.traffic_type), + Some("batch".to_string()) + ); + + let optional = + serde_json::from_value::(rows[0]["optional_params"].clone()) + .expect("optional params deserialize"); + assert_eq!(optional.service_tier.as_deref(), Some("priority")); + assert_eq!(optional.query.map(QueryCount::get), Some(2)); + + let realtime = serde_json::from_value::(rows[1]["response"].clone()) + .expect("realtime response deserializes"); + assert_eq!(realtime.results.len(), 1); + let result_response = realtime.results[0] + .response + .as_ref() + .expect("response present"); + assert_eq!(result_response.service_tier.as_deref(), Some("priority")); + let UsageInput::Responses { usage } = result_response.usage.clone().expect("usage present") + else { + panic!("realtime nested usage is responses-shaped"); + }; + assert_eq!(usage.input_tokens, 500); + assert_eq!( + usage.input_token_details.as_ref().map(|d| d.cached_tokens), + Some(100) + ); + + let image = serde_json::from_value::(rows[2]["response"].clone()) + .expect("image response deserializes"); + assert_eq!(image.data.get(), 2); + let usage = image.usage.expect("image usage present"); + assert_eq!( + (usage.input_tokens, usage.output_tokens, usage.total_tokens,), + (100, 50, 150) + ); + assert_eq!( + usage + .input_tokens_details + .as_ref() + .and_then(|d| d.image_tokens), + Some(60) + ); + assert_eq!( + usage + .output_tokens_details + .as_ref() + .and_then(|d| d.image_tokens), + Some(40) + ); +} + +#[test] +fn chat_usage_input_leniently_accepts_bool_string_and_integral_float_counts() { + let usage: ChatUsageInput = serde_json::from_value(serde_json::json!({ + "prompt_tokens": true, + "completion_tokens": " 42 ", + "total_tokens": 58.0, + "cache_read_input_tokens": null, + })) + .expect("lenient counts parse"); + assert_eq!(usage.prompt_tokens, 1); + assert_eq!(usage.completion_tokens, 42); + assert_eq!(usage.total_tokens, 58); + assert_eq!(usage.cache_read_input_tokens, None); + let _ = DataLen::default(); +}