mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(cost): define the Python wire contract with tagged usage and typed inputs
UsageInput is a #[serde(tag = "format")] enum over chat, responses, anthropic, interactions, and transcription payloads, so usage identity is caller-declared instead of key-sniffed. Response, hidden params, optional params, image response, and realtime results get typed structs holding only the fields the cost path reads, and the pricing catalog is embedded once via include_str! behind a LazyLock with an overlay constructor that runs raw entries through the same parser. generate_python_fixtures.py gains wire_usage and wire_response surfaces that model_dump() real Usage, ModelResponse, and ImagesResponse objects (merging the private cache attrs JSON drops), pinning the contract from executed Python.
This commit is contained in:
parent
492ac2e9c4
commit
a8725892a4
5 changed files with 1536 additions and 2 deletions
|
|
@ -54,6 +54,7 @@ pub mod tool_cost_dispatch;
|
|||
pub mod transcription_usage;
|
||||
pub mod usage_dispatch;
|
||||
pub mod vertex_cost;
|
||||
pub mod wire;
|
||||
pub mod xai_cost;
|
||||
pub mod zero_cost_diagnostic;
|
||||
|
||||
|
|
|
|||
630
litellm-rust/crates/cost/src/wire.rs
Normal file
630
litellm-rust/crates/cost/src/wire.rs
Normal file
|
|
@ -0,0 +1,630 @@
|
|||
//! The only module allowed to handle serde_json::Value: it deserializes the
|
||||
//! wire contract Python sends and exposes the embedded pricing catalog. The
|
||||
//! JSON shapes are pinned by the wire_usage and wire_response surfaces of
|
||||
//! tests/generate_python_fixtures.py.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::responses_usage::{
|
||||
CacheCreationTokenDetails, CompletionTokenDetails, PromptTokenDetails,
|
||||
};
|
||||
|
||||
static EMBEDDED_CATALOG_JSON: &str =
|
||||
include_str!("../../../../model_prices_and_context_window.json");
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct RawCatalogEntry(Map<String, Value>);
|
||||
|
||||
impl RawCatalogEntry {
|
||||
pub fn fields(&self) -> &Map<String, Value> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
fn merge(&mut self, overlay: RawCatalogEntry) {
|
||||
for (key, value) in overlay.0 {
|
||||
self.0.insert(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static EMBEDDED_CATALOG: LazyLock<BTreeMap<String, RawCatalogEntry>> = LazyLock::new(|| {
|
||||
let root: Map<String, Value> =
|
||||
serde_json::from_str(EMBEDDED_CATALOG_JSON).expect("embedded catalog JSON parses");
|
||||
root.into_iter()
|
||||
.filter(|(key, _)| key != "fallback_generalizations")
|
||||
.filter_map(|(key, value)| match value {
|
||||
Value::Object(entry) => Some((key, RawCatalogEntry(entry))),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
pub fn overlay_catalog(
|
||||
base: &BTreeMap<String, RawCatalogEntry>,
|
||||
overlays: impl IntoIterator<Item = (String, RawCatalogEntry)>,
|
||||
) -> BTreeMap<String, RawCatalogEntry> {
|
||||
let mut catalog: BTreeMap<_, _> = base
|
||||
.iter()
|
||||
.map(|(key, entry)| (key.clone(), entry.clone()))
|
||||
.collect();
|
||||
for (key, overlay) in overlays {
|
||||
match catalog.get_mut(&key) {
|
||||
Some(existing) => existing.merge(overlay),
|
||||
None => {
|
||||
catalog.insert(key, overlay);
|
||||
}
|
||||
}
|
||||
}
|
||||
catalog
|
||||
}
|
||||
|
||||
pub fn catalog_with_overlays(
|
||||
overlays: impl IntoIterator<Item = (String, RawCatalogEntry)>,
|
||||
) -> BTreeMap<String, RawCatalogEntry> {
|
||||
overlay_catalog(&EMBEDDED_CATALOG, overlays)
|
||||
}
|
||||
|
||||
fn count_value(value: &Value) -> Option<u64> {
|
||||
match value {
|
||||
Value::Null => Some(0),
|
||||
Value::Number(number) => number.as_u64().or_else(|| {
|
||||
number
|
||||
.as_f64()
|
||||
.filter(|value| value.is_finite() && *value >= 0.0 && *value < u64::MAX as f64)
|
||||
.map(|value| value as u64)
|
||||
}),
|
||||
Value::Bool(flag) => Some(u64::from(*flag)),
|
||||
Value::String(text) => text.trim().parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn lenient_count<'de, D>(deserializer: D) -> Result<u64, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
count_value(&value).ok_or_else(|| serde::de::Error::custom("invalid token count"))
|
||||
}
|
||||
|
||||
fn lenient_count_option<'de, D>(deserializer: D) -> Result<Option<u64>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = Option::<Value>::deserialize(deserializer)?;
|
||||
match value {
|
||||
None | Some(Value::Null) => Ok(None),
|
||||
Some(value) => count_value(&value)
|
||||
.map(Some)
|
||||
.ok_or_else(|| serde::de::Error::custom("invalid token count")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum WireNumber {
|
||||
Number(f64),
|
||||
Text(String),
|
||||
Flag(bool),
|
||||
}
|
||||
|
||||
impl WireNumber {
|
||||
pub fn cost(self) -> Option<f64> {
|
||||
match self {
|
||||
Self::Number(value) => Some(value),
|
||||
Self::Text(text) => text.trim().parse().ok(),
|
||||
Self::Flag(flag) => Some(f64::from(flag)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum ReportedCostInput {
|
||||
Number(f64),
|
||||
Detailed { total_cost: Option<f64> },
|
||||
}
|
||||
|
||||
impl ReportedCostInput {
|
||||
pub fn cost(self) -> Option<f64> {
|
||||
match self {
|
||||
Self::Number(value) => Some(value),
|
||||
Self::Detailed { total_cost } => total_cost,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ServerToolUse {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub web_search_requests: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub tool_search_requests: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub browser_open_requests: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ServerSideToolUsageDetails {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub web_search_calls: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
pub struct ChatUsageInput {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub prompt_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub completion_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_tokens: u64,
|
||||
pub cost: Option<ReportedCostInput>,
|
||||
pub prompt_tokens_details: Option<PromptTokenDetails>,
|
||||
pub completion_tokens_details: Option<CompletionTokenDetails>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_read_input_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_creation_input_tokens: Option<u64>,
|
||||
pub server_tool_use: Option<ServerToolUse>,
|
||||
pub speed: Option<String>,
|
||||
pub inference_geo: Option<String>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub _cache_read_input_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub _cache_creation_input_tokens: Option<u64>,
|
||||
pub server_side_tool_usage_details: Option<ServerSideToolUsageDetails>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub citation_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ResponsesUsageInput {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub output_tokens: u64,
|
||||
#[serde(default)]
|
||||
pub input_tokens_details: Option<PromptTokenDetails>,
|
||||
#[serde(default)]
|
||||
pub input_token_details: Option<PromptTokenDetails>,
|
||||
#[serde(default)]
|
||||
pub output_tokens_details: Option<CompletionTokenDetails>,
|
||||
#[serde(default)]
|
||||
pub output_token_details: Option<CompletionTokenDetails>,
|
||||
pub cost: Option<ReportedCostInput>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ThinkingTokenDetails {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub thinking_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct AnthropicIteration {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub output_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_read_input_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_creation_input_tokens: Option<u64>,
|
||||
pub cache_creation: Option<CacheCreationTokenDetails>,
|
||||
pub output_tokens_details: Option<ThinkingTokenDetails>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct AnthropicUsageInput {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub output_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_read_input_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub cache_creation_input_tokens: Option<u64>,
|
||||
pub cache_creation: Option<CacheCreationTokenDetails>,
|
||||
pub output_tokens_details: Option<ThinkingTokenDetails>,
|
||||
pub iterations: Option<Vec<AnthropicIteration>>,
|
||||
pub server_tool_use: Option<ServerToolUse>,
|
||||
pub speed: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ModalityTokens {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub tokens: u64,
|
||||
pub modality: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct GroundingToolCount {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: Option<String>,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct InteractionsUsageInput {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_output_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_tool_use_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_cached_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_reasoning_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_thought_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_tokens: u64,
|
||||
#[serde(default)]
|
||||
pub input_tokens_by_modality: Vec<ModalityTokens>,
|
||||
#[serde(default)]
|
||||
pub tool_use_tokens_by_modality: Vec<ModalityTokens>,
|
||||
#[serde(default)]
|
||||
pub cached_tokens_by_modality: Vec<ModalityTokens>,
|
||||
#[serde(default)]
|
||||
pub output_tokens_by_modality: Vec<ModalityTokens>,
|
||||
#[serde(default)]
|
||||
pub grounding_tool_count: Vec<GroundingToolCount>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
pub struct TranscriptionTokenDetails {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub text_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub audio_tokens: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum TranscriptionUsageInput {
|
||||
Duration {
|
||||
seconds: f64,
|
||||
},
|
||||
Tokens {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
output_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
total_tokens: u64,
|
||||
#[serde(default)]
|
||||
input_token_details: TranscriptionTokenDetails,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
#[serde(tag = "format", rename_all = "snake_case")]
|
||||
pub enum UsageInput {
|
||||
Chat {
|
||||
#[serde(flatten)]
|
||||
usage: ChatUsageInput,
|
||||
},
|
||||
Responses {
|
||||
#[serde(flatten)]
|
||||
usage: ResponsesUsageInput,
|
||||
},
|
||||
Anthropic {
|
||||
#[serde(flatten)]
|
||||
usage: AnthropicUsageInput,
|
||||
},
|
||||
Interactions {
|
||||
#[serde(flatten)]
|
||||
usage: InteractionsUsageInput,
|
||||
},
|
||||
Transcription {
|
||||
#[serde(flatten)]
|
||||
usage: TranscriptionUsageInput,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct BilledUnits {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub total_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub search_units: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ResponseMeta {
|
||||
pub billed_units: Option<BilledUnits>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct RealtimeResponse {
|
||||
pub usage: Option<UsageInput>,
|
||||
pub service_tier: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct RealtimeResultInput {
|
||||
pub response: Option<RealtimeResponse>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ResponseInput {
|
||||
pub model: Option<String>,
|
||||
pub usage: Option<UsageInput>,
|
||||
pub created: Option<f64>,
|
||||
pub ended: Option<f64>,
|
||||
#[serde(rename = "_response_ms")]
|
||||
pub response_ms: Option<f64>,
|
||||
pub provider_reported_cost_usd: Option<f64>,
|
||||
pub duration: Option<f64>,
|
||||
pub meta: Option<ResponseMeta>,
|
||||
#[serde(default)]
|
||||
pub results: Vec<RealtimeResultInput>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ProviderSpecificFields {
|
||||
pub traffic_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct HiddenParamsInput {
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub region_name: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub litellm_model_name: Option<String>,
|
||||
pub additional_headers: Option<BTreeMap<String, WireNumber>>,
|
||||
pub audio_transcription_duration: Option<f64>,
|
||||
pub provider_specific_fields: Option<ProviderSpecificFields>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct QueryCount(pub u64);
|
||||
|
||||
impl QueryCount {
|
||||
pub fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for QueryCount {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
let count = value.as_array().map_or(1, Vec::len) as u64;
|
||||
Ok(Self(count))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct OptionalParamsInput {
|
||||
pub service_tier: Option<String>,
|
||||
pub query: Option<QueryCount>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ImageInputTokenDetails {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub text_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub image_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ImageOutputTokenDetails {
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub text_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub image_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub audio_tokens: Option<u64>,
|
||||
#[serde(default, deserialize_with = "lenient_count_option")]
|
||||
pub reasoning_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ImageUsageInput {
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub input_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub output_tokens: u64,
|
||||
#[serde(default, deserialize_with = "lenient_count")]
|
||||
pub total_tokens: u64,
|
||||
#[serde(default)]
|
||||
pub input_tokens_details: Option<ImageInputTokenDetails>,
|
||||
#[serde(default)]
|
||||
pub completion_tokens_details: Option<ImageOutputTokenDetails>,
|
||||
#[serde(default)]
|
||||
pub output_tokens_details: Option<ImageOutputTokenDetails>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq)]
|
||||
pub struct DataLen(pub u64);
|
||||
|
||||
impl DataLen {
|
||||
pub fn get(self) -> u64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for DataLen {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
let count = value.as_array().map_or(0, Vec::len) as u64;
|
||||
Ok(Self(count))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq)]
|
||||
pub struct ImageResponseInput {
|
||||
pub usage: Option<ImageUsageInput>,
|
||||
#[serde(default)]
|
||||
pub data: DataLen,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn entry(fields: &[(&str, Value)]) -> RawCatalogEntry {
|
||||
RawCatalogEntry(Map::from_iter(
|
||||
fields
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.clone())),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlay_merges_fields_into_existing_entries_and_adds_new_ones() {
|
||||
let base = BTreeMap::from([(
|
||||
"synthetic-base".to_string(),
|
||||
entry(&[
|
||||
("input_cost_per_token", serde_json::json!(3e-6)),
|
||||
("output_cost_per_token", serde_json::json!(15e-6)),
|
||||
]),
|
||||
)]);
|
||||
let overlaid = overlay_catalog(
|
||||
&base,
|
||||
[
|
||||
(
|
||||
"synthetic-base".to_string(),
|
||||
entry(&[("output_cost_per_token", serde_json::json!(9e-6))]),
|
||||
),
|
||||
(
|
||||
"synthetic-new".to_string(),
|
||||
entry(&[("input_cost_per_token", serde_json::json!(1e-6))]),
|
||||
),
|
||||
],
|
||||
);
|
||||
assert_eq!(overlaid.len(), 2);
|
||||
let merged = &overlaid["synthetic-base"];
|
||||
assert_eq!(
|
||||
merged.fields().get("input_cost_per_token"),
|
||||
Some(&serde_json::json!(3e-6))
|
||||
);
|
||||
assert_eq!(
|
||||
merged.fields().get("output_cost_per_token"),
|
||||
Some(&serde_json::json!(9e-6))
|
||||
);
|
||||
assert!(
|
||||
overlaid["synthetic-new"]
|
||||
.fields()
|
||||
.contains_key("input_cost_per_token")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_catalog_excludes_generalizations_meta_key() {
|
||||
let catalog = LazyLock::force(&EMBEDDED_CATALOG);
|
||||
assert!(!catalog.is_empty());
|
||||
assert!(!catalog.contains_key("fallback_generalizations"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_usage_reads_lenient_counts_and_cost_projection() {
|
||||
let usage: ChatUsageInput = serde_json::from_value(serde_json::json!({
|
||||
"prompt_tokens": true,
|
||||
"completion_tokens": " 42 ",
|
||||
"total_tokens": 58.0,
|
||||
"cost": {"total_cost": 0.5},
|
||||
"cache_read_input_tokens": 7,
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(usage.prompt_tokens, 1);
|
||||
assert_eq!(usage.completion_tokens, 42);
|
||||
assert_eq!(usage.total_tokens, 58);
|
||||
assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.5)));
|
||||
assert_eq!(usage.cache_read_input_tokens, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_usage_rejects_negative_and_garbage_counts() {
|
||||
assert!(
|
||||
serde_json::from_value::<ChatUsageInput>(serde_json::json!({
|
||||
"prompt_tokens": -1
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<ChatUsageInput>(serde_json::json!({
|
||||
"prompt_tokens": "not a number"
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_input_dispatches_on_format_tag_not_key_shapes() {
|
||||
let chat: UsageInput = serde_json::from_value(serde_json::json!({
|
||||
"format": "chat",
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"input_tokens": 999,
|
||||
"output_tokens": 99,
|
||||
}))
|
||||
.unwrap();
|
||||
let UsageInput::Chat { usage } = chat else {
|
||||
panic!("chat tag must select the chat payload");
|
||||
};
|
||||
assert_eq!(usage.prompt_tokens, 10);
|
||||
let transcription: UsageInput = serde_json::from_value(serde_json::json!({
|
||||
"format": "transcription",
|
||||
"type": "tokens",
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"input_token_details": {"text_tokens": 60, "audio_tokens": 40},
|
||||
}))
|
||||
.unwrap();
|
||||
let UsageInput::Transcription { usage } = transcription else {
|
||||
panic!("transcription tag must select the transcription payload");
|
||||
};
|
||||
let TranscriptionUsageInput::Tokens {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
..
|
||||
} = usage
|
||||
else {
|
||||
panic!("transcription type tag must select the tokens variant");
|
||||
};
|
||||
assert_eq!((input_tokens, output_tokens), (100, 200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hidden_params_read_wire_number_header_costs() {
|
||||
let hidden: HiddenParamsInput = serde_json::from_value(serde_json::json!({
|
||||
"custom_llm_provider": "openai",
|
||||
"additional_headers": {"llm_provider-x-litellm-response-cost": " 0.0123 "},
|
||||
}))
|
||||
.unwrap();
|
||||
let header = hidden
|
||||
.additional_headers
|
||||
.as_ref()
|
||||
.and_then(|headers| headers.get("llm_provider-x-litellm-response-cost"))
|
||||
.cloned();
|
||||
assert_eq!(header.and_then(WireNumber::cost), Some(0.0123));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_count_counts_arrays_and_defaults_non_arrays_to_one() {
|
||||
let params: OptionalParamsInput =
|
||||
serde_json::from_value(serde_json::json!({"query": ["a", "b", "c"]})).unwrap();
|
||||
assert_eq!(params.query.map(QueryCount::get), Some(3));
|
||||
let single: OptionalParamsInput =
|
||||
serde_json::from_value(serde_json::json!({"query": "galaxy"})).unwrap();
|
||||
assert_eq!(single.query.map(QueryCount::get), Some(1));
|
||||
}
|
||||
}
|
||||
|
|
@ -22,11 +22,255 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
)
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.utils import register_model
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai.types.images_response import ImagesResponse
|
||||
|
||||
UTC = timezone.utc
|
||||
|
||||
|
||||
def private_cache_attrs(usage: Usage) -> dict[str, int]:
|
||||
return {
|
||||
"_cache_read_input_tokens": usage._cache_read_input_tokens,
|
||||
"_cache_creation_input_tokens": usage._cache_creation_input_tokens,
|
||||
}
|
||||
|
||||
|
||||
def chat_usage_payload(usage: Usage) -> dict[str, Any]:
|
||||
return {"format": "chat", **usage.model_dump(), **private_cache_attrs(usage)}
|
||||
|
||||
|
||||
def tagged_response_payload(response: ModelResponse) -> dict[str, Any]:
|
||||
dump = response.model_dump()
|
||||
if dump.get("usage") is not None:
|
||||
dump["usage"] = {"format": "chat", **dump["usage"], **private_cache_attrs(response.usage)}
|
||||
return dump
|
||||
|
||||
|
||||
WIRE_USAGE_CASES: tuple[tuple[str, str, dict[str, Any]], ...] = (
|
||||
(
|
||||
"chat_full_projection",
|
||||
"chat",
|
||||
chat_usage_payload(
|
||||
Usage(
|
||||
prompt_tokens=1_000,
|
||||
completion_tokens=500,
|
||||
reasoning_tokens=120,
|
||||
prompt_tokens_details={
|
||||
"cached_tokens": 800,
|
||||
"cache_write_tokens": 300,
|
||||
"web_search_requests": 2,
|
||||
"cache_creation_token_details": {
|
||||
"ephemeral_5m_input_tokens": 200,
|
||||
"ephemeral_1h_input_tokens": 100,
|
||||
},
|
||||
},
|
||||
completion_tokens_details={"reasoning_tokens": 120, "text_tokens": 380},
|
||||
server_tool_use={"web_search_requests": 3},
|
||||
cost=0.75,
|
||||
cache_read_input_tokens=800,
|
||||
**{
|
||||
"inference_geo": "us",
|
||||
"speed": "fast",
|
||||
"citation_tokens": 1_500,
|
||||
"server_side_tool_usage_details": {"web_search_calls": 4},
|
||||
},
|
||||
)
|
||||
),
|
||||
),
|
||||
(
|
||||
"chat_minimal",
|
||||
"chat",
|
||||
chat_usage_payload(Usage(prompt_tokens=10, completion_tokens=5)),
|
||||
),
|
||||
(
|
||||
"chat_gateway_merged_input_token_keys",
|
||||
"chat",
|
||||
chat_usage_payload(Usage(prompt_tokens=100, completion_tokens=50, input_tokens=100, output_tokens=50)),
|
||||
),
|
||||
(
|
||||
"responses_singular_details",
|
||||
"responses",
|
||||
{
|
||||
"input_tokens": 1_200,
|
||||
"output_tokens": 300,
|
||||
"input_token_details": {"cached_tokens": 800, "web_search_requests": 2},
|
||||
"output_token_details": {"reasoning_tokens": 100},
|
||||
},
|
||||
),
|
||||
(
|
||||
"responses_plural_details",
|
||||
"responses",
|
||||
{
|
||||
"input_tokens": 900,
|
||||
"output_tokens": 250,
|
||||
"input_tokens_details": {"text_tokens": 700, "cached_tokens": 200},
|
||||
"output_tokens_details": {"text_tokens": 150, "audio_tokens": 100},
|
||||
"cost": {"total_cost": 0.25},
|
||||
},
|
||||
),
|
||||
(
|
||||
"anthropic_full",
|
||||
"anthropic",
|
||||
{
|
||||
"input_tokens": 5_000,
|
||||
"output_tokens": 200,
|
||||
"cache_read_input_tokens": 3_000,
|
||||
"cache_creation_input_tokens": 2_000,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 1_500, "ephemeral_1h_input_tokens": 500},
|
||||
"server_tool_use": {"web_search_requests": 2},
|
||||
"speed": "fast",
|
||||
},
|
||||
),
|
||||
(
|
||||
"anthropic_iterations",
|
||||
"anthropic",
|
||||
{
|
||||
"input_tokens": 600,
|
||||
"output_tokens": 300,
|
||||
"iterations": [
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 150,
|
||||
"cache_read_input_tokens": 20,
|
||||
"cache_creation_input_tokens": 10,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 10},
|
||||
"output_tokens_details": {"thinking_tokens": 90},
|
||||
},
|
||||
{
|
||||
"input_tokens": 200,
|
||||
"output_tokens": 150,
|
||||
"output_tokens_details": {"thinking_tokens": 60},
|
||||
},
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
"interactions_modality_split",
|
||||
"interactions",
|
||||
{
|
||||
"total_input_tokens": 1_000,
|
||||
"total_output_tokens": 300,
|
||||
"total_tool_use_tokens": 50,
|
||||
"total_cached_tokens": 400,
|
||||
"total_reasoning_tokens": 120,
|
||||
"total_tokens": 1_470,
|
||||
"input_tokens_by_modality": [
|
||||
{"tokens": 600, "modality": "TEXT"},
|
||||
{"tokens": 50, "modality": "AUDIO"},
|
||||
],
|
||||
"cached_tokens_by_modality": [{"tokens": 400, "modality": "TEXT"}],
|
||||
"output_tokens_by_modality": [{"tokens": 180, "modality": "TEXT"}],
|
||||
"grounding_tool_count": [{"type": "google_search", "count": 3}],
|
||||
},
|
||||
),
|
||||
(
|
||||
"transcription_tokens",
|
||||
"transcription",
|
||||
{
|
||||
"type": "tokens",
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"input_token_details": {"text_tokens": 60, "audio_tokens": 40},
|
||||
},
|
||||
),
|
||||
(
|
||||
"transcription_duration",
|
||||
"transcription",
|
||||
{"type": "duration", "seconds": 12.5},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def wire_usage_surface() -> list[dict[str, Any]]:
|
||||
return [{"id": name, "payload": {"format": declared, **payload}} for name, declared, payload in WIRE_USAGE_CASES]
|
||||
|
||||
|
||||
def wire_response_surface() -> list[dict[str, Any]]:
|
||||
usage = Usage(
|
||||
prompt_tokens=2_000,
|
||||
completion_tokens=400,
|
||||
prompt_tokens_details={"cached_tokens": 1_500, "cache_write_tokens": 100},
|
||||
server_tool_use={"web_search_requests": 1},
|
||||
cost=0.5,
|
||||
)
|
||||
usage.inference_geo = "us"
|
||||
chat_response = ModelResponse(
|
||||
id="chatcmpl-wire",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content="ok", role="assistant"),
|
||||
)
|
||||
],
|
||||
created=1_774_000_000,
|
||||
model="synthetic-chat",
|
||||
usage=usage,
|
||||
_response_ms=1_234.5,
|
||||
)
|
||||
chat_response.ended = 1_774_000_012.5
|
||||
image_response = ImagesResponse(
|
||||
created=1_774_000_000,
|
||||
data=[
|
||||
{"url": "https://synthetic/1", "revised_prompt": None},
|
||||
{"url": "https://synthetic/2", "revised_prompt": None},
|
||||
],
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": "chat_response_full",
|
||||
"response": tagged_response_payload(chat_response),
|
||||
"hidden_params": {
|
||||
"custom_llm_provider": "openai",
|
||||
"region_name": "us-west-2",
|
||||
"model": "synthetic-chat",
|
||||
"litellm_model_name": "synthetic-chat-alias",
|
||||
"additional_headers": {"llm_provider-x-litellm-response-cost": "0.0123"},
|
||||
"provider_specific_fields": {"traffic_type": "batch"},
|
||||
},
|
||||
"optional_params": {"service_tier": "priority", "query": ["synthetic-a", "synthetic-b"]},
|
||||
},
|
||||
{
|
||||
"id": "realtime_results",
|
||||
"response": {
|
||||
"results": [
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"usage": {
|
||||
"format": "responses",
|
||||
"input_tokens": 500,
|
||||
"output_tokens": 120,
|
||||
"input_token_details": {"cached_tokens": 100},
|
||||
},
|
||||
"service_tier": "priority",
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "image_response_usage",
|
||||
"response": {
|
||||
**image_response.model_dump(),
|
||||
"usage": {
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"input_tokens_details": {"text_tokens": 40, "image_tokens": 60},
|
||||
"output_tokens_details": {
|
||||
"text_tokens": 10,
|
||||
"image_tokens": 40,
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def details(
|
||||
cached: int | None = None,
|
||||
cache_write: int | None = None,
|
||||
|
|
@ -824,6 +1068,8 @@ def main() -> None:
|
|||
"responses_usage": responses_usage_surface(),
|
||||
"caching_savings": caching_savings_surface(),
|
||||
"anthropic_usage": anthropic_usage_surface(),
|
||||
"wire_usage": wire_usage_surface(),
|
||||
"wire_response": wire_response_surface(),
|
||||
}
|
||||
target = Path(__file__).parent / "python_fixtures.json"
|
||||
target.write_text(json.dumps(fixture, indent=1) + "\n")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"python_commit": "b401bc9be2509e46fc0126087db2542b129ad56c",
|
||||
"python_commit": "492ac2e9c43ed8fc9ae18650f6d41987ff7ae0fc",
|
||||
"generic": [
|
||||
{
|
||||
"id": "audio_mixed_modalities",
|
||||
|
|
@ -1774,5 +1774,353 @@
|
|||
"cache_read_input_tokens": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"wire_usage": [
|
||||
{
|
||||
"id": "chat_full_projection",
|
||||
"payload": {
|
||||
"format": "chat",
|
||||
"completion_tokens": 500,
|
||||
"prompt_tokens": 1000,
|
||||
"total_tokens": 0,
|
||||
"completion_tokens_details": {
|
||||
"accepted_prediction_tokens": null,
|
||||
"audio_tokens": null,
|
||||
"reasoning_tokens": 120,
|
||||
"rejected_prediction_tokens": null,
|
||||
"text_tokens": 380,
|
||||
"image_tokens": null,
|
||||
"video_tokens": null
|
||||
},
|
||||
"prompt_tokens_details": {
|
||||
"audio_tokens": null,
|
||||
"cached_tokens": 800,
|
||||
"text_tokens": null,
|
||||
"image_tokens": null,
|
||||
"video_tokens": null,
|
||||
"web_search_requests": 2,
|
||||
"cache_write_tokens": 300,
|
||||
"cache_creation_tokens": 300,
|
||||
"cache_creation_token_details": {
|
||||
"ephemeral_5m_input_tokens": 200,
|
||||
"ephemeral_1h_input_tokens": 100
|
||||
}
|
||||
},
|
||||
"server_tool_use": {
|
||||
"web_search_requests": 3,
|
||||
"tool_search_requests": null,
|
||||
"browser_open_requests": null
|
||||
},
|
||||
"cost": 0.75,
|
||||
"cache_read_input_tokens": 800,
|
||||
"inference_geo": "us",
|
||||
"speed": "fast",
|
||||
"citation_tokens": 1500,
|
||||
"server_side_tool_usage_details": {
|
||||
"web_search_calls": 4
|
||||
},
|
||||
"_cache_read_input_tokens": 800,
|
||||
"_cache_creation_input_tokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_minimal",
|
||||
"payload": {
|
||||
"format": "chat",
|
||||
"completion_tokens": 5,
|
||||
"prompt_tokens": 10,
|
||||
"total_tokens": 0,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null,
|
||||
"_cache_read_input_tokens": 0,
|
||||
"_cache_creation_input_tokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_gateway_merged_input_token_keys",
|
||||
"payload": {
|
||||
"format": "chat",
|
||||
"completion_tokens": 50,
|
||||
"prompt_tokens": 100,
|
||||
"total_tokens": 0,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null,
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
"_cache_read_input_tokens": 0,
|
||||
"_cache_creation_input_tokens": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "responses_singular_details",
|
||||
"payload": {
|
||||
"format": "responses",
|
||||
"input_tokens": 1200,
|
||||
"output_tokens": 300,
|
||||
"input_token_details": {
|
||||
"cached_tokens": 800,
|
||||
"web_search_requests": 2
|
||||
},
|
||||
"output_token_details": {
|
||||
"reasoning_tokens": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "responses_plural_details",
|
||||
"payload": {
|
||||
"format": "responses",
|
||||
"input_tokens": 900,
|
||||
"output_tokens": 250,
|
||||
"input_tokens_details": {
|
||||
"text_tokens": 700,
|
||||
"cached_tokens": 200
|
||||
},
|
||||
"output_tokens_details": {
|
||||
"text_tokens": 150,
|
||||
"audio_tokens": 100
|
||||
},
|
||||
"cost": {
|
||||
"total_cost": 0.25
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "anthropic_full",
|
||||
"payload": {
|
||||
"format": "anthropic",
|
||||
"input_tokens": 5000,
|
||||
"output_tokens": 200,
|
||||
"cache_read_input_tokens": 3000,
|
||||
"cache_creation_input_tokens": 2000,
|
||||
"cache_creation": {
|
||||
"ephemeral_5m_input_tokens": 1500,
|
||||
"ephemeral_1h_input_tokens": 500
|
||||
},
|
||||
"server_tool_use": {
|
||||
"web_search_requests": 2
|
||||
},
|
||||
"speed": "fast"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "anthropic_iterations",
|
||||
"payload": {
|
||||
"format": "anthropic",
|
||||
"input_tokens": 600,
|
||||
"output_tokens": 300,
|
||||
"iterations": [
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 150,
|
||||
"cache_read_input_tokens": 20,
|
||||
"cache_creation_input_tokens": 10,
|
||||
"cache_creation": {
|
||||
"ephemeral_5m_input_tokens": 10
|
||||
},
|
||||
"output_tokens_details": {
|
||||
"thinking_tokens": 90
|
||||
}
|
||||
},
|
||||
{
|
||||
"input_tokens": 200,
|
||||
"output_tokens": 150,
|
||||
"output_tokens_details": {
|
||||
"thinking_tokens": 60
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "interactions_modality_split",
|
||||
"payload": {
|
||||
"format": "interactions",
|
||||
"total_input_tokens": 1000,
|
||||
"total_output_tokens": 300,
|
||||
"total_tool_use_tokens": 50,
|
||||
"total_cached_tokens": 400,
|
||||
"total_reasoning_tokens": 120,
|
||||
"total_tokens": 1470,
|
||||
"input_tokens_by_modality": [
|
||||
{
|
||||
"tokens": 600,
|
||||
"modality": "TEXT"
|
||||
},
|
||||
{
|
||||
"tokens": 50,
|
||||
"modality": "AUDIO"
|
||||
}
|
||||
],
|
||||
"cached_tokens_by_modality": [
|
||||
{
|
||||
"tokens": 400,
|
||||
"modality": "TEXT"
|
||||
}
|
||||
],
|
||||
"output_tokens_by_modality": [
|
||||
{
|
||||
"tokens": 180,
|
||||
"modality": "TEXT"
|
||||
}
|
||||
],
|
||||
"grounding_tool_count": [
|
||||
{
|
||||
"type": "google_search",
|
||||
"count": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "transcription_tokens",
|
||||
"payload": {
|
||||
"format": "transcription",
|
||||
"type": "tokens",
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"input_token_details": {
|
||||
"text_tokens": 60,
|
||||
"audio_tokens": 40
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "transcription_duration",
|
||||
"payload": {
|
||||
"format": "transcription",
|
||||
"type": "duration",
|
||||
"seconds": 12.5
|
||||
}
|
||||
}
|
||||
],
|
||||
"wire_response": [
|
||||
{
|
||||
"id": "chat_response_full",
|
||||
"response": {
|
||||
"id": "chatcmpl-wire",
|
||||
"created": 1774000000,
|
||||
"model": "synthetic-chat",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": null,
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "ok",
|
||||
"role": "assistant",
|
||||
"tool_calls": null,
|
||||
"function_call": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"format": "chat",
|
||||
"completion_tokens": 400,
|
||||
"prompt_tokens": 2000,
|
||||
"total_tokens": 0,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": {
|
||||
"audio_tokens": null,
|
||||
"cached_tokens": 1500,
|
||||
"text_tokens": null,
|
||||
"image_tokens": null,
|
||||
"video_tokens": null,
|
||||
"cache_write_tokens": 100,
|
||||
"cache_creation_tokens": 100
|
||||
},
|
||||
"server_tool_use": {
|
||||
"web_search_requests": 1,
|
||||
"tool_search_requests": null,
|
||||
"browser_open_requests": null
|
||||
},
|
||||
"cost": 0.5,
|
||||
"inference_geo": "us",
|
||||
"_cache_read_input_tokens": 0,
|
||||
"_cache_creation_input_tokens": 0
|
||||
},
|
||||
"_response_ms": 1234.5,
|
||||
"ended": 1774000012.5
|
||||
},
|
||||
"hidden_params": {
|
||||
"custom_llm_provider": "openai",
|
||||
"region_name": "us-west-2",
|
||||
"model": "synthetic-chat",
|
||||
"litellm_model_name": "synthetic-chat-alias",
|
||||
"additional_headers": {
|
||||
"llm_provider-x-litellm-response-cost": "0.0123"
|
||||
},
|
||||
"provider_specific_fields": {
|
||||
"traffic_type": "batch"
|
||||
}
|
||||
},
|
||||
"optional_params": {
|
||||
"service_tier": "priority",
|
||||
"query": [
|
||||
"synthetic-a",
|
||||
"synthetic-b"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "realtime_results",
|
||||
"response": {
|
||||
"results": [
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"usage": {
|
||||
"format": "responses",
|
||||
"input_tokens": 500,
|
||||
"output_tokens": 120,
|
||||
"input_token_details": {
|
||||
"cached_tokens": 100
|
||||
}
|
||||
},
|
||||
"service_tier": "priority"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "image_response_usage",
|
||||
"response": {
|
||||
"created": 1774000000,
|
||||
"background": null,
|
||||
"data": [
|
||||
{
|
||||
"b64_json": null,
|
||||
"revised_prompt": null,
|
||||
"url": "https://synthetic/1"
|
||||
},
|
||||
{
|
||||
"b64_json": null,
|
||||
"revised_prompt": null,
|
||||
"url": "https://synthetic/2"
|
||||
}
|
||||
],
|
||||
"output_format": null,
|
||||
"quality": null,
|
||||
"size": null,
|
||||
"usage": {
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"input_tokens_details": {
|
||||
"text_tokens": 40,
|
||||
"image_tokens": 60
|
||||
},
|
||||
"output_tokens_details": {
|
||||
"text_tokens": 10,
|
||||
"image_tokens": 40,
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
309
litellm-rust/crates/cost/tests/python_wire_contract.rs
Normal file
309
litellm-rust/crates/cost/tests/python_wire_contract.rs
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
// mirrors: crates/cost/tests/generate_python_fixtures.py (wire_usage + wire_response surfaces)
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use litellm_cost::wire::{
|
||||
ChatUsageInput, DataLen, HiddenParamsInput, ImageResponseInput, OptionalParamsInput,
|
||||
QueryCount, ReportedCostInput, ResponseInput, TranscriptionUsageInput, UsageInput, WireNumber,
|
||||
};
|
||||
|
||||
fn fixture() -> Value {
|
||||
serde_json::from_str(include_str!("python_fixtures.json")).expect("fixture parses")
|
||||
}
|
||||
|
||||
fn usage_input(payload: &Value) -> UsageInput {
|
||||
serde_json::from_value(payload.clone()).expect("wire usage payload deserializes")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_usage_payloads_match_their_declared_format() {
|
||||
for row in fixture()["wire_usage"].as_array().expect("wire_usage rows") {
|
||||
let payload = &row["payload"];
|
||||
let input = usage_input(payload);
|
||||
let declared = payload["format"].as_str().expect("format declared");
|
||||
let actual = match input {
|
||||
UsageInput::Chat { .. } => "chat",
|
||||
UsageInput::Responses { .. } => "responses",
|
||||
UsageInput::Anthropic { .. } => "anthropic",
|
||||
UsageInput::Interactions { .. } => "interactions",
|
||||
UsageInput::Transcription { .. } => "transcription",
|
||||
};
|
||||
assert_eq!(actual, declared, "row {}", row["id"]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wire_usage_carries_every_field_the_cost_path_reads() {
|
||||
let input = usage_input(&fixture()["wire_usage"][0]["payload"]);
|
||||
let UsageInput::Chat { usage } = input else {
|
||||
panic!("chat tag selects chat payload");
|
||||
};
|
||||
assert_eq!(usage.prompt_tokens, 1_000);
|
||||
assert_eq!(usage.completion_tokens, 500);
|
||||
assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.75)));
|
||||
assert_eq!(usage.cache_read_input_tokens, Some(800));
|
||||
let details = usage.prompt_tokens_details.expect("prompt details present");
|
||||
assert_eq!(details.cached_tokens, 800);
|
||||
assert_eq!(details.cache_write_tokens, Some(300));
|
||||
assert_eq!(details.web_search_requests, Some(2));
|
||||
let creation = details
|
||||
.cache_creation_token_details
|
||||
.expect("cache creation details present");
|
||||
assert_eq!(creation.ephemeral_5m_input_tokens, Some(200));
|
||||
assert_eq!(creation.ephemeral_1h_input_tokens, Some(100));
|
||||
let completion = usage
|
||||
.completion_tokens_details
|
||||
.expect("completion details present");
|
||||
assert_eq!(completion.reasoning_tokens, Some(120));
|
||||
assert_eq!(completion.text_tokens, Some(380));
|
||||
let server_tool_use = usage.server_tool_use.expect("server tool use present");
|
||||
assert_eq!(server_tool_use.web_search_requests, Some(3));
|
||||
assert_eq!(usage.speed.as_deref(), Some("fast"));
|
||||
assert_eq!(usage.inference_geo.as_deref(), Some("us"));
|
||||
assert_eq!(usage._cache_read_input_tokens, Some(800));
|
||||
let server_side = usage
|
||||
.server_side_tool_usage_details
|
||||
.expect("server side details present");
|
||||
assert_eq!(server_side.web_search_calls, Some(4));
|
||||
assert_eq!(usage.citation_tokens, Some(1_500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wire_usage_with_gateway_merged_token_keys_stays_chat() {
|
||||
let input = usage_input(&fixture()["wire_usage"][2]["payload"]);
|
||||
let UsageInput::Chat { usage } = input else {
|
||||
panic!("gateway merged usage must stay chat");
|
||||
};
|
||||
assert_eq!(usage.prompt_tokens, 100);
|
||||
assert_eq!(usage.completion_tokens, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_wire_usage_reads_both_detail_spellings() {
|
||||
let singular = usage_input(&fixture()["wire_usage"][3]["payload"]);
|
||||
let UsageInput::Responses { usage } = singular else {
|
||||
panic!("responses tag selects responses payload");
|
||||
};
|
||||
assert_eq!(usage.input_tokens, 1_200);
|
||||
assert_eq!(usage.output_tokens, 300);
|
||||
assert_eq!(
|
||||
usage.input_token_details.as_ref().map(|d| d.cached_tokens),
|
||||
Some(800)
|
||||
);
|
||||
let plural = usage_input(&fixture()["wire_usage"][4]["payload"]);
|
||||
let UsageInput::Responses { usage } = plural else {
|
||||
panic!("responses tag selects responses payload");
|
||||
};
|
||||
assert_eq!(
|
||||
usage.input_tokens_details.as_ref().map(|d| d.cached_tokens),
|
||||
Some(200)
|
||||
);
|
||||
assert_eq!(usage.cost.map(ReportedCostInput::cost), Some(Some(0.25)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anthropic_wire_usage_types_iterations_and_cache_breakdown() {
|
||||
let full = usage_input(&fixture()["wire_usage"][5]["payload"]);
|
||||
let UsageInput::Anthropic { usage } = full else {
|
||||
panic!("anthropic tag selects anthropic payload");
|
||||
};
|
||||
assert_eq!(usage.input_tokens, 5_000);
|
||||
assert_eq!(usage.cache_read_input_tokens, Some(3_000));
|
||||
let creation = usage
|
||||
.cache_creation
|
||||
.as_ref()
|
||||
.expect("cache creation present");
|
||||
assert_eq!(creation.ephemeral_5m_input_tokens, Some(1_500));
|
||||
assert_eq!(creation.ephemeral_1h_input_tokens, Some(500));
|
||||
assert_eq!(
|
||||
usage
|
||||
.server_tool_use
|
||||
.as_ref()
|
||||
.and_then(|s| s.web_search_requests),
|
||||
Some(2)
|
||||
);
|
||||
assert_eq!(usage.speed.as_deref(), Some("fast"));
|
||||
let iterations = usage_input(&fixture()["wire_usage"][6]["payload"]);
|
||||
let UsageInput::Anthropic { usage } = iterations else {
|
||||
panic!("anthropic tag selects anthropic payload");
|
||||
};
|
||||
let iterations = usage.iterations.expect("iterations present");
|
||||
assert_eq!(iterations.len(), 2);
|
||||
assert_eq!(iterations[0].cache_read_input_tokens, Some(20));
|
||||
assert_eq!(
|
||||
iterations[0]
|
||||
.output_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|d| d.thinking_tokens),
|
||||
Some(90)
|
||||
);
|
||||
assert_eq!(
|
||||
iterations[0]
|
||||
.cache_creation
|
||||
.as_ref()
|
||||
.and_then(|d| d.ephemeral_5m_input_tokens),
|
||||
Some(10)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interactions_wire_usage_types_modality_and_grounding_counts() {
|
||||
let input = usage_input(&fixture()["wire_usage"][7]["payload"]);
|
||||
let UsageInput::Interactions { usage } = input else {
|
||||
panic!("interactions tag selects interactions payload");
|
||||
};
|
||||
assert_eq!(usage.total_input_tokens, 1_000);
|
||||
assert_eq!(usage.total_tool_use_tokens, 50);
|
||||
assert_eq!(usage.total_cached_tokens, 400);
|
||||
assert_eq!(usage.total_reasoning_tokens, 120);
|
||||
assert_eq!(usage.input_tokens_by_modality.len(), 2);
|
||||
assert_eq!(
|
||||
usage.input_tokens_by_modality[0].modality.as_deref(),
|
||||
Some("TEXT")
|
||||
);
|
||||
assert_eq!(usage.grounding_tool_count[0].count, 3);
|
||||
assert_eq!(
|
||||
usage.grounding_tool_count[0].kind.as_deref(),
|
||||
Some("google_search")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_wire_usage_types_duration_and_tokens_variants() {
|
||||
let tokens = usage_input(&fixture()["wire_usage"][8]["payload"]);
|
||||
let UsageInput::Transcription { usage } = tokens else {
|
||||
panic!("transcription tag selects transcription payload");
|
||||
};
|
||||
let TranscriptionUsageInput::Tokens {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
total_tokens,
|
||||
input_token_details,
|
||||
} = usage
|
||||
else {
|
||||
panic!("type tag selects the tokens variant");
|
||||
};
|
||||
assert_eq!((input_tokens, output_tokens, total_tokens), (100, 200, 300));
|
||||
assert_eq!(input_token_details.text_tokens, 60);
|
||||
assert_eq!(input_token_details.audio_tokens, 40);
|
||||
let duration = usage_input(&fixture()["wire_usage"][9]["payload"]);
|
||||
let UsageInput::Transcription { usage } = duration else {
|
||||
panic!("transcription tag selects transcription payload");
|
||||
};
|
||||
let TranscriptionUsageInput::Duration { seconds } = usage else {
|
||||
panic!("type tag selects the duration variant");
|
||||
};
|
||||
assert_eq!(seconds, 12.5);
|
||||
}
|
||||
|
||||
fn wire_response_rows() -> Vec<Value> {
|
||||
let fixture = fixture();
|
||||
fixture["wire_response"]
|
||||
.as_array()
|
||||
.expect("wire_response rows")
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wire_response_rows_deserialize_into_typed_structs() {
|
||||
let rows = wire_response_rows();
|
||||
let chat = serde_json::from_value::<ResponseInput>(rows[0]["response"].clone())
|
||||
.expect("chat response deserializes");
|
||||
assert_eq!(chat.model.as_deref(), Some("synthetic-chat"));
|
||||
assert_eq!(chat.created, Some(1_774_000_000.0));
|
||||
assert_eq!(chat.ended, Some(1_774_000_012.5));
|
||||
assert_eq!(chat.response_ms, Some(1_234.5));
|
||||
let UsageInput::Chat { usage } = chat.usage.expect("usage present") else {
|
||||
panic!("nested usage stays chat");
|
||||
};
|
||||
assert_eq!(usage.prompt_tokens, 2_000);
|
||||
assert_eq!(
|
||||
usage.server_tool_use.and_then(|s| s.web_search_requests),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(usage.inference_geo.as_deref(), Some("us"));
|
||||
|
||||
let hidden = serde_json::from_value::<HiddenParamsInput>(rows[0]["hidden_params"].clone())
|
||||
.expect("hidden params deserialize");
|
||||
assert_eq!(hidden.custom_llm_provider.as_deref(), Some("openai"));
|
||||
assert_eq!(hidden.region_name.as_deref(), Some("us-west-2"));
|
||||
assert_eq!(
|
||||
hidden.litellm_model_name.as_deref(),
|
||||
Some("synthetic-chat-alias")
|
||||
);
|
||||
let header = hidden
|
||||
.additional_headers
|
||||
.as_ref()
|
||||
.and_then(|headers| headers.get("llm_provider-x-litellm-response-cost"))
|
||||
.cloned();
|
||||
assert_eq!(header.and_then(WireNumber::cost), Some(0.0123));
|
||||
assert_eq!(
|
||||
hidden
|
||||
.provider_specific_fields
|
||||
.and_then(|fields| fields.traffic_type),
|
||||
Some("batch".to_string())
|
||||
);
|
||||
|
||||
let optional =
|
||||
serde_json::from_value::<OptionalParamsInput>(rows[0]["optional_params"].clone())
|
||||
.expect("optional params deserialize");
|
||||
assert_eq!(optional.service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(optional.query.map(QueryCount::get), Some(2));
|
||||
|
||||
let realtime = serde_json::from_value::<ResponseInput>(rows[1]["response"].clone())
|
||||
.expect("realtime response deserializes");
|
||||
assert_eq!(realtime.results.len(), 1);
|
||||
let result_response = realtime.results[0]
|
||||
.response
|
||||
.as_ref()
|
||||
.expect("response present");
|
||||
assert_eq!(result_response.service_tier.as_deref(), Some("priority"));
|
||||
let UsageInput::Responses { usage } = result_response.usage.clone().expect("usage present")
|
||||
else {
|
||||
panic!("realtime nested usage is responses-shaped");
|
||||
};
|
||||
assert_eq!(usage.input_tokens, 500);
|
||||
assert_eq!(
|
||||
usage.input_token_details.as_ref().map(|d| d.cached_tokens),
|
||||
Some(100)
|
||||
);
|
||||
|
||||
let image = serde_json::from_value::<ImageResponseInput>(rows[2]["response"].clone())
|
||||
.expect("image response deserializes");
|
||||
assert_eq!(image.data.get(), 2);
|
||||
let usage = image.usage.expect("image usage present");
|
||||
assert_eq!(
|
||||
(usage.input_tokens, usage.output_tokens, usage.total_tokens,),
|
||||
(100, 50, 150)
|
||||
);
|
||||
assert_eq!(
|
||||
usage
|
||||
.input_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|d| d.image_tokens),
|
||||
Some(60)
|
||||
);
|
||||
assert_eq!(
|
||||
usage
|
||||
.output_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|d| d.image_tokens),
|
||||
Some(40)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_usage_input_leniently_accepts_bool_string_and_integral_float_counts() {
|
||||
let usage: ChatUsageInput = serde_json::from_value(serde_json::json!({
|
||||
"prompt_tokens": true,
|
||||
"completion_tokens": " 42 ",
|
||||
"total_tokens": 58.0,
|
||||
"cache_read_input_tokens": null,
|
||||
}))
|
||||
.expect("lenient counts parse");
|
||||
assert_eq!(usage.prompt_tokens, 1);
|
||||
assert_eq!(usage.completion_tokens, 42);
|
||||
assert_eq!(usage.total_tokens, 58);
|
||||
assert_eq!(usage.cache_read_input_tokens, None);
|
||||
let _ = DataLen::default();
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue