fix(llm): capture Venice top-level costs

This commit is contained in:
Bryan Helmkamp 2026-08-21 15:55:45 -04:00
parent e5e7875274
commit 68e3cb8419
No known key found for this signature in database
6 changed files with 172 additions and 5 deletions

View file

@ -63,7 +63,9 @@ pub(super) fn decode_response(
let wire_usage = api_resp.usage.as_ref();
let usage = wire_usage.map_or_else(TokenCounts::default, ApiUsage::token_counts);
let cost_usd = wire_usage.and_then(|u| u.cost);
let cost_usd = wire_usage
.and_then(|usage| usage.cost)
.or_else(|| api_resp.cost.as_ref().and_then(|cost| cost.usd));
let cost_source = translate::authoritative_cost_source(cost_usd);
Ok(Response {

View file

@ -29,8 +29,8 @@ pub(super) struct StreamState {
/// True after `finish_events()` has run (guards against duplicates).
finished: bool,
rate_limit: Option<RateLimitInfo>,
/// In-band USD cost from the usage chunk (OpenRouter), surfaced as
/// authoritative on the final response.
/// In-band USD cost from the response, surfaced as authoritative on the
/// final response.
cost_usd: Option<f64>,
}
@ -72,9 +72,13 @@ impl StreamState {
// Capture usage if present (often in a dedicated chunk).
if let Some(usage) = &chunk.usage {
self.usage = usage.token_counts();
// Keep a previously seen cost when a later usage chunk omits it.
self.cost_usd = usage.cost.or(self.cost_usd);
}
let cost_usd = chunk
.usage
.as_ref()
.and_then(|usage| usage.cost)
.or_else(|| chunk.cost.as_ref().and_then(|cost| cost.usd));
self.cost_usd = cost_usd.or(self.cost_usd);
let choices = chunk.choices.as_mut()?;
let choice = choices.first_mut()?;

View file

@ -125,6 +125,12 @@ pub(super) struct ApiResponse {
pub model: String,
pub choices: Vec<ApiChoice>,
pub usage: Option<ApiUsage>,
pub cost: Option<ApiCost>,
}
#[derive(serde::Deserialize)]
pub(super) struct ApiCost {
pub usd: Option<f64>,
}
#[derive(serde::Deserialize)]
@ -377,6 +383,7 @@ pub(super) struct StreamChunk {
pub model: Option<String>,
pub choices: Option<Vec<StreamChoice>>,
pub usage: Option<ApiUsage>,
pub cost: Option<ApiCost>,
}
#[derive(serde::Deserialize)]

View file

@ -795,6 +795,31 @@ async fn decode_usage_openrouter_cost_and_cache_write() {
fabro_test::fabro_json_snapshot!(response);
}
/// Venice reports authoritative USD cost in a top-level object rather than
/// the OpenRouter `usage.cost` field.
#[tokio::test]
async fn decode_usage_venice_top_level_cost() {
let response = decode_response(serde_json::json!({
"id": "chatcmpl_venice_test",
"object": "chat.completion",
"created": CREATED_TS,
"model": MODEL,
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop"
}],
"cost": {"usd": 0.00042, "diem": 0.0},
"usage": {
"prompt_tokens": 12,
"completion_tokens": 2,
"total_tokens": 14
}
}))
.await;
fabro_test::fabro_json_snapshot!(response);
}
// ---------------------------------------------------------------------------
// Stream
// ---------------------------------------------------------------------------
@ -841,6 +866,20 @@ async fn stream_usage_openrouter_cost() {
fabro_test::fabro_json_snapshot!(events);
}
/// Venice streams authoritative USD cost in a top-level object on the usage
/// chunk.
#[tokio::test]
async fn stream_usage_venice_top_level_cost() {
let sse = support::sse_data_transcript(&[
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}"#,
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#,
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"cost":{"usd":0.00031,"diem":0.0},"usage":{"prompt_tokens":12,"completion_tokens":2,"total_tokens":14}}"#,
"[DONE]",
]);
let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await;
fabro_test::fabro_json_snapshot!(events);
}
#[tokio::test]
async fn stream_tool_call_deltas() {
let sse = support::sse_data_transcript(&[

View file

@ -0,0 +1,55 @@
---
source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs
expression: rendered
---
{
"id": "chatcmpl_venice_test",
"model": "test-model",
"provider": "openai-compatible",
"message": {
"role": "assistant",
"content": [
{
"kind": "text",
"data": "ok"
}
]
},
"finish_reason": "stop",
"usage": {
"input_tokens": 12,
"output_tokens": 2,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"raw": {
"id": "chatcmpl_venice_test",
"object": "chat.completion",
"created": 1700000000,
"model": "test-model",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "ok"
},
"finish_reason": "stop"
}
],
"cost": {
"usd": 0.00042,
"diem": 0.0
},
"usage": {
"prompt_tokens": 12,
"completion_tokens": 2,
"total_tokens": 14
}
},
"warnings": [],
"rate_limit": null,
"cost_usd": 0.00042,
"cost_source": "authoritative"
}

View file

@ -0,0 +1,60 @@
---
source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs
expression: rendered
---
[
{
"type": "stream_start"
},
{
"type": "text_start",
"text_id": null
},
{
"type": "text_delta",
"delta": "Hi",
"text_id": null
},
{
"type": "text_end",
"text_id": null
},
{
"type": "finish",
"finish_reason": "stop",
"usage": {
"input_tokens": 12,
"output_tokens": 2,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"response": {
"id": "chatcmpl_venice_stream",
"model": "test-model",
"provider": "openai-compatible",
"message": {
"role": "assistant",
"content": [
{
"kind": "text",
"data": "Hi"
}
]
},
"finish_reason": "stop",
"usage": {
"input_tokens": 12,
"output_tokens": 2,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"raw": null,
"warnings": [],
"rate_limit": null,
"cost_usd": 0.00031,
"cost_source": "authoritative"
}
}
]