mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-09 03:20:56 +00:00
fix(llm): capture Venice top-level costs
This commit is contained in:
parent
e5e7875274
commit
68e3cb8419
6 changed files with 172 additions and 5 deletions
|
|
@ -63,7 +63,9 @@ pub(super) fn decode_response(
|
|||
|
||||
let wire_usage = api_resp.usage.as_ref();
|
||||
let usage = wire_usage.map_or_else(TokenCounts::default, ApiUsage::token_counts);
|
||||
let cost_usd = wire_usage.and_then(|u| u.cost);
|
||||
let cost_usd = wire_usage
|
||||
.and_then(|usage| usage.cost)
|
||||
.or_else(|| api_resp.cost.as_ref().and_then(|cost| cost.usd));
|
||||
let cost_source = translate::authoritative_cost_source(cost_usd);
|
||||
|
||||
Ok(Response {
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ pub(super) struct StreamState {
|
|||
/// True after `finish_events()` has run (guards against duplicates).
|
||||
finished: bool,
|
||||
rate_limit: Option<RateLimitInfo>,
|
||||
/// In-band USD cost from the usage chunk (OpenRouter), surfaced as
|
||||
/// authoritative on the final response.
|
||||
/// In-band USD cost from the response, surfaced as authoritative on the
|
||||
/// final response.
|
||||
cost_usd: Option<f64>,
|
||||
}
|
||||
|
||||
|
|
@ -72,9 +72,13 @@ impl StreamState {
|
|||
// Capture usage if present (often in a dedicated chunk).
|
||||
if let Some(usage) = &chunk.usage {
|
||||
self.usage = usage.token_counts();
|
||||
// Keep a previously seen cost when a later usage chunk omits it.
|
||||
self.cost_usd = usage.cost.or(self.cost_usd);
|
||||
}
|
||||
let cost_usd = chunk
|
||||
.usage
|
||||
.as_ref()
|
||||
.and_then(|usage| usage.cost)
|
||||
.or_else(|| chunk.cost.as_ref().and_then(|cost| cost.usd));
|
||||
self.cost_usd = cost_usd.or(self.cost_usd);
|
||||
|
||||
let choices = chunk.choices.as_mut()?;
|
||||
let choice = choices.first_mut()?;
|
||||
|
|
|
|||
|
|
@ -125,6 +125,12 @@ pub(super) struct ApiResponse {
|
|||
pub model: String,
|
||||
pub choices: Vec<ApiChoice>,
|
||||
pub usage: Option<ApiUsage>,
|
||||
pub cost: Option<ApiCost>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(super) struct ApiCost {
|
||||
pub usd: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
|
|
@ -377,6 +383,7 @@ pub(super) struct StreamChunk {
|
|||
pub model: Option<String>,
|
||||
pub choices: Option<Vec<StreamChoice>>,
|
||||
pub usage: Option<ApiUsage>,
|
||||
pub cost: Option<ApiCost>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
|
|
|
|||
|
|
@ -795,6 +795,31 @@ async fn decode_usage_openrouter_cost_and_cache_write() {
|
|||
fabro_test::fabro_json_snapshot!(response);
|
||||
}
|
||||
|
||||
/// Venice reports authoritative USD cost in a top-level object rather than
|
||||
/// the OpenRouter `usage.cost` field.
|
||||
#[tokio::test]
|
||||
async fn decode_usage_venice_top_level_cost() {
|
||||
let response = decode_response(serde_json::json!({
|
||||
"id": "chatcmpl_venice_test",
|
||||
"object": "chat.completion",
|
||||
"created": CREATED_TS,
|
||||
"model": MODEL,
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop"
|
||||
}],
|
||||
"cost": {"usd": 0.00042, "diem": 0.0},
|
||||
"usage": {
|
||||
"prompt_tokens": 12,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 14
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
fabro_test::fabro_json_snapshot!(response);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stream
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -841,6 +866,20 @@ async fn stream_usage_openrouter_cost() {
|
|||
fabro_test::fabro_json_snapshot!(events);
|
||||
}
|
||||
|
||||
/// Venice streams authoritative USD cost in a top-level object on the usage
|
||||
/// chunk.
|
||||
#[tokio::test]
|
||||
async fn stream_usage_venice_top_level_cost() {
|
||||
let sse = support::sse_data_transcript(&[
|
||||
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}"#,
|
||||
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#,
|
||||
r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"cost":{"usd":0.00031,"diem":0.0},"usage":{"prompt_tokens":12,"completion_tokens":2,"total_tokens":14}}"#,
|
||||
"[DONE]",
|
||||
]);
|
||||
let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await;
|
||||
fabro_test::fabro_json_snapshot!(events);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_tool_call_deltas() {
|
||||
let sse = support::sse_data_transcript(&[
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
---
|
||||
source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs
|
||||
expression: rendered
|
||||
---
|
||||
{
|
||||
"id": "chatcmpl_venice_test",
|
||||
"model": "test-model",
|
||||
"provider": "openai-compatible",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"kind": "text",
|
||||
"data": "ok"
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 2,
|
||||
"reasoning_tokens": 0,
|
||||
"cache_read_tokens": 0,
|
||||
"cache_write_tokens": 0
|
||||
},
|
||||
"raw": {
|
||||
"id": "chatcmpl_venice_test",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "test-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "ok"
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"cost": {
|
||||
"usd": 0.00042,
|
||||
"diem": 0.0
|
||||
},
|
||||
"usage": {
|
||||
"prompt_tokens": 12,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 14
|
||||
}
|
||||
},
|
||||
"warnings": [],
|
||||
"rate_limit": null,
|
||||
"cost_usd": 0.00042,
|
||||
"cost_source": "authoritative"
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
---
|
||||
source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs
|
||||
expression: rendered
|
||||
---
|
||||
[
|
||||
{
|
||||
"type": "stream_start"
|
||||
},
|
||||
{
|
||||
"type": "text_start",
|
||||
"text_id": null
|
||||
},
|
||||
{
|
||||
"type": "text_delta",
|
||||
"delta": "Hi",
|
||||
"text_id": null
|
||||
},
|
||||
{
|
||||
"type": "text_end",
|
||||
"text_id": null
|
||||
},
|
||||
{
|
||||
"type": "finish",
|
||||
"finish_reason": "stop",
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 2,
|
||||
"reasoning_tokens": 0,
|
||||
"cache_read_tokens": 0,
|
||||
"cache_write_tokens": 0
|
||||
},
|
||||
"response": {
|
||||
"id": "chatcmpl_venice_stream",
|
||||
"model": "test-model",
|
||||
"provider": "openai-compatible",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"kind": "text",
|
||||
"data": "Hi"
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 2,
|
||||
"reasoning_tokens": 0,
|
||||
"cache_read_tokens": 0,
|
||||
"cache_write_tokens": 0
|
||||
},
|
||||
"raw": null,
|
||||
"warnings": [],
|
||||
"rate_limit": null,
|
||||
"cost_usd": 0.00031,
|
||||
"cost_source": "authoritative"
|
||||
}
|
||||
}
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue