diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 9ed13ae5ed5..f65516265c5 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -88,7 +88,8 @@ "call_type" ], "stream_opened": [ - "logger" + "logger", + "head" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 21f563d9f3b..70b7ecb45fd 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -536,7 +536,7 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } - Streaming::Opened.call(py, (self.logger()?.object(py),))?; + Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() { Python::initialize(); Python::attach(|py| { - let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None"); + let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}"); let mut logging = LegacyLogging { operation: crate::LoggingOperation::Messages, ..logged(py, &locals, true) }; logging - .on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind()) + .on_stream_open(py, &local(&locals, "head").unbind()) .unwrap(); logging .on_stream_chunk(py, &local(&locals, "first").unbind()) @@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response &locals, c" assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.calls[0][1] is head chunks = logger.calls[1][1] assert len(chunks) == 2 assert chunks[0] is first diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 46c93369100..f252a45b562 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -84,7 +84,7 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger: logger.record('stream_opened', None), + 'stream_opened': lambda logger, head: logger.record('stream_opened', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index f308714146e..a5526a39963 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -298,6 +298,7 @@ mod tests { fn row(span_id: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: String::new(), name: "run".into(), diff --git a/litellm-rust/crates/traces-cache/src/spend.rs b/litellm-rust/crates/traces-cache/src/spend.rs index 1cfcde841d0..6ea2cc79257 100644 --- a/litellm-rust/crates/traces-cache/src/spend.rs +++ b/litellm-rust/crates/traces-cache/src/spend.rs @@ -48,6 +48,7 @@ pub(super) async fn spend( let params = SpendByResponseIdsParams { access: access.clone(), response_ids: lookup.response_ids, + provider_request_ids: lookup.provider_request_ids, request_ids: lookup.request_ids, trace_ids: lookup.trace_ids, start_ms: window.start, diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index d714d7c9ed4..3f76d559a2b 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -260,6 +260,7 @@ fn access() -> ReadAccessParams { fn span(index: usize) -> TraceSpansRow { TraceSpansRow { trace_id: "trace".into(), + original_trace_id: String::new(), span_id: format!("span-{index}"), parent_span_id: if index == 0 { String::new() @@ -523,6 +524,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { litellm_call_id: String::new(), response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), team_id: "team".into(), diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index 42085d3e95b..0a206004080 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120); fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), diff --git a/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql new file mode 100644 index 00000000000..443ad0688e6 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql @@ -0,0 +1,4 @@ +ALTER TABLE {database}.spend_logs + ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id, + ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id + TYPE bloom_filter(0.001) GRANULARITY 1 diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql index 3918286a61f..55f8fe88a88 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -15,7 +15,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql index da64dafbc39..e6402fff6cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql @@ -1,4 +1,4 @@ -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -14,7 +14,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql index 679edfdec2e..d6ab416dfd3 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql index b81bba61e7c..27a8e2ac0da 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql index 967ef2fcf1f..a084325feaa 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index 2e0ac4f6dfb..306692e9709 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/named.rs b/litellm-rust/crates/traces-clickhouse/src/query/named.rs index beb4b42f76d..b7e70632480 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/named.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/named.rs @@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams; struct TraceSpansRowEncoding { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding { #[serde(flatten)] pub access: contracts::ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, #[serde(deserialize_with = "super::number::deserialize")] @@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, @@ -344,7 +349,7 @@ mod tests { quoted, ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), quoted, ); round_trip::( @@ -356,7 +361,7 @@ mod tests { quoted, ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), quoted, ); } @@ -374,7 +379,7 @@ mod tests { quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), quoted, ); } diff --git a/litellm-rust/crates/traces-clickhouse/src/span_row.rs b/litellm-rust/crates/traces-clickhouse/src/span_row.rs index 0189dbff2c8..ad2863d6c5d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/span_row.rs +++ b/litellm-rust/crates/traces-clickhouse/src/span_row.rs @@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str { .flatten() .find_map(|key| match key { CallKey::ProviderResponse(id) => Some(id.as_str()), - CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, + CallKey::LiteLlmRequest(_) + | CallKey::ProviderRequest(_) + | CallKey::Transport + | CallKey::GatewayAttempt => None, }) .unwrap_or_default() } diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index d90c366118b..e51d9083c59 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins( Parameter::Strings(vec!["response-1".into()]), ), ("request_ids".into(), Parameter::Strings(Vec::new())), + ( + "provider_request_ids".into(), + Parameter::Strings(Vec::new()), + ), ("trace_ids".into(), Parameter::Strings(Vec::new())), ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), @@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility( "api_key_hash": legacy_key.unwrap_or_default(), }))?, response_ids: vec!["shared-response".into()], + provider_request_ids: Vec::new(), request_ids: Vec::new(), trace_ids: Vec::new(), start_ms: timestamp / 1_000_000 - 1, diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs index 371424c63d6..7532d9aecc9 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs @@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap Some(id.as_str()), - litellm_traces::CallKey::Transport + litellm_traces::CallKey::ProviderRequest(_) + | litellm_traces::CallKey::Transport | litellm_traces::CallKey::GatewayAttempt => None, }) .unwrap_or_default() diff --git a/litellm-rust/crates/traces-clickhouse/tests/reads.rs b/litellm-rust/crates/traces-clickhouse/tests/reads.rs index e1996d6af04..b3d1b72bec1 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/reads.rs @@ -601,7 +601,7 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( #[rstest] #[tokio::test] -async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback( +async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads( #[future(awt)] migrated_database: TestResult, ) -> TestResult { let fixture = migrated_database?; @@ -610,31 +610,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let start_ms = 1_790_000_000_000_i64; let cases = [ ( - "gateway", - "provider-request", - "gateway", + "same-key", + "provider_response:same-key", "team-a", "key-a", Some(0.25), ), - ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)), - ("conflict", "conflict", "different", "team-a", "key-a", None), + ( + "other-key", + "provider_response:other-key", + "team-a", + "key-b", + None, + ), ( "foreign-team", - "request", - "foreign-team", + "provider_response:foreign-team", "team-b", "key-a", None, ), ( - "foreign-key", - "request", - "foreign-key", + "call-id-other-key", + "litellm_request:call-id-other-key", "team-a", "key-b", None, ), + ( + "call-id-foreign-team", + "litellm_request:call-id-foreign-team", + "team-b", + "key-a", + None, + ), + ("transport-only", "transport:", "team-a", "key-a", None), ]; insert_rows( client, @@ -643,7 +653,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::OtelTraces, cases .iter() - .map(|(id, _, _, _, _, _)| { + .map(|(id, key, _, _, _)| { BTreeMap::from([ ("Timestamp".into(), json!(start_ms * 1_000_000)), ("Duration".into(), json!(1_000_000)), @@ -652,7 +662,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback ("ObservationType".into(), json!("llm")), ("TeamId".into(), json!("team-a")), ("ApiKeyHash".into(), json!("key-a")), - ("CallKeys".into(), json!([format!("litellm_request:{id}")])), + ("CallKeys".into(), json!([key])), ("CallEvidence".into(), json!("complete")), ]) }) @@ -666,11 +676,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::SpendLogs, cases .iter() - .map(|(_, request, call_id, team, key, _)| { + .map(|(id, _, team, key, _)| { BTreeMap::from([ - ("request_id".into(), json!(request)), - ("response_id".into(), json!("provider-response")), - ("litellm_call_id".into(), json!(call_id)), + ("request_id".into(), json!(format!("request-{id}"))), + ("response_id".into(), json!(id)), + ("litellm_call_id".into(), json!(id)), ("team_id".into(), json!(team)), ("api_key".into(), json!(key)), ("start_time".into(), json!(start_ms)), @@ -689,13 +699,13 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let access = ReadAccessParams { all_teams: false, user_id: String::new(), - team_ids: vec!["team-a".into()], + team_ids: vec!["team-a".into(), "team-b".into()], }; let page = reader .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) .await?; assert_eq!(page.data.len(), cases.len()); - for (id, _, _, _, _, expected) in cases { + for (id, _, _, _, expected) in cases { let summary = page .data .iter() @@ -707,6 +717,122 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .ok_or("missing trace")?; assert_eq!(detail.summary.spend, expected, "{id}"); assert_eq!(summary.spend, expected, "{id}"); + assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}"); } Ok(()) } + +#[rstest] +#[case::provider_request(true)] +#[case::transport(false)] +#[tokio::test] +async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners( + #[future(awt)] migrated_database: TestResult, + #[case] response_header: bool, + #[values(false, true)] grouped: bool, +) -> TestResult { + let fixture = migrated_database?; + let client = &fixture.database.client; + let writer = Connection::writer(&fixture.database.url)?; + let trace_id = if grouped { + "grouped-trace" + } else { + "original-trace" + }; + let start_ms = 1_790_000_000_000_i64; + let keys = if response_header { + vec!["provider_response:req_native"] + } else { + Vec::new() + }; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::OtelTraces, + vec![BTreeMap::from([ + ("Timestamp".into(), json!(start_ms * 1_000_000)), + ("TraceId".into(), json!(trace_id)), + ("SpanId".into(), json!("native-call")), + ("SpanName".into(), json!("claude_code.llm_request")), + ("ObservationType".into(), json!("llm")), + ("Framework".into(), json!("claude-code")), + ("TeamId".into(), json!("team-a")), + ("ApiKeyHash".into(), json!("key-a")), + ("CallKeys".into(), json!(keys)), + ( + "CallEvidence".into(), + json!(if response_header { + "complete" + } else { + "unknown" + }), + ), + ( + "SpanAttributes".into(), + json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}), + ), + ])], + ) + .await?; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::SpendLogs, + ["key-a", "key-b"] + .into_iter() + .map(|key| { + BTreeMap::from([ + ("request_id".into(), json!(format!("log-{key}"))), + ("response_id".into(), json!("msg_native")), + ("provider_request_id".into(), json!("req_native")), + ( + "trace_id".into(), + json!(if response_header { + "" + } else { + "original-trace" + }), + ), + ( + "span_id".into(), + json!(if response_header { "" } else { "native-call" }), + ), + ("team_id".into(), json!("team-a")), + ("api_key".into(), json!(key)), + ("start_time".into(), json!(start_ms)), + ("end_time".into(), json!(start_ms + 1)), + ("spend".into(), json!(0.25)), + ]) + }) + .collect(), + ) + .await?; + let connection = fixture + .readers + .connection(client, &QueryScope::All, "fixture-secret") + .await?; + let (reader, store) = make_reader(client, connection); + let access = ReadAccessParams { + all_teams: true, + user_id: String::new(), + team_ids: Vec::new(), + }; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; + assert_eq!(page.data.len(), 1); + let summary = &page.data[0]; + let detail = reader + .get_trace(&store, &access, trace_id, &summary.trace_ref) + .await? + .ok_or("missing trace")?; + assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1)); + assert_eq!(detail.summary.spend, summary.spend); + assert_eq!( + detail.spans[0].spend_log_request_id.as_deref(), + Some("log-key-a") + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index f79fa59bdba..b51701638e8 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -44,5 +44,6 @@ pub use tenant::Tenant; pub use truncate::{truncate_messages, truncate_value}; pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content}; pub use view::{ - AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary, + AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage, + TraceSummary, }; diff --git a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs index 18d7d74dc44..d24b9e6d8d2 100644 --- a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs +++ b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs @@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts}; use crate::{ Error, normalize::{ - CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey, + CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, ObservationType, RoleEvidence, SpanContext, attr, present, tokens, }, otlp::DecodedEvent, @@ -265,7 +265,7 @@ impl Format for ClaudeCode { output: llm_output(attributes), calls: present(attributes, &["gen_ai.response.id", "request_id"]) .map_or(CallEvidence::Unknown, |id| { - CallEvidence::complete(CallKey::ProviderResponse(id)) + CallEvidence::complete(crate::normalize::claude_call_key(id)) }), ..base }, diff --git a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs index 550afae77ca..1003834de79 100644 --- a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs @@ -216,7 +216,16 @@ impl Instrumentation { fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts { let calls = [ - present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse), + present(context.attributes, &["gen_ai.response.id"]).map(|id| { + if matches!( + context.scope, + crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE + ) { + crate::normalize::claude_call_key(id) + } else { + CallKey::ProviderResponse(id) + } + }), present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest), ] .into_iter() diff --git a/litellm-rust/crates/traces/src/normalize/mod.rs b/litellm-rust/crates/traces/src/normalize/mod.rs index f2c5376af10..53d5e521799 100644 --- a/litellm-rust/crates/traces/src/normalize/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/mod.rs @@ -65,16 +65,26 @@ pub enum CallKey { LiteLlmRequest(String), /// The provider response id returned to the caller (`spend_logs.response_id`). ProviderResponse(String), + ProviderRequest(String), /// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id. Transport, GatewayAttempt, } +pub(crate) fn claude_call_key(id: String) -> CallKey { + if id.starts_with("msg_") { + CallKey::ProviderResponse(id) + } else { + CallKey::ProviderRequest(id) + } +} + impl fmt::Display for CallKey { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"), Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"), + Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"), Self::Transport => formatter.write_str("transport:"), Self::GatewayAttempt => formatter.write_str("gateway_attempt:"), } @@ -89,6 +99,9 @@ impl FromStr for CallKey { Some(("provider_response", id)) if !id.is_empty() => { Ok(Self::ProviderResponse(id.to_owned())) } + Some(("provider_request", id)) if !id.is_empty() => { + Ok(Self::ProviderRequest(id.to_owned())) + } Some(("litellm_request", id)) if !id.is_empty() => { Ok(Self::LiteLlmRequest(id.to_owned())) } @@ -127,24 +140,45 @@ pub enum CallEvidence { impl CallEvidence { pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet { - if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { + let native = Self::native_request(row); + let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())]) } else { row.call_keys.iter().cloned().collect() + }; + if !native { + return keys; } + if keys.is_empty() { + return BTreeSet::from([CallKey::Transport]); + } + keys.into_iter() + .map(|key| match key { + CallKey::ProviderResponse(id) => claude_call_key(id), + key => key, + }) + .collect() + } + + fn native_request(row: &crate::query::named::TraceSpansRow) -> bool { + matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk") + && row.name == "claude_code.llm_request" } pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self { - let kind = row - .call_evidence - .unwrap_or(if Self::row_keys(row).is_empty() { + let keys = Self::row_keys(row); + let kind = if Self::native_request(row) { + CallEvidenceKind::Complete + } else { + row.call_evidence.unwrap_or(if keys.is_empty() { CallEvidenceKind::Unknown } else { CallEvidenceKind::Complete - }); + }) + }; match kind { - CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)), - CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)), + CallEvidenceKind::Complete => Self::Complete(keys), + CallEvidenceKind::Partial => Self::Partial(keys), CallEvidenceKind::Unknown => Self::Unknown, } } diff --git a/litellm-rust/crates/traces/src/otlp/span.rs b/litellm-rust/crates/traces/src/otlp/span.rs index 4e63d78d286..58ccb9ea3b4 100644 --- a/litellm-rust/crates/traces/src/otlp/span.rs +++ b/litellm-rust/crates/traces/src/otlp/span.rs @@ -193,9 +193,9 @@ fn decoded_span( .into_iter() .flatten() .map(|key| match key { - crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => { - id.len() + size_of::() - } + crate::CallKey::LiteLlmRequest(id) + | crate::CallKey::ProviderResponse(id) + | crate::CallKey::ProviderRequest(id) => id.len() + size_of::(), crate::CallKey::Transport | crate::CallKey::GatewayAttempt => { size_of::() } diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index a981bc46471..757012de16f 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -68,6 +68,8 @@ pub struct TraceSpansParams { pub struct TraceSpansRow { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -113,6 +115,16 @@ pub struct TraceSpansRow { pub user_id: String, } +impl TraceSpansRow { + pub(crate) fn transport_trace_id(&self) -> &str { + if self.original_trace_id.is_empty() { + &self.trace_id + } else { + &self.original_trace_id + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TracePageSpansParams { #[serde(flatten)] @@ -163,6 +175,7 @@ pub struct SpendByResponseIdsParams { #[serde(flatten)] pub access: ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, pub start_ms: i64, @@ -175,6 +188,8 @@ pub struct SpendByResponseIdsRow { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, diff --git a/litellm-rust/crates/traces/src/resolve/resolution.rs b/litellm-rust/crates/traces/src/resolve/resolution.rs index 9b47755a16f..e19a98a1ede 100644 --- a/litellm-rust/crates/traces/src/resolve/resolution.rs +++ b/litellm-rust/crates/traces/src/resolve/resolution.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use indexmap::IndexMap; use crate::{ + SpendMatch, normalize::{CallKey, ObservationType}, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> { types: HashMap<&'a str, ObservationType>, tool_failures: HashMap<&'a str, &'a TraceSpansRow>, pub(super) model_calls: Vec, + call_matches: HashMap>, } +pub(super) type CallMatch<'a> = (Option>, SpendMatch); + impl<'a> Resolution<'a> { pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self { let graph = Graph::new(rows); @@ -36,7 +40,7 @@ impl<'a> Resolution<'a> { let types: HashMap<&str, ObservationType> = (0..rows.len()) .map(|index| (graph.id(index), resolved_type(&graph, index, named_agents))) .collect(); - let model_calls = (0..rows.len()) + let model_calls: Vec = (0..rows.len()) .filter(|index| { types[graph.id(*index)] == ObservationType::Llm && !graph @@ -45,7 +49,7 @@ impl<'a> Resolution<'a> { .any(|descendant| types[graph.id(descendant)] == ObservationType::Llm) }) .collect(); - Self { + let resolution = Self { ownership: Ownership { team_id: &rows[0].team_id, api_key_hash: &rows[0].api_key_hash, @@ -65,6 +69,16 @@ impl<'a> Resolution<'a> { .map(|row| (row.tool_call_id.as_str(), row)) .collect(), model_calls, + call_matches: HashMap::new(), + }; + let call_matches = resolution + .model_calls + .iter() + .map(|call| (*call, resolution.resolve_call_match(*call))) + .collect(); + Self { + call_matches, + ..resolution } } @@ -118,7 +132,24 @@ impl<'a> Resolution<'a> { spend::requests(self.row(index), &self.ownership, self.spend) } + pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> { + self.call_matches.get(&call) + } + pub(super) fn call_requests(&self, call: usize) -> Option> { + self.call_match(call) + .and_then(|(requests, _)| requests.clone()) + } + + fn resolve_call_match(&self, call: usize) -> CallMatch<'a> { + if let Some(requests) = self.resolve_call_requests(call) { + return (Some(requests), SpendMatch::Matched); + } + let evidence = self.requests(call); + (None, evidence.unmatched_reason()) + } + + fn resolve_call_requests(&self, call: usize) -> Option> { let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| { self.kind(*ancestor) == ObservationType::Llm && self diff --git a/litellm-rust/crates/traces/src/resolve/spend.rs b/litellm-rust/crates/traces/src/resolve/spend.rs index ce23362b7f8..a6b2fc0dbc5 100644 --- a/litellm-rust/crates/traces/src/resolve/spend.rs +++ b/litellm-rust/crates/traces/src/resolve/spend.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use indexmap::IndexMap; use crate::{ - CallEvidence, CallEvidenceKind, CallKey, + CallEvidence, CallEvidenceKind, CallKey, SpendMatch, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -12,6 +12,7 @@ use crate::{ pub struct SpendLookup { pub response_ids: Vec, pub request_ids: Vec, + pub provider_request_ids: Vec, /// Traces whose transport spans LiteLLM logged by `traceparent`. pub trace_ids: Vec, } @@ -45,13 +46,21 @@ impl SpendLookup { }) .collect(), ), + provider_request_ids: sorted( + keys() + .filter_map(|(_, key)| match key { + CallKey::ProviderRequest(id) => Some(id.clone()), + _ => None, + }) + .collect(), + ), trace_ids: sorted( keys() .filter_map(|(row, key)| match key { CallKey::Transport | CallKey::GatewayAttempt - if !row.trace_id.is_empty() => + if !row.transport_trace_id().is_empty() => { - Some(row.trace_id.clone()) + Some(row.transport_trace_id().to_owned()) } _ => None, }) @@ -61,7 +70,10 @@ impl SpendLookup { } pub fn is_empty(&self) -> bool { - self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty() + self.response_ids.is_empty() + && self.request_ids.is_empty() + && self.provider_request_ids.is_empty() + && self.trace_ids.is_empty() } } @@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>; enum KeyFamily { GatewayCall, ProviderResponse, + ProviderRequest, Transport, } @@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily { match key { CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall, CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse, + CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest, CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport, } } pub(super) enum KeyMatch<'a> { Missing, + Conflicting, Unique(&'a SpendRow), Ambiguous(Requests<'a>), } @@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> { fn unique(&self) -> Option<&'a SpendRow> { match self { Self::Unique(request) => Some(request), - Self::Missing | Self::Ambiguous(_) => None, + Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None, } } fn agrees_with(&self, selected: &[&SpendRow]) -> bool { match self { - Self::Missing => false, + Self::Missing | Self::Conflicting => false, Self::Unique(request) => selected .iter() .any(|row| row.identity() == request.identity()), @@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> { } impl<'a> SpendEvidence<'a> { + pub(super) fn unmatched_reason(&self) -> SpendMatch { + match self { + Self::Unknown => SpendMatch::NoCallId, + Self::Partial(_) => SpendMatch::IncompleteEvidence, + Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId, + Self::Complete(matches) + if matches.iter().any(|evidence| { + matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_)) + }) => + { + SpendMatch::Ambiguous + } + Self::Complete(matches) + if matches + .iter() + .any(|evidence| matches!(evidence, KeyMatch::Missing)) => + { + SpendMatch::NoSpendLog + } + Self::Complete(_) => SpendMatch::Ambiguous, + } + } + pub(super) fn complete_requests(&self) -> Option> { match self { Self::Complete(matches) if !matches.is_empty() => { @@ -186,15 +224,16 @@ fn matches<'a>( CallKey::ProviderResponse(id) => { !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id) } + CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id, CallKey::LiteLlmRequest(id) => { !id.is_empty() && (spend.litellm_call_id == *id || (spend.litellm_call_id.is_empty() && spend.request_id == *id)) } CallKey::Transport | CallKey::GatewayAttempt => { - !row.trace_id.is_empty() + !row.transport_trace_id().is_empty() && !row.span_id.is_empty() - && spend.trace_id == row.trace_id + && spend.trace_id == row.transport_trace_id() && spend.span_id == row.span_id } }; @@ -253,7 +292,8 @@ pub(super) fn requests<'a>( let matches = aliases .into_iter() .map(|(_, requests)| { - KeyMatch::new( + let had_candidates = !requests.is_empty(); + let matched = KeyMatch::new( requests .into_iter() .filter(|request| { @@ -262,7 +302,12 @@ pub(super) fn requests<'a>( .all(|family| family.contains(&request.identity())) }) .collect(), - ) + ); + if had_candidates && matches!(matched, KeyMatch::Missing) { + KeyMatch::Conflicting + } else { + matched + } }) .collect(); match evidence.kind() { @@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option { }) } -pub(super) fn total(calls: &[Option>]) -> Option { - if calls.is_empty() { - return None; - } - let requests: Option> = calls - .iter() - .map(|requests| requests.as_ref()) - .collect::>>() - .map(|calls| calls.into_iter().flatten().copied().collect()); - let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests? +pub(super) fn unique<'a>(requests: impl IntoIterator) -> Requests<'a> { + requests .into_iter() .map(|request| (request.identity(), request)) - .collect(); - request_cost(&unique.into_values().collect::>()) + .collect::>() + .into_values() + .collect() +} + +pub(super) struct Priced { + pub(super) spend: Option, + pub(super) priced_calls: u64, +} + +pub(super) fn total(calls: &[Option>]) -> Priced { + let priced: Vec<&Requests<'_>> = calls + .iter() + .flatten() + .filter(|requests| request_cost(requests).is_some()) + .collect(); + let spend = if priced.is_empty() { + None + } else { + request_cost(&unique( + priced.iter().flat_map(|requests| requests.iter().copied()), + )) + }; + Priced { + spend, + priced_calls: priced.len() as u64, + } } diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 87688d9b36e..9e9edd51bf4 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -6,7 +6,7 @@ use time::OffsetDateTime; use crate::{ normalize::ObservationType, query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow}, - view::{AgentNode, Span, SpanStatus, Trace, TraceSummary}, + view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary}, }; use super::{ @@ -23,7 +23,18 @@ fn optional(value: &str) -> Option { fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span { let row = resolution.row(index); let status = resolution.status_source(index); - let requests = resolution.requests(index).complete_requests(); + let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) { + (requests.clone(), Some(*matched)) + } else { + (resolution.requests(index).complete_requests(), None) + }; + let spend = requests + .as_ref() + .and_then(|requests| request_cost(requests)); + let spend_log_request_id = match (spend_match, requests.as_deref()) { + (Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()), + _ => None, + }; Span { span_id: row.span_id.clone(), parent_span_id: optional(&row.parent_span_id), @@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span input_tokens: row.input_tokens, output_tokens: row.output_tokens, litellm_request_id: optional(&row.litellm_request_id), - spend: requests - .as_ref() - .and_then(|requests| request_cost(requests)), + spend, + spend_log_request_id, + spend_match, } } @@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .filter(|(owner, _)| *owner == name) .map(|(_, requests)| requests.clone()) .collect(); + let priced = total(&owned_calls); AgentNode { name: name.to_owned(), parent_agent, @@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .map(|span| graph.rows[*span].duration_ns) .sum::() as f64 / NANOS_PER_MS, - spend: total(&owned_calls), + spend: priced.spend, + priced_calls: priced.priced_calls, } }) .collect() @@ -160,6 +173,12 @@ pub fn resolve_trace( } else { calls.iter().map(|call| &rows[*call]).collect() }; + let priced = total( + &calls + .iter() + .map(|call| resolution.call_requests(*call)) + .collect::>(), + ); let first_input = spans .iter() .zip(rows) @@ -208,12 +227,8 @@ pub fn resolve_trace( input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(), output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(), models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())), - spend: total( - &calls - .iter() - .map(|call| resolution.call_requests(*call)) - .collect::>(), - ), + spend: priced.spend, + priced_calls: priced.priced_calls, }; Some(Trace { summary, @@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary { output_tokens: row.output_tokens, models: row.models.clone(), spend: None, + priced_calls: 0, } } diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index a864c740526..5a214d33da7 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -37,6 +37,19 @@ pub struct Span { pub output_tokens: u32, pub litellm_request_id: Option, pub spend: Option, + pub spend_log_request_id: Option, + pub spend_match: Option, +} + +#[macro_rules_attribute::apply(response_type)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum SpendMatch { + Matched, + NoCallId, + NoSpendLog, + Ambiguous, + IncompleteEvidence, } /// One distinct agent in a trace: 200 invocations of `researcher` are one node. @@ -50,6 +63,7 @@ pub struct AgentNode { pub tool_calls: u64, pub duration_ms: f64, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] @@ -80,6 +94,7 @@ pub struct TraceSummary { pub output_tokens: u64, pub models: Vec, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] diff --git a/litellm-rust/crates/traces/tests/captures.rs b/litellm-rust/crates/traces/tests/captures.rs index ca498fc914a..ff20db617f6 100644 --- a/litellm-rust/crates/traces/tests/captures.rs +++ b/litellm-rust/crates/traces/tests/captures.rs @@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec TraceSpansRow { .into_iter() .flatten() .find_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()), CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, }) .unwrap_or_default(); TraceSpansRow { trace_id: span.trace_id, + original_trace_id: String::new(), span_id: span.span_id, parent_span_id: span.parent_span_id, name: span.name, @@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow { .expect("valid unrelated transport timestamp"); TraceSpansRow { trace_id: call.trace_id.clone(), + original_trace_id: call.original_trace_id.clone(), span_id: format!("unrelated-transport-{}", call.span_id), parent_span_id: call.parent_span_id.clone(), name: "unrelated-http".into(), @@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence( let response_ids: Vec<_> = keys .iter() .filter_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => { + Some(id.clone()) + } CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => { None } diff --git a/litellm-rust/crates/traces/tests/normalization_formats.rs b/litellm-rust/crates/traces/tests/normalization_formats.rs index 124f03593c8..e8b11ece64a 100644 --- a/litellm-rust/crates/traces/tests/normalization_formats.rs +++ b/litellm-rust/crates/traces/tests/normalization_formats.rs @@ -928,3 +928,28 @@ fn convention_markers_keep_genai_call_evidence( let marked = decode(span, scope, &marked_attributes, vec![]).unwrap(); assert_eq!(marked.normalized.calls, plain.normalized.calls); } + +#[rstest] +#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))] +#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))] +fn native_claude_preserves_the_provider_id_family( + span: Span, + #[case] id: &str, + #[case] key: CallKey, +) { + let native = Span { + name: "claude_code.llm_request".into(), + ..span + }; + let decoded = decode( + native, + "com.anthropic.claude_code.tracing", + &[("gen_ai.response.id", id)], + Vec::new(), + ) + .unwrap(); + assert_eq!( + decoded.normalized.calls, + CallEvidence::Complete(std::collections::BTreeSet::from([key])) + ); +} diff --git a/litellm-rust/crates/traces/tests/normalize.rs b/litellm-rust/crates/traces/tests/normalize.rs index 923e2006088..bfff82fcdd7 100644 --- a/litellm-rust/crates/traces/tests/normalize.rs +++ b/litellm-rust/crates/traces/tests/normalize.rs @@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) { #[rstest] #[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))] #[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))] +#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))] #[case::transport(litellm_traces::CallKey::Transport)] #[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)] fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { @@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { #[rstest] #[case::missing_separator("provider_response")] #[case::missing_response("provider_response:")] +#[case::missing_provider_request("provider_request:")] #[case::missing_request("litellm_request:")] #[case::transport_id("transport:unexpected")] #[case::gateway_attempt_separator("gateway_attempt")] diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 6b853ebf151..3ac9668e42a 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases( json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), )); round_trip::(request( - json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), )); } @@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() { json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), @@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() { json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}), ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), ); } diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index 641d15db5b9..8d65adb0c0f 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -1,5 +1,5 @@ use litellm_traces::{ - AgentNode, SpanStatus, iso_time, listed_summary, + AgentNode, SpanStatus, SpendMatch, iso_time, listed_summary, query::named::{ListTracesRow, SpendByResponseIdsRow, TraceSpansRow}, resolve_trace, }; @@ -11,6 +11,7 @@ const MS: i64 = 1_000_000; fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), @@ -32,8 +33,8 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac call_keys: Vec::new(), call_evidence: None, tool_call_id: String::new(), - team_id: String::new(), - api_key_hash: String::new(), + team_id: "team".into(), + api_key_hash: "key".into(), user_id: String::new(), } } @@ -55,31 +56,18 @@ fn llm(span_id: &str, parent: &str, agent: &str, response_id: &str) -> TraceSpan } } -fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { - span.team_id = team.into(); - span.user_id = user.into(); - span.api_key_hash = key.into(); - span -} - -fn spend( - request_id: &str, - response_id: &str, - team: &str, - user: &str, - key: &str, - cost: f64, -) -> SpendByResponseIdsRow { +fn spend(request_id: &str, response_id: &str, cost: f64) -> SpendByResponseIdsRow { SpendByResponseIdsRow { request_id: request_id.into(), - response_id: response_id.into(), litellm_call_id: String::new(), + response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), - team_id: team.into(), - api_key: key.into(), - user: user.into(), + api_key: "key".into(), + user: String::new(), + team_id: "team".into(), spend: Some(cost), start_ms: T0 / MS, } @@ -221,6 +209,7 @@ fn repeated_subagent_invocations_aggregate_into_one_node() { tool_calls: 1, duration_ms: 1000.0, spend: None, + priced_calls: 0, } ); let researcher = &trace.agents[1]; @@ -384,38 +373,15 @@ fn names_and_frameworks_are_sorted_and_distinct() { } #[rstest] -fn repeated_response_counts_once_and_other_owners_are_ignored() { +fn repeated_response_id_counts_once() { let rows = [ - owned( - row("root", "", "agent", "agent", "agent"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-1", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-2", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), + row("root", "", "agent", "agent", "agent"), + llm("llm-1", "root", "agent", "response-1"), + llm("llm-2", "root", "agent", "response-1"), ]; let spend = [ - spend("request-other", "response-1", "team-b", "", "key-b", 99.0), - spend("request-1", "response-1", "team-a", "", "key-a", 0.25), - spend( - "request-other-key", - "unrelated-response", - "team-a", - "", - "key-c", - 50.0, - ), + spend("request-1", "response-1", 0.25), + spend("request-other", "unrelated-response", 50.0), ]; let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); assert_eq!(trace.summary.spend, Some(0.25)); @@ -431,352 +397,363 @@ fn repeated_response_counts_once_and_other_owners_are_ignored() { } #[rstest] -fn ambiguous_response_id_keeps_cost_unknown() { - let rows = [owned( - llm("llm-1", "", "agent", "response-1"), - "", - "user", - "key-a", - )]; - let spend = [ - spend("response-1", "response-1", "", "user", "key-a", 0.25), - spend( - "response-1_cache_hit123", - "response-1", - "", - "user", - "key-a", - 0.0, - ), - ]; - let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); - assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); -} - -#[rstest] -#[case::key_differs("team", "", "export", "team", "", "request", false)] -#[case::shared_key("team", "", "export", "team", "", "export", true)] -#[case::shared_user("", "user", "export", "", "user", "request", true)] -#[case::teamless_key("", "", "key", "", "", "key", true)] -#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] -#[case::other_user("", "user", "export", "", "other-user", "request", false)] -#[case::no_shared_identity("", "", "export", "", "", "request", false)] -#[case::no_identity("", "", "", "", "", "", false)] -#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] -fn cost_requires_shared_ownership( - #[case] trace_team: &str, - #[case] trace_user: &str, - #[case] trace_key: &str, - #[case] spend_team: &str, - #[case] spend_user: &str, - #[case] spend_key: &str, - #[case] known: bool, -) { +fn model_calls_link_the_spend_log_they_were_priced_from() { let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - trace_team, - trace_user, - trace_key, - ), - owned( - llm("llm", "agent", "agent", "response"), - trace_team, - trace_user, - trace_key, - ), + row("agent", "", "agent", "agent", "agent"), + llm("matched", "agent", "agent", "matched"), + llm("no-log", "agent", "agent", "missing"), + llm("no-id", "agent", "agent", ""), + llm("ambiguous", "agent", "agent", "cached"), ]; - let spend = [spend( - "request", "response", spend_team, spend_user, spend_key, 0.25, - )]; - let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); - let expected = known.then_some(0.25); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[1].spend, expected); -} - -#[rstest] -#[case::missing_id("missing_id")] -#[case::missing_spend("missing_spend")] -#[case::duplicate_spend("duplicate_spend")] -fn incomplete_call_cost_never_becomes_a_partial_total(#[case] failure: &str) { - let second_id = if failure == "missing_id" { - "" - } else { - "second" - }; - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "export", - ), - owned( - llm("first", "agent", "agent", "first"), - "team", - "", - "export", - ), - owned( - llm("second", "agent", "agent", second_id), - "team", - "", - "export", - ), + let logs = [ + spend("request-matched", "matched", 0.25), + spend("request-cached-a", "cached", 0.25), + spend("request-cached-b", "cached", 0.0), ]; - let first = spend("first", "first", "team", "", "export", 0.25); - let second = spend("second", "second", "team", "", "export", 0.25); - let duplicate = spend("duplicate", "second", "team", "", "export", 0.25); - let spend = if failure == "duplicate_spend" { - vec![first, second, duplicate] - } else { - vec![first] - }; - let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); - assert_eq!(trace.spans[1].spend, Some(0.25)); - assert_eq!(trace.spans[2].spend, None); - assert_eq!(trace.summary.spend, None); - assert_eq!(trace.agents[0].spend, None); -} - -#[rstest] -fn transport_spans_complete_a_call_without_its_own_id() { - let mut transport = row("http", "llm", "POST", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec!["transport:".parse().unwrap()]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, Some(0.5)); -} - -#[rstest] -#[case::lone_call(1, Some(0.5))] -#[case::two_calls(2, None)] -fn sibling_transports_belong_to_the_only_model_call_under_their_parent( - #[case] calls: usize, - #[case] expected: Option, -) { - let mut transport = at( - row("http", "step", "gateway.request", "framework", ""), - 2, - 10, + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + let links: Vec<_> = trace + .spans + .iter() + .map(|span| (span.spend_log_request_id.as_deref(), span.spend_match)) + .collect(); + assert_eq!( + links, + [ + (None, None), + (Some("request-matched"), Some(SpendMatch::Matched)), + (None, Some(SpendMatch::NoSpendLog)), + (None, Some(SpendMatch::NoCallId)), + (None, Some(SpendMatch::Ambiguous)), + ] ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut rows = vec![ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - for index in 0..calls { - let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); - call.call_evidence = None; - rows.push(owned(call, "team", "", "key")); - } - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] -fn sibling_transport_does_not_lose_model_call_spend( - #[case] transport_timing: Option<(i64, u64)>, - #[case] unrelated_spend: bool, +fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_a_wrapper() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", ""), + ]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-request", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!( + ( + trace.spans[1].spend, + trace.spans[1].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn unpriced_call_leaves_a_partial_total_of_the_priced_calls() { + let rows = [ + row("agent", "", "agent", "agent", "agent"), + llm("first", "agent", "agent", "first"), + llm("second", "agent", "agent", "second"), + llm("third", "agent", "agent", ""), + ]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("first", "first", 0.25)]).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.summary.priced_calls, + trace.summary.llm_calls + ), + (Some(0.25), 1, 3) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), None, None] + ); +} + +#[rstest] +fn no_priced_call_leaves_cost_unknown() { + let rows = [llm("call", "", "agent", "response")]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("other", "other", 0.25)]).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +fn gateway_logged(request_id: &str, call_id: &str, cost: f64) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..spend(request_id, &format!("chatcmpl-{request_id}"), cost) + } +} + +#[rstest] +#[case::response_id(litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), Some(0.25))] +#[case::gateway_call_id(litellm_traces::CallKey::LiteLlmRequest("gateway".into()), Some(0.25))] +#[case::other_call_id(litellm_traces::CallKey::LiteLlmRequest("other".into()), None)] +#[case::request_id_is_not_a_call_id(litellm_traces::CallKey::LiteLlmRequest("request".into()), None)] +#[case::transport(litellm_traces::CallKey::Transport, None)] +#[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt, None)] +fn only_ids_litellm_assigned_join_spend( #[case] key: litellm_traces::CallKey, #[case] expected: Option, ) { - let call = owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![litellm_traces::CallKey::ProviderResponse( - "chatcmpl-1".into(), - )], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("chat", "step", "agent", "chatcmpl-1") - }, - "team", - "", - "key", + let rows = [TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![key], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "trace".into(), + ..gateway_logged("request", "gateway", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (expected, u64::from(expected.is_some())) ); - let base_rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - call, - ]; - let rows: Vec<_> = base_rows - .into_iter() - .chain(transport_timing.map(|(start, duration)| { - let mut transport = at( - row("tool-http", "step", "GET", "framework", ""), - start, - duration, - ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![key]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - owned(transport, "team", "", "key") - })) - .collect(); - let logs: Vec<_> = std::iter::once(spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5)) - .chain(unrelated_spend.then(|| SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "tool-http".into(), - ..spend("unrelated", "unrelated", "team", "", "key", 0.75) - })) - .collect(); - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::agreeing_ids( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] -#[case::conflicting_response_id( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-b"), - None -)] -#[case::conflicting_gateway_and_response( - litellm_traces::CallKey::Transport, - "call-b", - Some("response-b"), - None -)] -#[case::agreeing_gateway_attempt( - litellm_traces::CallKey::GatewayAttempt, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] -fn gateway_attempt_identifiers_must_match_one_spend_row( - #[case] transport: litellm_traces::CallKey, - #[case] call_id: &str, - #[case] response_id: Option<&str>, +#[case::same_team("team", Some(0.25), SpendMatch::Matched)] +#[case::other_team("other-team", None, SpendMatch::NoSpendLog)] +fn spend_logs_price_only_calls_in_their_own_team( + #[case] logged_team: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let keys = [ - transport, - litellm_traces::CallKey::LiteLlmRequest(call_id.into()), - ] - .into_iter() - .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) - .collect(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(llm("call", "agent", "agent", ""), "team", "", "key"), - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: keys, - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..row("attempt", "call", "gateway.request", "framework", "") - }, - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - litellm_call_id: "call-a".into(), - trace_id: "trace".into(), - span_id: "attempt".into(), - ..spend("request-a", "response-a", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - litellm_call_id: "call-b".into(), - trace_id: "trace".into(), - span_id: "other-attempt".into(), - ..spend("request-b", "response-b", "team", "", "key", 0.5) - }, - ]; + let rows = [TraceSpansRow { + team_id: "team".into(), + ..llm("call", "", "agent", "response") + }]; + let logs = [SpendByResponseIdsRow { + team_id: logged_team.into(), + ..spend("request", "response", 0.25) + }]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[2].spend, expected); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); } #[rstest] -#[case::legacy_row("", Some(0.5))] -#[case::other_call("other-call", None)] -fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( +#[case::legacy_row_without_call_id("", Some(0.25), SpendMatch::Matched)] +#[case::call_id_names_nothing("missing", None, SpendMatch::Ambiguous)] +fn recorded_gateway_id_must_agree_unless_spend_predates_gateway_ids( #[case] logged_call_id: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let mut transport = row("http", "llm", "gateway.request", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec![ - "transport:".parse().unwrap(), - "litellm_request:gateway-call".parse().unwrap(), - ]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [gateway_logged("request", logged_call_id, 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.spans[0].spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); +} + +#[rstest] +#[case::missing_cost(None, SpendMatch::Matched)] +#[case::finite_cost(Some(0.25), SpendMatch::Matched)] +fn a_single_matched_row_is_matched_whatever_its_cost( + #[case] cost: Option, + #[case] matched: SpendMatch, +) { + let rows = [llm("call", "", "agent", "response")]; + let logs = [SpendByResponseIdsRow { + spend: cost, + ..spend("request", "response", 0.0) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.spans[0].spend_match, + trace.spans[0].spend_log_request_id.as_deref() ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), + (cost, Some(matched), Some("request")) + ); + assert_eq!(trace.summary.priced_calls, u64::from(cost.is_some())); +} + +#[rstest] +#[case::same_request("gateway", Some(0.25), SpendMatch::Matched)] +#[case::conflicting_requests("other", None, SpendMatch::Ambiguous)] +fn response_id_and_call_id_must_name_the_same_request( + #[case] call_id: &str, + #[case] expected: Option, + #[case] matched: SpendMatch, +) { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + gateway_logged("request", "gateway", 0.25), + gateway_logged("other-request", "other", 0.5), ]; - let mut logged = spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - logged.litellm_call_id = logged_call_id.into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.summary.spend, + trace.spans[0].spend_match + ), + (expected, expected, Some(matched)) + ); +} + +#[rstest] +fn call_id_picks_the_call_when_its_response_id_has_a_cache_hit_twin() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-shared".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-shared", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "cache-hit".into(), + ..spend("request_cache_hit", "chatcmpl-shared", 0.0) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.spans[0].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn wrapper_recording_a_retry_and_its_final_attempt_prices_both() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-retry".into()), + litellm_traces::CallKey::ProviderResponse("chatcmpl-final".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + spend("retry", "chatcmpl-retry", 0.25), + spend("final", "chatcmpl-final", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (Some(0.75), Some(SpendMatch::Matched)) + ); +} + +#[rstest] +fn call_id_prices_a_legacy_spend_log_without_litellm_call_id() { + let rows = [TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("legacy".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [spend("legacy", "chatcmpl-legacy", 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn spend_lookup_collects_only_assigned_ids() { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..row("span", "", "operation", "llm", "") + }; + assert_eq!( + litellm_traces::SpendLookup::new(&[recorded]), + litellm_traces::SpendLookup { + response_ids: vec!["response".into()], + provider_request_ids: Vec::new(), + request_ids: vec!["request".into()], + trace_ids: vec!["trace".into()], + } + ); +} + +#[rstest] +fn upstream_response_id_inside_a_managed_id_joins_spend() { + let rows = [llm("call", "", "agent", "chatcmpl-upstream")]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "chatcmpl-upstream".into(), + ..spend("request", "resp_managed", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn conflicting_complete_wrapper_cannot_price_an_unrelated_call() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "retry-response".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", "response"), + ]; + let logs = [ + spend("retry", "retry-response", 0.25), + spend("final", "response", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +#[rstest] +fn ambiguous_response_id_keeps_cost_unknown() { + let rows = [llm("llm-1", "", "agent", "response-1")]; + let spend = [ + spend("response-1", "response-1", 0.25), + spend("response-1_cache_hit123", "response-1", 0.0), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); } #[rstest] @@ -827,37 +804,6 @@ fn iso_time_matches_python_isoformat(#[case] ms: i64, #[case] expected: &str) { assert_eq!(iso_time(ms), expected); } -#[rstest] -#[case::narrows_ambiguity("request-a", Some(0.25))] -#[case::conflicting_exact_request("request-c", None)] -fn complete_wrapper_reconciles_ambiguous_response( - #[case] exact_id: &str, - #[case] expected: Option, -) { - let wrapper = TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") - }; - let rows = [ - wrapper, - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("request-a", "response", "team", "", "key", 0.25), - spend("request-b", "response", "team", "", "key", 0.5), - spend("request-c", "other-response", "team", "", "key", 0.75), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - #[rstest] #[case::missing(None, None)] #[case::free(Some(0.0), Some(0.0))] @@ -868,15 +814,10 @@ fn complete_correlation_requires_known_finite_cost( #[case] cost: Option, #[case] expected: Option, ) { - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let logged = SpendByResponseIdsRow { spend: cost, - ..spend("request", "response", "team", "", "key", 0.25) + ..spend("request", "response", 0.25) }; let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); assert_eq!(trace.summary.spend, expected); @@ -884,160 +825,39 @@ fn complete_correlation_requires_known_finite_cost( assert_eq!(trace.spans[0].spend, expected); } -#[rstest] -#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] -#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] -#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] -#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] -fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( - #[case] retry_logged: bool, - #[case] retry_evidence: litellm_traces::CallEvidenceKind, - #[case] expected: Option, -) { - let transport = |id: &str| { - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec!["transport:".parse().unwrap()], - call_evidence: Some(if id == "first" { - retry_evidence - } else { - litellm_traces::CallEvidenceKind::Complete - }), - ..row(id, "call", "POST", "framework", "") - }, - "team", - "", - "key", - ) - }; - let rows = [ - owned( - llm("call", "", "agent", "final-response"), - "team", - "", - "key", - ), - transport("first"), - transport("second"), - ]; - let logs = [ - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "first".into(), - ..spend("retry", "retry-response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "second".into(), - ..spend("final", "final-response", "team", "", "key", 0.5) - }, - spend("cached", "final-response", "team", "", "key", 0.0), - ]; - let available = if retry_logged { &logs[..] } else { &logs[1..] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - -#[rstest] -#[case::same_request(false)] -#[case::ambiguous_response(true)] -fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![ - "provider_response:response".parse().unwrap(), - "litellm_request:request".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - spend("request", "response", "team", "", "key", 0.25), - spend("cached", "response", "team", "", "key", 0.5), - ]; - let available = if cached_row { &logs[..] } else { &logs[..1] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.agents[0].spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::finite(0.25, Some(0.5))] #[case::overflow(f64::MAX, None)] fn trace_cost_requires_a_finite_total(#[case] cost: f64, #[case] expected: Option) { let rows = [ - owned(llm("first", "", "agent", "response-a"), "team", "", "key"), - owned(llm("second", "", "agent", "response-b"), "team", "", "key"), + llm("first", "", "agent", "response-a"), + llm("second", "", "agent", "response-b"), ]; let logs = [ - spend("request-a", "response-a", "team", "", "key", cost), - spend("request-b", "response-b", "team", "", "key", cost), + spend("request-a", "response-a", cost), + spend("request-b", "response-b", cost), ]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.agents[0].spend, expected); } -#[rstest] -fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![ - "litellm_request:retry".parse().unwrap(), - "litellm_request:final".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("retry", "retry-response", "team", "", "key", 0.25), - spend("final", "response", "team", "", "key", 0.5), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.75)); - assert_eq!(trace.agents[0].spend, Some(0.75)); -} - #[rstest] #[case::legacy(None, Some(0.25))] #[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] #[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] #[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] -fn legacy_request_id_fallback_respects_recorded_evidence( +fn stored_response_id_requires_complete_call_evidence( #[case] evidence: Option, #[case] expected: Option, ) { - let span = owned( - TraceSpansRow { - call_evidence: evidence, - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - ); + let span = TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }; let stored = serde_json::to_value(span).unwrap(); let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); - let logs = [spend("request", "response", "team", "", "key", 0.25)]; + let logs = [spend("request", "response", 0.25)]; let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.spans[0].spend, expected); @@ -1058,28 +878,6 @@ fn malformed_stored_span_fields_are_rejected( assert!(serde_json::from_value::(encoded).is_err()); } -#[rstest] -#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] -#[case::complete(litellm_traces::CallEvidenceKind::Complete)] -fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( - #[case] evidence: litellm_traces::CallEvidenceKind, -) { - let recorded = TraceSpansRow { - trace_id: "trace".to_owned(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".to_owned()), - litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), - litellm_traces::CallKey::Transport, - ], - call_evidence: Some(evidence), - ..row("span", "", "operation", "llm", "") - }; - let lookup = litellm_traces::SpendLookup::new(&[recorded]); - assert_eq!(lookup.response_ids, ["response"]); - assert_eq!(lookup.request_ids, ["request"]); - assert_eq!(lookup.trace_ids, ["trace"]); -} - #[rstest] #[case::parent_first(false)] #[case::child_first(true)] @@ -1150,68 +948,23 @@ fn empty_root_preview_uses_the_earliest_agent_or_model_input() { fn repeated_request_ids_preserve_storage_identity(#[case] reverse: bool) { let first = SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) + ..spend("same", "response", 0.25) }; let second = SpendByResponseIdsRow { start_ms: 200, - ..spend("same", "response", "team", "", "key", 0.5) + ..spend("same", "response", 0.5) }; let logs = if reverse { [second, first] } else { [first, second] }; - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, None); assert_eq!(trace.spans[0].spend, None); } -#[rstest] -#[case::gateway(false)] -#[case::transport(true)] -fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { - let rows = [owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".into()), - if transport { - litellm_traces::CallKey::Transport - } else { - litellm_traces::CallKey::LiteLlmRequest("gateway".into()) - }, - ], - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - litellm_call_id: "gateway".into(), - trace_id: "trace".into(), - span_id: "call".into(), - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "other".into(), - ..spend("same", "response", "team", "", "key", 0.5) - }, - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::same_row(true, Some(0.25))] #[case::distinct_rows(false, Some(0.75))] @@ -1220,27 +973,19 @@ fn totals_deduplicate_only_equal_storage_identities( #[case] expected: Option, ) { let rows = [ - owned(llm("first", "", "agent", "a"), "team", "", "key"), - owned( - llm("second", "", "agent", if duplicate { "a" } else { "b" }), - "team", - "", - "key", - ), + llm("first", "", "agent", "a"), + llm("second", "", "agent", if duplicate { "a" } else { "b" }), ]; let logs = [ SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "a", "team", "", "key", 0.25) + ..spend("same", "a", 0.25) }, SpendByResponseIdsRow { start_ms: if duplicate { 100 } else { 200 }, ..spend( "same", if duplicate { "a" } else { "b" }, - "team", - "", - "key", if duplicate { 0.25 } else { 0.5 }, ) }, @@ -1254,80 +999,6 @@ fn totals_deduplicate_only_equal_storage_identities( ); } -#[rstest] -fn conflicting_keys_cannot_agree_on_request_id_alone() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "gateway".into(), - ..spend("same", "other", "team", "", "key", 0.5) - }, - ]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - None - ); -} - -#[rstest] -#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] -#[case::legacy("", "gateway", "team", "key", Some(0.25))] -#[case::conflict("other", "gateway", "team", "key", None)] -#[case::other_team("gateway", "provider-id", "other-team", "key", None)] -#[case::other_key("gateway", "provider-id", "team", "other-key", None)] -fn gateway_lookup_respects_legacy_fallback_and_ownership( - #[case] call_id: &str, - #[case] request_id: &str, - #[case] team: &str, - #[case] key: &str, - #[case] expected: Option, -) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("call", "", "agent", "") - }, - "team", - "", - "key", - )]; - let logs = [SpendByResponseIdsRow { - litellm_call_id: call_id.into(), - ..spend(request_id, "provider", team, "", key, 0.25) - }]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - expected - ); -} - #[rstest] #[case::matching("call-one", "claude_code.tool.execution", SpanStatus::Error)] #[case::other_tool("other-call", "claude_code.tool.execution", SpanStatus::Ok)] @@ -1389,3 +1060,790 @@ fn native_tool_failure_log_matches_by_call_id_without_double_counting( assert_eq!(trace.spans[0].status, expected); assert_eq!(trace.summary.error_count, 1); } + +fn owned_spend( + request_id: &str, + response_id: &str, + team: &str, + user: &str, + key: &str, + cost: f64, +) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + request_id: request_id.into(), + response_id: response_id.into(), + litellm_call_id: String::new(), + upstream_response_id: String::new(), + provider_request_id: String::new(), + trace_id: String::new(), + span_id: String::new(), + team_id: team.into(), + api_key: key.into(), + user: user.into(), + spend: Some(cost), + start_ms: T0 / MS, + } +} + +fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { + span.team_id = team.into(); + span.user_id = user.into(); + span.api_key_hash = key.into(); + span +} + +#[rstest] +fn repeated_response_counts_once_and_other_owners_are_ignored() { + let rows = [ + owned( + row("root", "", "agent", "agent", "agent"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-1", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-2", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + ]; + let spend = [ + owned_spend("request-other", "response-1", "team-b", "", "key-b", 99.0), + owned_spend("request-1", "response-1", "team-a", "", "key-a", 0.25), + owned_spend( + "request-other-key", + "unrelated-response", + "team-a", + "", + "key-c", + 50.0, + ), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), Some(0.25)] + ); +} + +#[rstest] +#[case::key_differs("team", "", "export", "team", "", "request", false)] +#[case::shared_key("team", "", "export", "team", "", "export", true)] +#[case::shared_user("", "user", "export", "", "user", "request", true)] +#[case::teamless_key("", "", "key", "", "", "key", true)] +#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] +#[case::other_user("", "user", "export", "", "other-user", "request", false)] +#[case::no_shared_identity("", "", "export", "", "", "request", false)] +#[case::no_identity("", "", "", "", "", "", false)] +#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] +fn cost_requires_shared_ownership( + #[case] trace_team: &str, + #[case] trace_user: &str, + #[case] trace_key: &str, + #[case] spend_team: &str, + #[case] spend_user: &str, + #[case] spend_key: &str, + #[case] known: bool, +) { + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + trace_team, + trace_user, + trace_key, + ), + owned( + llm("llm", "agent", "agent", "response"), + trace_team, + trace_user, + trace_key, + ), + ]; + let spend = [owned_spend( + "request", "response", spend_team, spend_user, spend_key, 0.25, + )]; + let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); + let expected = known.then_some(0.25); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[1].spend, expected); +} + +#[rstest] +#[case::missing_id("missing_id")] +#[case::missing_spend("missing_spend")] +#[case::duplicate_spend("duplicate_spend")] +fn partial_trace_total_counts_only_complete_calls(#[case] failure: &str) { + let second_id = if failure == "missing_id" { + "" + } else { + "second" + }; + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "export", + ), + owned( + llm("first", "agent", "agent", "first"), + "team", + "", + "export", + ), + owned( + llm("second", "agent", "agent", second_id), + "team", + "", + "export", + ), + ]; + let first = owned_spend("first", "first", "team", "", "export", 0.25); + let second = owned_spend("second", "second", "team", "", "export", 0.25); + let duplicate = owned_spend("duplicate", "second", "team", "", "export", 0.25); + let spend = if failure == "duplicate_spend" { + vec![first, second, duplicate] + } else { + vec![first] + }; + let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.spans[1].spend, Some(0.25)); + assert_eq!(trace.spans[2].spend, None); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); +} + +#[rstest] +fn transport_spans_complete_a_call_without_its_own_id() { + let mut transport = row("http", "llm", "POST", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec!["transport:".parse().unwrap()]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, Some(0.5)); +} + +#[rstest] +#[case::lone_call(1, Some(0.5))] +#[case::two_calls(2, None)] +fn sibling_transports_belong_to_the_only_model_call_under_their_parent( + #[case] calls: usize, + #[case] expected: Option, +) { + let mut transport = at( + row("http", "step", "gateway.request", "framework", ""), + 2, + 10, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut rows = vec![ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + for index in 0..calls { + let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); + call.call_evidence = None; + rows.push(owned(call, "team", "", "key")); + } + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] +fn sibling_transport_does_not_lose_model_call_spend( + #[case] transport_timing: Option<(i64, u64)>, + #[case] unrelated_spend: bool, + #[case] key: litellm_traces::CallKey, + #[case] expected: Option, +) { + let call = owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "chatcmpl-1".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("chat", "step", "agent", "chatcmpl-1") + }, + "team", + "", + "key", + ); + let base_rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + call, + ]; + let rows: Vec<_> = base_rows + .into_iter() + .chain(transport_timing.map(|(start, duration)| { + let mut transport = at( + row("tool-http", "step", "GET", "framework", ""), + start, + duration, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![key]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + owned(transport, "team", "", "key") + })) + .collect(); + let logs: Vec<_> = std::iter::once(owned_spend( + "chatcmpl-1", + "chatcmpl-1", + "team", + "", + "key", + 0.5, + )) + .chain(unrelated_spend.then(|| SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "tool-http".into(), + ..owned_spend("unrelated", "unrelated", "team", "", "key", 0.75) + })) + .collect(); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::agreeing_ids( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] +#[case::conflicting_response_id( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-b"), + None +)] +#[case::conflicting_gateway_and_response( + litellm_traces::CallKey::Transport, + "call-b", + Some("response-b"), + None +)] +#[case::agreeing_gateway_attempt( + litellm_traces::CallKey::GatewayAttempt, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] +fn gateway_attempt_identifiers_must_match_one_spend_row( + #[case] transport: litellm_traces::CallKey, + #[case] call_id: &str, + #[case] response_id: Option<&str>, + #[case] expected: Option, +) { + let keys = [ + transport, + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ] + .into_iter() + .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) + .collect(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(llm("call", "agent", "agent", ""), "team", "", "key"), + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: keys, + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..row("attempt", "call", "gateway.request", "framework", "") + }, + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "call-a".into(), + trace_id: "trace".into(), + span_id: "attempt".into(), + ..owned_spend("request-a", "response-a", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "call-b".into(), + trace_id: "trace".into(), + span_id: "other-attempt".into(), + ..owned_spend("request-b", "response-b", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[2].spend, expected); +} + +#[rstest] +#[case::legacy_row("", Some(0.5))] +#[case::other_call("other-call", None)] +fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( + #[case] logged_call_id: &str, + #[case] expected: Option, +) { + let mut transport = row("http", "llm", "gateway.request", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec![ + "transport:".parse().unwrap(), + "litellm_request:gateway-call".parse().unwrap(), + ]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + logged.litellm_call_id = logged_call_id.into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); +} + +#[rstest] +#[case::narrows_ambiguity("request-a", Some(0.25))] +#[case::conflicting_exact_request("request-c", None)] +fn complete_wrapper_reconciles_ambiguous_response( + #[case] exact_id: &str, + #[case] expected: Option, +) { + let wrapper = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") + }; + let rows = [ + wrapper, + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("request-a", "response", "team", "", "key", 0.25), + owned_spend("request-b", "response", "team", "", "key", 0.5), + owned_spend("request-c", "other-response", "team", "", "key", 0.75), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] +#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] +#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] +#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] +fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( + #[case] retry_logged: bool, + #[case] retry_evidence: litellm_traces::CallEvidenceKind, + #[case] expected: Option, +) { + let transport = |id: &str| { + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec!["transport:".parse().unwrap()], + call_evidence: Some(if id == "first" { + retry_evidence + } else { + litellm_traces::CallEvidenceKind::Complete + }), + ..row(id, "call", "POST", "framework", "") + }, + "team", + "", + "key", + ) + }; + let rows = [ + owned( + llm("call", "", "agent", "final-response"), + "team", + "", + "key", + ), + transport("first"), + transport("second"), + ]; + let logs = [ + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "first".into(), + ..owned_spend("retry", "retry-response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "second".into(), + ..owned_spend("final", "final-response", "team", "", "key", 0.5) + }, + owned_spend("cached", "final-response", "team", "", "key", 0.0), + ]; + let available = if retry_logged { &logs[..] } else { &logs[1..] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::same_request(false)] +#[case::ambiguous_response(true)] +fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![ + "provider_response:response".parse().unwrap(), + "litellm_request:request".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + owned_spend("request", "response", "team", "", "key", 0.25), + owned_spend("cached", "response", "team", "", "key", 0.5), + ]; + let available = if cached_row { &logs[..] } else { &logs[..1] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![ + "litellm_request:retry".parse().unwrap(), + "litellm_request:final".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("retry", "retry-response", "team", "", "key", 0.25), + owned_spend("final", "response", "team", "", "key", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.75)); + assert_eq!(trace.agents[0].spend, Some(0.75)); +} + +#[rstest] +#[case::legacy(None, Some(0.25))] +#[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] +#[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] +#[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] +fn legacy_request_id_fallback_respects_recorded_evidence( + #[case] evidence: Option, + #[case] expected: Option, +) { + let span = owned( + TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + ); + let stored = serde_json::to_value(span).unwrap(); + let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); + let logs = [owned_spend("request", "response", "team", "", "key", 0.25)]; + let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.spans[0].spend, expected); +} + +#[rstest] +#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] +#[case::complete(litellm_traces::CallEvidenceKind::Complete)] +fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( + #[case] evidence: litellm_traces::CallEvidenceKind, +) { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(evidence), + ..row("span", "", "operation", "llm", "") + }; + let lookup = litellm_traces::SpendLookup::new(&[recorded]); + assert_eq!(lookup.response_ids, ["response"]); + assert_eq!(lookup.request_ids, ["request"]); + assert_eq!(lookup.trace_ids, ["trace"]); +} + +#[rstest] +#[case::gateway(false)] +#[case::transport(true)] +fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { + let rows = [owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".into()), + if transport { + litellm_traces::CallKey::Transport + } else { + litellm_traces::CallKey::LiteLlmRequest("gateway".into()) + }, + ], + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + litellm_call_id: "gateway".into(), + trace_id: "trace".into(), + span_id: "call".into(), + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "other".into(), + ..owned_spend("same", "response", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn conflicting_keys_cannot_agree_on_request_id_alone() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "gateway".into(), + ..owned_spend("same", "other", "team", "", "key", 0.5) + }, + ]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + None + ); +} + +#[rstest] +#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] +#[case::legacy("", "gateway", "team", "key", Some(0.25))] +#[case::conflict("other", "gateway", "team", "key", None)] +#[case::other_team("gateway", "provider-id", "other-team", "key", None)] +#[case::other_key("gateway", "provider-id", "team", "other-key", None)] +fn gateway_lookup_respects_legacy_fallback_and_ownership( + #[case] call_id: &str, + #[case] request_id: &str, + #[case] team: &str, + #[case] key: &str, + #[case] expected: Option, +) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("call", "", "agent", "") + }, + "team", + "", + "key", + )]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..owned_spend(request_id, "provider", team, "", key, 0.25) + }]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + expected + ); +} + +#[rstest] +#[case::native_id("provider_request:req_native", false, true)] +#[case::historical_id("provider_response:req_native", false, true)] +#[case::legacy_message_id("provider_response:msg_native", false, true)] +#[case::transport("", true, true)] +#[case::no_correlation("", false, false)] +fn native_cost_uses_typed_ids_or_the_original_transport( + #[case] encoded: &str, + #[case] traceparent: bool, + #[case] priced: bool, + #[values(false, true)] grouped: bool, +) { + let call = TraceSpansRow { + trace_id: if grouped { + "session-trace" + } else { + "original-trace" + } + .into(), + original_trace_id: if grouped { "original-trace" } else { "" }.into(), + name: "claude_code.llm_request".into(), + framework: "claude-code".into(), + call_keys: encoded.parse().into_iter().collect(), + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..llm("native-call", "", "agent", "") + }; + let log = SpendByResponseIdsRow { + provider_request_id: "req_native".into(), + trace_id: if traceparent { "original-trace" } else { "" }.into(), + span_id: if traceparent { "native-call" } else { "" }.into(), + ..spend("gateway-log", "msg_native", 0.25) + }; + let lookup = litellm_traces::SpendLookup::new(std::slice::from_ref(&call)); + let trace = resolve_trace("trace", "ref", &[call], &[log]).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (priced.then_some(0.25), u64::from(priced)) + ); + assert_eq!(trace.spans[0].spend, priced.then_some(0.25)); + if encoded.is_empty() { + assert_eq!(lookup.trace_ids, ["original-trace"]); + } else if !encoded.ends_with("msg_native") { + assert_eq!(lookup.provider_request_ids, ["req_native"]); + assert!(lookup.response_ids.is_empty()); + } +} + +#[rstest] +fn provider_request_id_cannot_match_a_message_id_of_the_same_value() { + let call = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderRequest( + "req_native".into(), + )], + ..llm("call", "", "agent", "") + }; + let rows = [spend("unrelated", "req_native", 0.25)]; + let trace = resolve_trace("trace", "ref", &[call], &rows).unwrap(); + assert_eq!(trace.summary.spend, None); + assert_eq!(trace.spans[0].spend_match, Some(SpendMatch::NoSpendLog)); +} diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 588f81ec50a..6c9f726056b 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger from litellm.integrations.clickhouse.context import is_lens_analysis from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.tracing.types import SpendLogRecord from litellm.types.utils import StandardLoggingPayload @@ -155,6 +156,11 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[ return SpendLogRecord( request_id=request_id, response_id=strip_cache_hit_suffix(request_id), + provider_request_id=( + get_provider_request_id(kwargs.get("response_headers")) + or get_provider_request_id(hidden_params.get("additional_headers")) + or "" + ), litellm_call_id=payload.get("litellm_call_id") or "", call_type=payload.get("call_type") or "", api_key=metadata.get("user_api_key_hash") or "", diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4fa78640e24..76ea2c25159 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,6 +11,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -495,7 +496,8 @@ def process_response_headers( **processed_headers, **additional_headers, } - return additional_headers + request_id: Final = get_provider_request_id(response_headers) + return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} def preserve_upstream_non_openai_attributes( diff --git a/litellm/litellm_core_utils/llm_response_utils/get_headers.py b/litellm/litellm_core_utils/llm_response_utils/get_headers.py index d04abcb6e7b..b86cc954b0e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_headers.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_headers.py @@ -1,6 +1,20 @@ from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter, ValidationError + +_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id") + + +def get_provider_request_id(headers: object) -> str | None: + try: + values: Final = _HEADERS.validate_python(headers) + except ValidationError: + return None + normalized: Final = {name.lower(): value for name, value in values.items()} + return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None) + def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict: """ @@ -29,7 +43,9 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> if "x-ratelimit-remaining-tokens" in _response_headers: openai_headers["x-ratelimit-remaining-tokens"] = _response_headers["x-ratelimit-remaining-tokens"] llm_provider_headers: Final = _get_llm_provider_headers(_response_headers) - return {**llm_provider_headers, **openai_headers} + request_id: Final = get_provider_request_id(_response_headers) + native_headers: Final = {"request-id": request_id} if request_id is not None else {} + return {**llm_provider_headers, **openai_headers, **native_headers} def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 86c36a7b413..53ea85a2740 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2214,6 +2214,7 @@ class BaseLLMHTTPHandler: # used for logging + cost tracking logging_obj.model_call_details["httpx_response"] = response + logging_obj.model_call_details["response_headers"] = dict(response.headers) initial_response: AsyncIterator | AnthropicMessagesResponse if stream: diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index e011b795000..46af4e236eb 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -21,6 +21,8 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +from pydantic import TypeAdapter + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -337,9 +339,15 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t return hook(kwargs, error, call_type) -def stream_opened(logger: Logging) -> None: +_STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True + logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python( + head.get("additional_headers") or {} + ) def stream_success( diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 127e86e9160..4a09e731228 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -59,6 +59,7 @@ class AgentNode(typing_extensions.TypedDict): tool_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] duration_ms: ReadOnly[float] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] SpanType: TypeAlias = Literal[ @@ -77,6 +78,9 @@ SpanType: TypeAlias = Literal[ ] +SpendMatch: TypeAlias = Literal["matched", "no_call_id", "no_spend_log", "ambiguous", "incomplete_evidence"] + + class TraceScope(typing_extensions.TypedDict): all_teams: ReadOnly[Literal[0, 1]] user_id: ReadOnly[str] @@ -120,6 +124,7 @@ class TraceSummary(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] models: ReadOnly[tuple[str, ...]] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] class Span(typing_extensions.TypedDict): @@ -140,6 +145,8 @@ class Span(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=4294967295)]] litellm_request_id: ReadOnly[str | None] spend: ReadOnly[float | None] + spend_log_request_id: ReadOnly[str | None] + spend_match: ReadOnly[SpendMatch | None | None] class Trace(typing_extensions.TypedDict): diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index e9930dc7bf7..8bd119940c2 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -8,6 +8,7 @@ class SpendLogRecord(TypedDict): request_id: ReadOnly[str] response_id: ReadOnly[str] + provider_request_id: ReadOnly[str] litellm_call_id: ReadOnly[str] call_type: ReadOnly[str] api_key: ReadOnly[str] diff --git a/scripts/trace_codegen/schemas/traces/Trace.json b/scripts/trace_codegen/schemas/traces/Trace.json index dc66f77412c..ebd99880956 100644 --- a/scripts/trace_codegen/schemas/traces/Trace.json +++ b/scripts/trace_codegen/schemas/traces/Trace.json @@ -28,6 +28,12 @@ "null" ] }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "spend": { "format": "double", "type": [ @@ -49,7 +55,8 @@ "llm_calls", "tool_calls", "duration_ms", - "spend" + "spend", + "priced_calls" ], "type": "object" }, @@ -120,6 +127,22 @@ "null" ] }, + "spend_log_request_id": { + "type": [ + "string", + "null" + ] + }, + "spend_match": { + "anyOf": [ + { + "$ref": "#/$defs/SpendMatch" + }, + { + "type": "null" + } + ] + }, "start_offset_ms": { "format": "double", "type": "number" @@ -148,7 +171,9 @@ "input_tokens", "output_tokens", "litellm_request_id", - "spend" + "spend", + "spend_log_request_id", + "spend_match" ], "type": "object" }, @@ -177,6 +202,16 @@ ], "type": "string" }, + "SpendMatch": { + "enum": [ + "matched", + "no_call_id", + "no_spend_log", + "ambiguous", + "incomplete_evidence" + ], + "type": "string" + }, "TraceSummary": { "properties": { "agent_count": { @@ -245,6 +280,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -306,7 +347,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/scripts/trace_codegen/schemas/traces/TracePage.json b/scripts/trace_codegen/schemas/traces/TracePage.json index dfa02ba0d39..429635ec3b1 100644 --- a/scripts/trace_codegen/schemas/traces/TracePage.json +++ b/scripts/trace_codegen/schemas/traces/TracePage.json @@ -76,6 +76,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -137,7 +143,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/tests/llm_translation/test_llm_response_utils/test_get_headers.py b/tests/llm_translation/test_llm_response_utils/test_get_headers.py index 380f89bbdd4..60a6a4bebcd 100644 --- a/tests/llm_translation/test_llm_response_utils/test_get_headers.py +++ b/tests/llm_translation/test_llm_response_utils/test_get_headers.py @@ -8,6 +8,7 @@ import pytest from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, _get_llm_provider_headers, + get_provider_request_id, ) @@ -72,3 +73,17 @@ def test_get_llm_provider_headers(): } result = _get_llm_provider_headers(input_headers) assert result == expected_output, "Unexpected output for _get_llm_provider_headers" + + +@pytest.mark.parametrize("header", ("request-id", "Request-Id", "x-request-id", "llm_provider-request-id")) +def test_native_clients_receive_the_provider_request_id(header: str) -> None: + result = get_response_headers({header: "req_test", "unrelated": "value"}) + assert result["request-id"] == "req_test" + assert get_provider_request_id(result) == "req_test" + assert result["llm_provider-unrelated"] == "value" + assert "unrelated" not in result + + +@pytest.mark.parametrize("headers", (None, {}, {"request-id": ""}, {"request-id": 42})) +def test_invalid_provider_request_ids_remain_absent(headers: object) -> None: + assert get_provider_request_id(headers) is None diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py index 491f8651b52..fe47c1a59f3 100644 --- a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -163,6 +163,7 @@ def test_success_row_mapping(): assert row["request_id"] == "chatcmpl-abc123" assert row["response_id"] == "chatcmpl-abc123" assert row["litellm_call_id"] == "gateway-call" + assert row["provider_request_id"] == "" assert row["spend"] == 0.00042 assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30) assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7) @@ -186,6 +187,18 @@ def test_success_row_mapping(): assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key" +@pytest.mark.parametrize("header_source", ("response_headers", "additional_headers")) +def test_provider_request_id_stays_separate_from_message_and_gateway_ids(header_source: str) -> None: + headers: Final = {"request-id": "req_native"} + hidden: Final = {"additional_headers": headers} if header_source == "additional_headers" else {} + kwargs: Final = {"response_cost": 0.00042, "response_headers": headers if header_source == "response_headers" else None} + payload: Final = cast(StandardLoggingPayload, _payload(id="msg_native", hidden_params=hidden)) + row: Final = spend_log_row_from_payload(payload, kwargs) + assert (row["provider_request_id"], row["response_id"], row["litellm_call_id"]) == ( + "req_native", "msg_native", "gateway-call" + ) + + @pytest.mark.parametrize("status", ("success", "failure")) @pytest.mark.asyncio async def test_custom_request_metadata_is_redacted_before_clickhouse_logging( diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 2a6dd347d5f..9f48ea62a72 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,6 +1,7 @@ """Tests for litellm_core_utils.core_helpers module.""" import logging +from typing import Final import httpx import pytest @@ -16,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, map_finish_reason, normalize_drop_params, + process_response_headers, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -25,6 +27,14 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ImageResponse, TranscriptionResponse +@pytest.mark.parametrize("header", ("request-id", "x-request-id", "llm_provider-request-id")) +def test_native_request_id_survives_stream_header_processing(header: str) -> None: + processed: Final = process_response_headers(httpx.Headers({header: "req_native"})) + + assert processed["request-id"] == "req_native" + assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native" + + class TestBudgetReservationBinding: """The request-end release skips a reservation a cost callback has claimed, so the claim must land on the one dict auth stamped, through whichever metadata field or auth object diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 93d18727f22..2d59ab337d4 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -895,6 +895,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp additional_headers = result._hidden_params["additional_headers"] assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123" assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def" + assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers) collected = b"".join([chunk async for chunk in result]) assert b"message_start" in collected @@ -1601,8 +1602,7 @@ async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeyp expected_response = {"id": "msg_1", "content": []} mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response) - ok_response = Mock() - ok_response.raise_for_status = Mock(return_value=None) + ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) mock_client = AsyncMock(spec=AsyncHTTPHandler) mock_client.post = AsyncMock(return_value=ok_response) @@ -2072,8 +2072,7 @@ async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks( captured_kwargs.update(call_kwargs) return sentinel_response - mock_httpx_response = Mock() - mock_httpx_response.status_code = 200 + mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) with ( patch.object( diff --git a/tests/unit/rust_bridge/messages/test_route_host.py b/tests/unit/rust_bridge/messages/test_route_host.py index 1be42e2249d..7b15553a055 100644 --- a/tests/unit/rust_bridge/messages/test_route_host.py +++ b/tests/unit/rust_bridge/messages/test_route_host.py @@ -161,4 +161,4 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert isinstance(additional, dict) assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" - assert "request-id" not in additional + assert additional["request-id"] == additional["llm_provider-request-id"] diff --git a/tests/unit/rust_bridge/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/test_callbacks_legacy_python.py index 045dcb52ce9..c78fbb3f246 100644 --- a/tests/unit/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/test_callbacks_legacy_python.py @@ -34,6 +34,14 @@ def _supplied_logger() -> Logging: ) +def test_native_stream_headers_reach_spend_callbacks() -> None: + logger: Final = _supplied_logger() + legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}}) + + assert logger.stream is True + assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"} + + def test_setup_reuses_a_supplied_logger() -> None: supplied: Final = _supplied_logger() result: Final = setup( @@ -159,9 +167,7 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation pending.close() -CONTRACT_PATH: Final = ( - Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" -) +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" def test_the_rust_contract_matches_the_shim_signatures() -> None: diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts index 27c46a528e7..f33f86cd280 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts @@ -26,6 +26,8 @@ function makeTrace(scene: Scenario, index: number, now: number) { parent_span_id: null, span_id: spanId(0), spend: null, + spend_log_request_id: null, + spend_match: null, start_offset_ms: 0, status: "ok", type: "agent", @@ -53,6 +55,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { input_tokens: 520 + index * 41, output_tokens: 48 + index * 7, spend: 0.003 + index * 0.0002, + spend_match: "matched", }; const trace: Trace = { summary: { @@ -71,6 +74,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { service: "demo-agents", span_count: toolCount + 2, spend: model.spend, + priced_calls: 1, start_time: iso(now - (index + 1) * 35 * 60_000), status: scene.failed ? "error" : "ok", tool_calls: toolCount, @@ -85,6 +89,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { llm_calls: 1, tool_calls: toolCount, spend: model.spend, + priced_calls: 1, }, ], spans: [base, ...tools, model], diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts index 866f1042b73..44eac42755c 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts @@ -85,6 +85,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0), output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0), spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0), + priced_calls: models.length, }; const finalDetail = run.details.at(-1)!; const history = caseDetails.filter((_, index) => index % 3 === 2); @@ -102,6 +103,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { llm_calls: caseCount, tool_calls: caseCount, spend: summary.spend - (final.spend ?? 0), + priced_calls: caseCount, }, ], }, diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx index 16afa455be6..33fb073d0a9 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx @@ -10,8 +10,8 @@ import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; import type { TraceHandoff } from "../../api"; +import { runCost } from "../../list/AgentTracesTable"; import { traceRefOf, traceShareUrl } from "../../routing"; -import { formatCost } from "../../list/AgentTracesTable"; import { IdChip } from "../../ui/IdChip"; import { SpanIcon } from "../../ui/SpanIcon"; import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework"; @@ -89,6 +89,7 @@ interface RunHeaderProps { export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) { const { summary } = trace; const failed = summary.status === "error"; + const cost = runCost(summary); return (
@@ -127,7 +128,10 @@ export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) - + {summary.error_count > 0 && }
diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx index fb9285d0777..cf2c32ff5c7 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx @@ -13,10 +13,31 @@ import type { Span, SpanDetail, SpanErrorPage, Trace } from "../../types"; vi.mock("../../../../networking", () => ({ agentTraceSpanCall: vi.fn(), agentTraceSpanErrorCall: vi.fn(), + uiSpendLogsCall: vi.fn(), getProxyBaseUrl: () => "http://proxy.test/", })); -import { agentTraceSpanCall, agentTraceSpanErrorCall } from "../../../../networking"; +vi.mock("../../../../logs/detail", () => ({ + LogDetailsDrawer: ({ + open, + logEntry, + onClose, + backTo, + }: { + open: boolean; + logEntry: { request_id: string } | null; + onClose: () => void; + backTo?: { label: string }; + }) => + open && logEntry ? ( +
+ {logEntry.request_id} + {backTo && } +
+ ) : null, +})); + +import { agentTraceSpanCall, agentTraceSpanErrorCall, uiSpendLogsCall } from "../../../../networking"; type SpanFields = Partial & Pick; @@ -37,6 +58,8 @@ const span = (overrides: SpanFields): Span => ({ output_tokens: 0, litellm_request_id: null, spend: null, + spend_log_request_id: null, + spend_match: null, ...overrides, }); @@ -49,6 +72,9 @@ const llmFields: SpanFields = { input_tokens: 659, output_tokens: 60, litellm_request_id: "chatcmpl-abc", + spend: 0.0002, + spend_log_request_id: "req_7f3a9c2e1b44", + spend_match: "matched", }; const failedToolFields: SpanFields = { span_id: "tool1", @@ -244,13 +270,47 @@ describe("DetailPane", () => { expect(input).toHaveTextContent("acme-404"); }); - it("shows the LiteLLM request facts on the Request tab", async () => { + it("links a priced LLM step to its spend log in the header and opens it by that request id", async () => { const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); renderPane(spanRow(llm)); + const link = screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" }); + expect(link).toHaveTextContent("LiteLLM Spend Log$0.0002"); + await user.click(link); + expect(await screen.findByRole("dialog", { name: "Request log" })).toHaveTextContent("req_7f3a9c2e1b44"); + expect(vi.mocked(uiSpendLogsCall).mock.calls[0][0].params).toEqual({ request_id: "req_7f3a9c2e1b44" }); + }); + + it("returns from the spend log to the same agent trace step", async () => { + const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); + renderPane(spanRow(llm)); + await user.click(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })); + const drawer = await screen.findByRole("dialog", { name: "Request log" }); + await user.click(within(drawer).getByRole("button", { name: "Back to Lens trace" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Request log" })).not.toBeInTheDocument()); + expect(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })).toBeVisible(); + }); + + it.each([ + { spend_match: "no_call_id" as const, reason: "This step records no usable call identifier or transport context" }, + { spend_match: "no_spend_log" as const, reason: "No matching spend log belongs to this run" }, + { spend_match: "incomplete_evidence" as const, reason: "This step does not account for every request in the call" }, + ])("says why an unpriced LLM step has no cost ($spend_match)", async ({ spend_match, reason }) => { + const user = userEvent.setup(); + const unpriced: SpanFields = { ...llmFields, spend: null, spend_log_request_id: null, spend_match }; + renderPane(spanRow(span(unpriced))); + expect(screen.getByTitle(reason)).toHaveTextContent("Cost not matched"); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("tab", { name: "Request" })); - expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument(); - expect(screen.getByText("659")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument(); + expect(await screen.findByText(reason, { selector: "p" })).toBeInTheDocument(); + }); + + it("keeps the Cost fact for a priced step that has no single spend log to link", () => { + const pricedFromRetries: SpanFields = { ...llmFields, spend: 0.0005, spend_log_request_id: null }; + renderPane(spanRow(span(pricedFromRetries))); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); + expect(screen.getByText("Cost").parentElement).toHaveTextContent("Cost$0.0005"); }); it("summarizes a ×N group with its failure pattern", () => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx index 4757c55ab80..240ea149199 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx @@ -16,11 +16,12 @@ interface PaneHeaderProps { idValue?: string; facts: readonly (readonly [label: string, value: string])[]; actions?: React.ReactNode; + links?: React.ReactNode; onClose: () => void; } /** Selected step identity: tile, name, id, then its time and usage on one quiet line. */ -export function PaneHeader({ type, model, failed, title, idValue, facts, actions, onClose }: PaneHeaderProps) { +export function PaneHeader({ type, model, failed, title, idValue, facts, actions, links, onClose }: PaneHeaderProps) { return (
@@ -40,15 +41,18 @@ export function PaneHeader({ type, model, failed, title, idValue, facts, actions
- {facts.length > 0 && ( -
- {facts.map(([label, value]) => ( -
-
{label}
-
{value}
-
- ))} -
+ {(facts.length > 0 || links) && ( +
+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ {links} +
)} ); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx index 855a7060336..cd36d2686ef 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx @@ -1,19 +1,14 @@ "use client"; -import { ArrowUpRight } from "lucide-react"; -import { useState } from "react"; - import CopyButton from "@/components/shared/CopyButton"; -import { Button } from "@/components/ui/button"; -import { LogDetailsDrawer } from "../../../../logs/detail"; import { fieldEntries } from "../content/payload"; import { formatCost } from "../../list/AgentTracesTable"; import type { Span } from "../../types"; import { fmtMs, fmtTok } from "../../utils"; -import { useSpanRequestLog } from "../useSpanRequestLog"; import { FieldTree } from "../content/FieldTree"; import { DetailGroup } from "./DetailGroup"; +import { SpendLogLink, unmatchedReason } from "./SpendLogLink"; interface RequestTabProps { span: Span; @@ -22,17 +17,7 @@ interface RequestTabProps { traceStartMs: number; } -/** Usage and the linked LiteLLM request for an LLM span; "Open request log" opens the request drawer over the run. */ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) { - const [drawerOpen, setDrawerOpen] = useState(false); - const logQuery = useSpanRequestLog( - accessToken, - span.litellm_request_id, - traceStartMs + span.start_offset_ms, - drawerOpen, - ); - const logNotFound = drawerOpen && logQuery.isSuccess && logQuery.data === null; - if (span.type !== "llm") { return
This span is not a model request.
; } @@ -51,30 +36,22 @@ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) - - {span.litellm_request_id ? ( -
+ +
+ {span.spend_log_request_id && (
- +
- +
- - {logNotFound &&

No request log found for this call.

} + )} +
+
- ) : ( -

Not linked to a LiteLLM request

- )} + {unmatchedReason(span) &&

{unmatchedReason(span)}

} +
- setDrawerOpen(false)} - logEntry={logQuery.data ?? null} - accessToken={accessToken} - />
); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx index cfa92f67577..d74d7ad8ccd 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx @@ -12,6 +12,7 @@ import { ContentTab, useSpanDetail } from "../content/ContentTab"; import { AttributesTab } from "./AttributesTab"; import { PaneHeader } from "./PaneHeader"; import { RequestTab } from "./RequestTab"; +import { SpendLogLink } from "./SpendLogLink"; const TAB_LABELS: Record = { content: "Content", request: "Request", attributes: "Attributes" }; @@ -26,7 +27,7 @@ const spanFacts = (span: Span): readonly Fact[] => { const tokens = span.input_tokens + span.output_tokens; const optional: readonly (Fact | null)[] = [ tokens > 0 ? ["Tokens", fmtTok(tokens)] : null, - span.spend != null ? ["Cost", formatCost(span.spend)] : null, + span.spend != null && span.spend_log_request_id == null ? ["Cost", formatCost(span.spend)] : null, span.type === "llm" && span.model ? ["Step", span.name] : null, ]; return [["Duration", fmtMs(span.duration_ms)], ...optional.filter((fact): fact is Fact => fact !== null)]; @@ -42,6 +43,7 @@ export function SpanPane({ }: SpanTabProps & { trace: Trace; span: Span; accessToken: string; onClose: () => void }) { const { trace_id: traceId, trace_ref: traceRef, start_time: startTime } = trace.summary; const handoff = useTracesApi(accessToken).handoff(traceId, span.span_id, traceRef); + const traceStartMs = Date.parse(startTime); const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null, traceRef); return (