From 5ed7ec851148cbb47d33c95e9264f19a0bf4d8e6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:55:26 -0700 Subject: [PATCH] fix(lens): price agent traces by joining gen_ai.response.id to spend logs (#44738) * fix(lens): price agent traces by joining gen_ai.response.id to spend logs Trace spend now joins each model call to spend_logs on one key: the span's response id (gen_ai.response.id or the id the normalizers read from OpenInference/LangChain output) against spend_logs.response_id or the upstream id embedded in a managed resp_ id. The litellm.call_id and traceparent transport join paths and the per-row ownership gate are removed; the spend SQL still restricts rows to what the reader can see. A run with some unpriced calls now reports the sum of its priced calls plus priced_calls, instead of an unknown total. * test(lens): cover response id spend join and partial trace totals * chore(lens): regenerate trace types for priced_calls * feat(lens): show partial run cost as a lower bound with priced call count * fix(lens): treat litellm.call_id as the same assigned call id for spend joins The id LiteLLM assigned to a call is either the response id it returned (gen_ai.response.id -> spend_logs.response_id) or its gateway call id (litellm.call_id -> spend_logs.litellm_call_id). Both are exact ids the gateway mints and logs, so the join stays one rule. Transport span matching and the ownership gate stay removed. * test(lens): cover litellm.call_id spend joins and restore captured totals * feat(lens): link each priced model call to its spend log Spans gain spend_log_request_id, the spend_logs.request_id the call was priced from, and spend_match, which says whether a model call matched or why not (no assigned id on the span, no spend log with that id, or an ambiguous match). A model call span is priced from the same ids as the run total, so its cost and the total agree. * test(lens): cover spend log links on model call spans * chore(lens): regenerate trace types for spend log links * feat(lens): open the matched spend log from an LLM step An LLM step's header now shows a Spend log chip with the matched request id and cost; clicking it opens the request log drawer over the run, fetched by the exact spend_logs.request_id instead of the span's own response id. Unpriced steps say why (no assigned id on the span, or no spend log with it). Tree rows show each model call's cost, and a partial run cost shows its priced call count inline. * test(lens): cover the spend log link and unmatched cost reasons * feat(lens): show the spend log link as a bordered LiteLLM Spend Log button * feat(lens): add a back link from the spend log drawer to the agent trace * feat(lens): label the spend log back link Back to Lens trace with the Lens icon * fix(lens): ignore assigned ids that name no spend log when pricing a call An id that names no row no longer vetoes the call, so a span carrying both a response id and a call id still prices from a spend row logged before litellm_call_id existed. An id naming two or more rows makes the call ambiguous, and the match reason comes from the same per-id result, so a single matched row with no cost is reported as matched. * fix(lens): price a trace only from spend logs in its own team A reader with several teams could see the same assigned id in another team's spend log; only rows from the trace's team now price it. The user and key ownership gate stays removed. * perf(lens): resolve each model call's spend once per trace Model call matches are computed once when the trace is resolved and looked up by span index, instead of scanning the model call list for every span and walking the graph again for spans, agents and the run total. * fix(lens): hide a step's Cost fact only when its spend log link shows the cost * chore(lens): drop narrative doc comments from the spend join * fix(lens): price a model call only when its ids agree on one spend log per span * fix(lens): keep pricing spend logs written before litellm_call_id by their request id * fix(lens): price every attempt a model call's ids name when they agree --- .../python_contract.json | 3 +- .../callbacks-legacy-python/src/adapter.rs | 7 +- .../src/test_support.rs | 2 +- litellm-rust/crates/traces-cache/src/cache.rs | 1 + litellm-rust/crates/traces-cache/src/spend.rs | 1 + .../crates/traces-cache/tests/read.rs | 2 + .../crates/traces-cache/tests/snapshots.rs | 1 + .../0016_spend_provider_request_id.sql | 4 + .../traces-clickhouse/query/spend_batch.sql | 5 +- .../query/spend_by_response_ids.sql | 5 +- .../query/trace_list_span_batch.sql | 2 +- .../query/trace_page_spans.sql | 2 +- .../query/trace_span_batch.sql | 2 +- .../traces-clickhouse/query/trace_spans.sql | 2 +- .../traces-clickhouse/src/query/named.rs | 11 +- .../crates/traces-clickhouse/src/span_row.rs | 5 +- .../traces-clickhouse/tests/migrations.rs | 5 + .../tests/queries/support.rs | 3 +- .../crates/traces-clickhouse/tests/reads.rs | 164 +- litellm-rust/crates/traces/src/lib.rs | 3 +- .../src/normalize/format/claude_code.rs | 4 +- .../src/normalize/instrumentation/mod.rs | 11 +- .../crates/traces/src/normalize/mod.rs | 48 +- litellm-rust/crates/traces/src/otlp/span.rs | 6 +- litellm-rust/crates/traces/src/query/named.rs | 15 + .../crates/traces/src/resolve/resolution.rs | 35 +- .../crates/traces/src/resolve/spend.rs | 106 +- .../crates/traces/src/resolve/view.rs | 40 +- litellm-rust/crates/traces/src/view.rs | 15 + litellm-rust/crates/traces/tests/captures.rs | 9 +- .../traces/tests/normalization_formats.rs | 25 + litellm-rust/crates/traces/tests/normalize.rs | 2 + .../crates/traces/tests/query/named.rs | 6 +- litellm-rust/crates/traces/tests/resolve.rs | 1850 ++++++++++------- .../clickhouse/clickhouse_spend_logger.py | 6 + litellm/litellm_core_utils/core_helpers.py | 4 +- .../llm_response_utils/get_headers.py | 18 +- litellm/llms/custom_httpx/llm_http_handler.py | 1 + .../rust_bridge/callbacks_legacy_python.py | 10 +- litellm/rust_bridge/trace/generated/types.py | 7 + litellm/tracing/types.py | 1 + .../trace_codegen/schemas/traces/Trace.json | 48 +- .../schemas/traces/TracePage.json | 9 +- .../test_get_headers.py | 15 + .../test_clickhouse_spend_logger.py | 13 + .../litellm_core_utils/test_core_helpers.py | 10 + .../custom_httpx/test_llm_http_handler.py | 7 +- .../rust_bridge/messages/test_route_host.py | 2 +- .../test_callbacks_legacy_python.py | 12 +- .../src/components/lens/data/demo/fixtures.ts | 5 + .../lens/data/demo/lensDemoLongTrace.ts | 2 + .../lens/traces/detail/run/RunHeader.tsx | 8 +- .../span/DetailPane.integration.test.tsx | 70 +- .../lens/traces/detail/span/PaneHeader.tsx | 24 +- .../lens/traces/detail/span/RequestTab.tsx | 45 +- .../lens/traces/detail/span/SpanPane.tsx | 7 +- .../lens/traces/detail/span/SpendLogLink.tsx | 75 + .../lens/traces/detail/tree/TreeRows.tsx | 6 + .../AgentTracesSection.integration.test.tsx | 2 +- .../traces/list/AgentTracesTable.test.tsx | 54 +- .../lens/traces/list/AgentTracesTable.tsx | 29 +- .../list/runSearch/__fixtures__/runs.ts | 1 + .../logs/detail/DrawerHeader.test.tsx | 36 + .../components/logs/detail/DrawerHeader.tsx | 18 +- .../logs/detail/LogDetailsDrawer.tsx | 5 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 + 66 files changed, 2098 insertions(+), 862 deletions(-) create mode 100644 litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/SpendLogLink.tsx diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 9ed13ae5ed5..f65516265c5 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -88,7 +88,8 @@ "call_type" ], "stream_opened": [ - "logger" + "logger", + "head" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 21f563d9f3b..70b7ecb45fd 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -536,7 +536,7 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } - Streaming::Opened.call(py, (self.logger()?.object(py),))?; + Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() { Python::initialize(); Python::attach(|py| { - let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None"); + let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}"); let mut logging = LegacyLogging { operation: crate::LoggingOperation::Messages, ..logged(py, &locals, true) }; logging - .on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind()) + .on_stream_open(py, &local(&locals, "head").unbind()) .unwrap(); logging .on_stream_chunk(py, &local(&locals, "first").unbind()) @@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response &locals, c" assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.calls[0][1] is head chunks = logger.calls[1][1] assert len(chunks) == 2 assert chunks[0] is first diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 46c93369100..f252a45b562 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -84,7 +84,7 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger: logger.record('stream_opened', None), + 'stream_opened': lambda logger, head: logger.record('stream_opened', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index f308714146e..a5526a39963 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -298,6 +298,7 @@ mod tests { fn row(span_id: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: String::new(), name: "run".into(), diff --git a/litellm-rust/crates/traces-cache/src/spend.rs b/litellm-rust/crates/traces-cache/src/spend.rs index 1cfcde841d0..6ea2cc79257 100644 --- a/litellm-rust/crates/traces-cache/src/spend.rs +++ b/litellm-rust/crates/traces-cache/src/spend.rs @@ -48,6 +48,7 @@ pub(super) async fn spend( let params = SpendByResponseIdsParams { access: access.clone(), response_ids: lookup.response_ids, + provider_request_ids: lookup.provider_request_ids, request_ids: lookup.request_ids, trace_ids: lookup.trace_ids, start_ms: window.start, diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index d714d7c9ed4..3f76d559a2b 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -260,6 +260,7 @@ fn access() -> ReadAccessParams { fn span(index: usize) -> TraceSpansRow { TraceSpansRow { trace_id: "trace".into(), + original_trace_id: String::new(), span_id: format!("span-{index}"), parent_span_id: if index == 0 { String::new() @@ -523,6 +524,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { litellm_call_id: String::new(), response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), team_id: "team".into(), diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index 42085d3e95b..0a206004080 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120); fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), diff --git a/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql new file mode 100644 index 00000000000..443ad0688e6 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql @@ -0,0 +1,4 @@ +ALTER TABLE {database}.spend_logs + ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id, + ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id + TYPE bloom_filter(0.001) GRANULARITY 1 diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql index 3918286a61f..55f8fe88a88 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -15,7 +15,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql index da64dafbc39..e6402fff6cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql @@ -1,4 +1,4 @@ -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -14,7 +14,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql index 679edfdec2e..d6ab416dfd3 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql index b81bba61e7c..27a8e2ac0da 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql index 967ef2fcf1f..a084325feaa 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index 2e0ac4f6dfb..306692e9709 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/named.rs b/litellm-rust/crates/traces-clickhouse/src/query/named.rs index beb4b42f76d..b7e70632480 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/named.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/named.rs @@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams; struct TraceSpansRowEncoding { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding { #[serde(flatten)] pub access: contracts::ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, #[serde(deserialize_with = "super::number::deserialize")] @@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, @@ -344,7 +349,7 @@ mod tests { quoted, ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), quoted, ); round_trip::( @@ -356,7 +361,7 @@ mod tests { quoted, ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), quoted, ); } @@ -374,7 +379,7 @@ mod tests { quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), quoted, ); } diff --git a/litellm-rust/crates/traces-clickhouse/src/span_row.rs b/litellm-rust/crates/traces-clickhouse/src/span_row.rs index 0189dbff2c8..ad2863d6c5d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/span_row.rs +++ b/litellm-rust/crates/traces-clickhouse/src/span_row.rs @@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str { .flatten() .find_map(|key| match key { CallKey::ProviderResponse(id) => Some(id.as_str()), - CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, + CallKey::LiteLlmRequest(_) + | CallKey::ProviderRequest(_) + | CallKey::Transport + | CallKey::GatewayAttempt => None, }) .unwrap_or_default() } diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index d90c366118b..e51d9083c59 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins( Parameter::Strings(vec!["response-1".into()]), ), ("request_ids".into(), Parameter::Strings(Vec::new())), + ( + "provider_request_ids".into(), + Parameter::Strings(Vec::new()), + ), ("trace_ids".into(), Parameter::Strings(Vec::new())), ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), @@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility( "api_key_hash": legacy_key.unwrap_or_default(), }))?, response_ids: vec!["shared-response".into()], + provider_request_ids: Vec::new(), request_ids: Vec::new(), trace_ids: Vec::new(), start_ms: timestamp / 1_000_000 - 1, diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs index 371424c63d6..7532d9aecc9 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs @@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap Some(id.as_str()), - litellm_traces::CallKey::Transport + litellm_traces::CallKey::ProviderRequest(_) + | litellm_traces::CallKey::Transport | litellm_traces::CallKey::GatewayAttempt => None, }) .unwrap_or_default() diff --git a/litellm-rust/crates/traces-clickhouse/tests/reads.rs b/litellm-rust/crates/traces-clickhouse/tests/reads.rs index e1996d6af04..b3d1b72bec1 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/reads.rs @@ -601,7 +601,7 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( #[rstest] #[tokio::test] -async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback( +async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads( #[future(awt)] migrated_database: TestResult, ) -> TestResult { let fixture = migrated_database?; @@ -610,31 +610,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let start_ms = 1_790_000_000_000_i64; let cases = [ ( - "gateway", - "provider-request", - "gateway", + "same-key", + "provider_response:same-key", "team-a", "key-a", Some(0.25), ), - ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)), - ("conflict", "conflict", "different", "team-a", "key-a", None), + ( + "other-key", + "provider_response:other-key", + "team-a", + "key-b", + None, + ), ( "foreign-team", - "request", - "foreign-team", + "provider_response:foreign-team", "team-b", "key-a", None, ), ( - "foreign-key", - "request", - "foreign-key", + "call-id-other-key", + "litellm_request:call-id-other-key", "team-a", "key-b", None, ), + ( + "call-id-foreign-team", + "litellm_request:call-id-foreign-team", + "team-b", + "key-a", + None, + ), + ("transport-only", "transport:", "team-a", "key-a", None), ]; insert_rows( client, @@ -643,7 +653,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::OtelTraces, cases .iter() - .map(|(id, _, _, _, _, _)| { + .map(|(id, key, _, _, _)| { BTreeMap::from([ ("Timestamp".into(), json!(start_ms * 1_000_000)), ("Duration".into(), json!(1_000_000)), @@ -652,7 +662,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback ("ObservationType".into(), json!("llm")), ("TeamId".into(), json!("team-a")), ("ApiKeyHash".into(), json!("key-a")), - ("CallKeys".into(), json!([format!("litellm_request:{id}")])), + ("CallKeys".into(), json!([key])), ("CallEvidence".into(), json!("complete")), ]) }) @@ -666,11 +676,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::SpendLogs, cases .iter() - .map(|(_, request, call_id, team, key, _)| { + .map(|(id, _, team, key, _)| { BTreeMap::from([ - ("request_id".into(), json!(request)), - ("response_id".into(), json!("provider-response")), - ("litellm_call_id".into(), json!(call_id)), + ("request_id".into(), json!(format!("request-{id}"))), + ("response_id".into(), json!(id)), + ("litellm_call_id".into(), json!(id)), ("team_id".into(), json!(team)), ("api_key".into(), json!(key)), ("start_time".into(), json!(start_ms)), @@ -689,13 +699,13 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let access = ReadAccessParams { all_teams: false, user_id: String::new(), - team_ids: vec!["team-a".into()], + team_ids: vec!["team-a".into(), "team-b".into()], }; let page = reader .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) .await?; assert_eq!(page.data.len(), cases.len()); - for (id, _, _, _, _, expected) in cases { + for (id, _, _, _, expected) in cases { let summary = page .data .iter() @@ -707,6 +717,122 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .ok_or("missing trace")?; assert_eq!(detail.summary.spend, expected, "{id}"); assert_eq!(summary.spend, expected, "{id}"); + assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}"); } Ok(()) } + +#[rstest] +#[case::provider_request(true)] +#[case::transport(false)] +#[tokio::test] +async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners( + #[future(awt)] migrated_database: TestResult, + #[case] response_header: bool, + #[values(false, true)] grouped: bool, +) -> TestResult { + let fixture = migrated_database?; + let client = &fixture.database.client; + let writer = Connection::writer(&fixture.database.url)?; + let trace_id = if grouped { + "grouped-trace" + } else { + "original-trace" + }; + let start_ms = 1_790_000_000_000_i64; + let keys = if response_header { + vec!["provider_response:req_native"] + } else { + Vec::new() + }; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::OtelTraces, + vec![BTreeMap::from([ + ("Timestamp".into(), json!(start_ms * 1_000_000)), + ("TraceId".into(), json!(trace_id)), + ("SpanId".into(), json!("native-call")), + ("SpanName".into(), json!("claude_code.llm_request")), + ("ObservationType".into(), json!("llm")), + ("Framework".into(), json!("claude-code")), + ("TeamId".into(), json!("team-a")), + ("ApiKeyHash".into(), json!("key-a")), + ("CallKeys".into(), json!(keys)), + ( + "CallEvidence".into(), + json!(if response_header { + "complete" + } else { + "unknown" + }), + ), + ( + "SpanAttributes".into(), + json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}), + ), + ])], + ) + .await?; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::SpendLogs, + ["key-a", "key-b"] + .into_iter() + .map(|key| { + BTreeMap::from([ + ("request_id".into(), json!(format!("log-{key}"))), + ("response_id".into(), json!("msg_native")), + ("provider_request_id".into(), json!("req_native")), + ( + "trace_id".into(), + json!(if response_header { + "" + } else { + "original-trace" + }), + ), + ( + "span_id".into(), + json!(if response_header { "" } else { "native-call" }), + ), + ("team_id".into(), json!("team-a")), + ("api_key".into(), json!(key)), + ("start_time".into(), json!(start_ms)), + ("end_time".into(), json!(start_ms + 1)), + ("spend".into(), json!(0.25)), + ]) + }) + .collect(), + ) + .await?; + let connection = fixture + .readers + .connection(client, &QueryScope::All, "fixture-secret") + .await?; + let (reader, store) = make_reader(client, connection); + let access = ReadAccessParams { + all_teams: true, + user_id: String::new(), + team_ids: Vec::new(), + }; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; + assert_eq!(page.data.len(), 1); + let summary = &page.data[0]; + let detail = reader + .get_trace(&store, &access, trace_id, &summary.trace_ref) + .await? + .ok_or("missing trace")?; + assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1)); + assert_eq!(detail.summary.spend, summary.spend); + assert_eq!( + detail.spans[0].spend_log_request_id.as_deref(), + Some("log-key-a") + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index f79fa59bdba..b51701638e8 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -44,5 +44,6 @@ pub use tenant::Tenant; pub use truncate::{truncate_messages, truncate_value}; pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content}; pub use view::{ - AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary, + AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage, + TraceSummary, }; diff --git a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs index 18d7d74dc44..d24b9e6d8d2 100644 --- a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs +++ b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs @@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts}; use crate::{ Error, normalize::{ - CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey, + CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, ObservationType, RoleEvidence, SpanContext, attr, present, tokens, }, otlp::DecodedEvent, @@ -265,7 +265,7 @@ impl Format for ClaudeCode { output: llm_output(attributes), calls: present(attributes, &["gen_ai.response.id", "request_id"]) .map_or(CallEvidence::Unknown, |id| { - CallEvidence::complete(CallKey::ProviderResponse(id)) + CallEvidence::complete(crate::normalize::claude_call_key(id)) }), ..base }, diff --git a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs index 550afae77ca..1003834de79 100644 --- a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs @@ -216,7 +216,16 @@ impl Instrumentation { fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts { let calls = [ - present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse), + present(context.attributes, &["gen_ai.response.id"]).map(|id| { + if matches!( + context.scope, + crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE + ) { + crate::normalize::claude_call_key(id) + } else { + CallKey::ProviderResponse(id) + } + }), present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest), ] .into_iter() diff --git a/litellm-rust/crates/traces/src/normalize/mod.rs b/litellm-rust/crates/traces/src/normalize/mod.rs index f2c5376af10..53d5e521799 100644 --- a/litellm-rust/crates/traces/src/normalize/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/mod.rs @@ -65,16 +65,26 @@ pub enum CallKey { LiteLlmRequest(String), /// The provider response id returned to the caller (`spend_logs.response_id`). ProviderResponse(String), + ProviderRequest(String), /// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id. Transport, GatewayAttempt, } +pub(crate) fn claude_call_key(id: String) -> CallKey { + if id.starts_with("msg_") { + CallKey::ProviderResponse(id) + } else { + CallKey::ProviderRequest(id) + } +} + impl fmt::Display for CallKey { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"), Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"), + Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"), Self::Transport => formatter.write_str("transport:"), Self::GatewayAttempt => formatter.write_str("gateway_attempt:"), } @@ -89,6 +99,9 @@ impl FromStr for CallKey { Some(("provider_response", id)) if !id.is_empty() => { Ok(Self::ProviderResponse(id.to_owned())) } + Some(("provider_request", id)) if !id.is_empty() => { + Ok(Self::ProviderRequest(id.to_owned())) + } Some(("litellm_request", id)) if !id.is_empty() => { Ok(Self::LiteLlmRequest(id.to_owned())) } @@ -127,24 +140,45 @@ pub enum CallEvidence { impl CallEvidence { pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet { - if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { + let native = Self::native_request(row); + let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())]) } else { row.call_keys.iter().cloned().collect() + }; + if !native { + return keys; } + if keys.is_empty() { + return BTreeSet::from([CallKey::Transport]); + } + keys.into_iter() + .map(|key| match key { + CallKey::ProviderResponse(id) => claude_call_key(id), + key => key, + }) + .collect() + } + + fn native_request(row: &crate::query::named::TraceSpansRow) -> bool { + matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk") + && row.name == "claude_code.llm_request" } pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self { - let kind = row - .call_evidence - .unwrap_or(if Self::row_keys(row).is_empty() { + let keys = Self::row_keys(row); + let kind = if Self::native_request(row) { + CallEvidenceKind::Complete + } else { + row.call_evidence.unwrap_or(if keys.is_empty() { CallEvidenceKind::Unknown } else { CallEvidenceKind::Complete - }); + }) + }; match kind { - CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)), - CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)), + CallEvidenceKind::Complete => Self::Complete(keys), + CallEvidenceKind::Partial => Self::Partial(keys), CallEvidenceKind::Unknown => Self::Unknown, } } diff --git a/litellm-rust/crates/traces/src/otlp/span.rs b/litellm-rust/crates/traces/src/otlp/span.rs index 4e63d78d286..58ccb9ea3b4 100644 --- a/litellm-rust/crates/traces/src/otlp/span.rs +++ b/litellm-rust/crates/traces/src/otlp/span.rs @@ -193,9 +193,9 @@ fn decoded_span( .into_iter() .flatten() .map(|key| match key { - crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => { - id.len() + size_of::() - } + crate::CallKey::LiteLlmRequest(id) + | crate::CallKey::ProviderResponse(id) + | crate::CallKey::ProviderRequest(id) => id.len() + size_of::(), crate::CallKey::Transport | crate::CallKey::GatewayAttempt => { size_of::() } diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index a981bc46471..757012de16f 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -68,6 +68,8 @@ pub struct TraceSpansParams { pub struct TraceSpansRow { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -113,6 +115,16 @@ pub struct TraceSpansRow { pub user_id: String, } +impl TraceSpansRow { + pub(crate) fn transport_trace_id(&self) -> &str { + if self.original_trace_id.is_empty() { + &self.trace_id + } else { + &self.original_trace_id + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TracePageSpansParams { #[serde(flatten)] @@ -163,6 +175,7 @@ pub struct SpendByResponseIdsParams { #[serde(flatten)] pub access: ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, pub start_ms: i64, @@ -175,6 +188,8 @@ pub struct SpendByResponseIdsRow { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, diff --git a/litellm-rust/crates/traces/src/resolve/resolution.rs b/litellm-rust/crates/traces/src/resolve/resolution.rs index 9b47755a16f..e19a98a1ede 100644 --- a/litellm-rust/crates/traces/src/resolve/resolution.rs +++ b/litellm-rust/crates/traces/src/resolve/resolution.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use indexmap::IndexMap; use crate::{ + SpendMatch, normalize::{CallKey, ObservationType}, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> { types: HashMap<&'a str, ObservationType>, tool_failures: HashMap<&'a str, &'a TraceSpansRow>, pub(super) model_calls: Vec, + call_matches: HashMap>, } +pub(super) type CallMatch<'a> = (Option>, SpendMatch); + impl<'a> Resolution<'a> { pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self { let graph = Graph::new(rows); @@ -36,7 +40,7 @@ impl<'a> Resolution<'a> { let types: HashMap<&str, ObservationType> = (0..rows.len()) .map(|index| (graph.id(index), resolved_type(&graph, index, named_agents))) .collect(); - let model_calls = (0..rows.len()) + let model_calls: Vec = (0..rows.len()) .filter(|index| { types[graph.id(*index)] == ObservationType::Llm && !graph @@ -45,7 +49,7 @@ impl<'a> Resolution<'a> { .any(|descendant| types[graph.id(descendant)] == ObservationType::Llm) }) .collect(); - Self { + let resolution = Self { ownership: Ownership { team_id: &rows[0].team_id, api_key_hash: &rows[0].api_key_hash, @@ -65,6 +69,16 @@ impl<'a> Resolution<'a> { .map(|row| (row.tool_call_id.as_str(), row)) .collect(), model_calls, + call_matches: HashMap::new(), + }; + let call_matches = resolution + .model_calls + .iter() + .map(|call| (*call, resolution.resolve_call_match(*call))) + .collect(); + Self { + call_matches, + ..resolution } } @@ -118,7 +132,24 @@ impl<'a> Resolution<'a> { spend::requests(self.row(index), &self.ownership, self.spend) } + pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> { + self.call_matches.get(&call) + } + pub(super) fn call_requests(&self, call: usize) -> Option> { + self.call_match(call) + .and_then(|(requests, _)| requests.clone()) + } + + fn resolve_call_match(&self, call: usize) -> CallMatch<'a> { + if let Some(requests) = self.resolve_call_requests(call) { + return (Some(requests), SpendMatch::Matched); + } + let evidence = self.requests(call); + (None, evidence.unmatched_reason()) + } + + fn resolve_call_requests(&self, call: usize) -> Option> { let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| { self.kind(*ancestor) == ObservationType::Llm && self diff --git a/litellm-rust/crates/traces/src/resolve/spend.rs b/litellm-rust/crates/traces/src/resolve/spend.rs index ce23362b7f8..a6b2fc0dbc5 100644 --- a/litellm-rust/crates/traces/src/resolve/spend.rs +++ b/litellm-rust/crates/traces/src/resolve/spend.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use indexmap::IndexMap; use crate::{ - CallEvidence, CallEvidenceKind, CallKey, + CallEvidence, CallEvidenceKind, CallKey, SpendMatch, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -12,6 +12,7 @@ use crate::{ pub struct SpendLookup { pub response_ids: Vec, pub request_ids: Vec, + pub provider_request_ids: Vec, /// Traces whose transport spans LiteLLM logged by `traceparent`. pub trace_ids: Vec, } @@ -45,13 +46,21 @@ impl SpendLookup { }) .collect(), ), + provider_request_ids: sorted( + keys() + .filter_map(|(_, key)| match key { + CallKey::ProviderRequest(id) => Some(id.clone()), + _ => None, + }) + .collect(), + ), trace_ids: sorted( keys() .filter_map(|(row, key)| match key { CallKey::Transport | CallKey::GatewayAttempt - if !row.trace_id.is_empty() => + if !row.transport_trace_id().is_empty() => { - Some(row.trace_id.clone()) + Some(row.transport_trace_id().to_owned()) } _ => None, }) @@ -61,7 +70,10 @@ impl SpendLookup { } pub fn is_empty(&self) -> bool { - self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty() + self.response_ids.is_empty() + && self.request_ids.is_empty() + && self.provider_request_ids.is_empty() + && self.trace_ids.is_empty() } } @@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>; enum KeyFamily { GatewayCall, ProviderResponse, + ProviderRequest, Transport, } @@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily { match key { CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall, CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse, + CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest, CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport, } } pub(super) enum KeyMatch<'a> { Missing, + Conflicting, Unique(&'a SpendRow), Ambiguous(Requests<'a>), } @@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> { fn unique(&self) -> Option<&'a SpendRow> { match self { Self::Unique(request) => Some(request), - Self::Missing | Self::Ambiguous(_) => None, + Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None, } } fn agrees_with(&self, selected: &[&SpendRow]) -> bool { match self { - Self::Missing => false, + Self::Missing | Self::Conflicting => false, Self::Unique(request) => selected .iter() .any(|row| row.identity() == request.identity()), @@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> { } impl<'a> SpendEvidence<'a> { + pub(super) fn unmatched_reason(&self) -> SpendMatch { + match self { + Self::Unknown => SpendMatch::NoCallId, + Self::Partial(_) => SpendMatch::IncompleteEvidence, + Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId, + Self::Complete(matches) + if matches.iter().any(|evidence| { + matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_)) + }) => + { + SpendMatch::Ambiguous + } + Self::Complete(matches) + if matches + .iter() + .any(|evidence| matches!(evidence, KeyMatch::Missing)) => + { + SpendMatch::NoSpendLog + } + Self::Complete(_) => SpendMatch::Ambiguous, + } + } + pub(super) fn complete_requests(&self) -> Option> { match self { Self::Complete(matches) if !matches.is_empty() => { @@ -186,15 +224,16 @@ fn matches<'a>( CallKey::ProviderResponse(id) => { !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id) } + CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id, CallKey::LiteLlmRequest(id) => { !id.is_empty() && (spend.litellm_call_id == *id || (spend.litellm_call_id.is_empty() && spend.request_id == *id)) } CallKey::Transport | CallKey::GatewayAttempt => { - !row.trace_id.is_empty() + !row.transport_trace_id().is_empty() && !row.span_id.is_empty() - && spend.trace_id == row.trace_id + && spend.trace_id == row.transport_trace_id() && spend.span_id == row.span_id } }; @@ -253,7 +292,8 @@ pub(super) fn requests<'a>( let matches = aliases .into_iter() .map(|(_, requests)| { - KeyMatch::new( + let had_candidates = !requests.is_empty(); + let matched = KeyMatch::new( requests .into_iter() .filter(|request| { @@ -262,7 +302,12 @@ pub(super) fn requests<'a>( .all(|family| family.contains(&request.identity())) }) .collect(), - ) + ); + if had_candidates && matches!(matched, KeyMatch::Missing) { + KeyMatch::Conflicting + } else { + matched + } }) .collect(); match evidence.kind() { @@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option { }) } -pub(super) fn total(calls: &[Option>]) -> Option { - if calls.is_empty() { - return None; - } - let requests: Option> = calls - .iter() - .map(|requests| requests.as_ref()) - .collect::>>() - .map(|calls| calls.into_iter().flatten().copied().collect()); - let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests? +pub(super) fn unique<'a>(requests: impl IntoIterator) -> Requests<'a> { + requests .into_iter() .map(|request| (request.identity(), request)) - .collect(); - request_cost(&unique.into_values().collect::>()) + .collect::>() + .into_values() + .collect() +} + +pub(super) struct Priced { + pub(super) spend: Option, + pub(super) priced_calls: u64, +} + +pub(super) fn total(calls: &[Option>]) -> Priced { + let priced: Vec<&Requests<'_>> = calls + .iter() + .flatten() + .filter(|requests| request_cost(requests).is_some()) + .collect(); + let spend = if priced.is_empty() { + None + } else { + request_cost(&unique( + priced.iter().flat_map(|requests| requests.iter().copied()), + )) + }; + Priced { + spend, + priced_calls: priced.len() as u64, + } } diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 87688d9b36e..9e9edd51bf4 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -6,7 +6,7 @@ use time::OffsetDateTime; use crate::{ normalize::ObservationType, query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow}, - view::{AgentNode, Span, SpanStatus, Trace, TraceSummary}, + view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary}, }; use super::{ @@ -23,7 +23,18 @@ fn optional(value: &str) -> Option { fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span { let row = resolution.row(index); let status = resolution.status_source(index); - let requests = resolution.requests(index).complete_requests(); + let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) { + (requests.clone(), Some(*matched)) + } else { + (resolution.requests(index).complete_requests(), None) + }; + let spend = requests + .as_ref() + .and_then(|requests| request_cost(requests)); + let spend_log_request_id = match (spend_match, requests.as_deref()) { + (Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()), + _ => None, + }; Span { span_id: row.span_id.clone(), parent_span_id: optional(&row.parent_span_id), @@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span input_tokens: row.input_tokens, output_tokens: row.output_tokens, litellm_request_id: optional(&row.litellm_request_id), - spend: requests - .as_ref() - .and_then(|requests| request_cost(requests)), + spend, + spend_log_request_id, + spend_match, } } @@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .filter(|(owner, _)| *owner == name) .map(|(_, requests)| requests.clone()) .collect(); + let priced = total(&owned_calls); AgentNode { name: name.to_owned(), parent_agent, @@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .map(|span| graph.rows[*span].duration_ns) .sum::() as f64 / NANOS_PER_MS, - spend: total(&owned_calls), + spend: priced.spend, + priced_calls: priced.priced_calls, } }) .collect() @@ -160,6 +173,12 @@ pub fn resolve_trace( } else { calls.iter().map(|call| &rows[*call]).collect() }; + let priced = total( + &calls + .iter() + .map(|call| resolution.call_requests(*call)) + .collect::>(), + ); let first_input = spans .iter() .zip(rows) @@ -208,12 +227,8 @@ pub fn resolve_trace( input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(), output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(), models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())), - spend: total( - &calls - .iter() - .map(|call| resolution.call_requests(*call)) - .collect::>(), - ), + spend: priced.spend, + priced_calls: priced.priced_calls, }; Some(Trace { summary, @@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary { output_tokens: row.output_tokens, models: row.models.clone(), spend: None, + priced_calls: 0, } } diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index a864c740526..5a214d33da7 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -37,6 +37,19 @@ pub struct Span { pub output_tokens: u32, pub litellm_request_id: Option, pub spend: Option, + pub spend_log_request_id: Option, + pub spend_match: Option, +} + +#[macro_rules_attribute::apply(response_type)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum SpendMatch { + Matched, + NoCallId, + NoSpendLog, + Ambiguous, + IncompleteEvidence, } /// One distinct agent in a trace: 200 invocations of `researcher` are one node. @@ -50,6 +63,7 @@ pub struct AgentNode { pub tool_calls: u64, pub duration_ms: f64, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] @@ -80,6 +94,7 @@ pub struct TraceSummary { pub output_tokens: u64, pub models: Vec, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] diff --git a/litellm-rust/crates/traces/tests/captures.rs b/litellm-rust/crates/traces/tests/captures.rs index ca498fc914a..ff20db617f6 100644 --- a/litellm-rust/crates/traces/tests/captures.rs +++ b/litellm-rust/crates/traces/tests/captures.rs @@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec TraceSpansRow { .into_iter() .flatten() .find_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()), CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, }) .unwrap_or_default(); TraceSpansRow { trace_id: span.trace_id, + original_trace_id: String::new(), span_id: span.span_id, parent_span_id: span.parent_span_id, name: span.name, @@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow { .expect("valid unrelated transport timestamp"); TraceSpansRow { trace_id: call.trace_id.clone(), + original_trace_id: call.original_trace_id.clone(), span_id: format!("unrelated-transport-{}", call.span_id), parent_span_id: call.parent_span_id.clone(), name: "unrelated-http".into(), @@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence( let response_ids: Vec<_> = keys .iter() .filter_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => { + Some(id.clone()) + } CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => { None } diff --git a/litellm-rust/crates/traces/tests/normalization_formats.rs b/litellm-rust/crates/traces/tests/normalization_formats.rs index 124f03593c8..e8b11ece64a 100644 --- a/litellm-rust/crates/traces/tests/normalization_formats.rs +++ b/litellm-rust/crates/traces/tests/normalization_formats.rs @@ -928,3 +928,28 @@ fn convention_markers_keep_genai_call_evidence( let marked = decode(span, scope, &marked_attributes, vec![]).unwrap(); assert_eq!(marked.normalized.calls, plain.normalized.calls); } + +#[rstest] +#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))] +#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))] +fn native_claude_preserves_the_provider_id_family( + span: Span, + #[case] id: &str, + #[case] key: CallKey, +) { + let native = Span { + name: "claude_code.llm_request".into(), + ..span + }; + let decoded = decode( + native, + "com.anthropic.claude_code.tracing", + &[("gen_ai.response.id", id)], + Vec::new(), + ) + .unwrap(); + assert_eq!( + decoded.normalized.calls, + CallEvidence::Complete(std::collections::BTreeSet::from([key])) + ); +} diff --git a/litellm-rust/crates/traces/tests/normalize.rs b/litellm-rust/crates/traces/tests/normalize.rs index 923e2006088..bfff82fcdd7 100644 --- a/litellm-rust/crates/traces/tests/normalize.rs +++ b/litellm-rust/crates/traces/tests/normalize.rs @@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) { #[rstest] #[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))] #[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))] +#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))] #[case::transport(litellm_traces::CallKey::Transport)] #[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)] fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { @@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { #[rstest] #[case::missing_separator("provider_response")] #[case::missing_response("provider_response:")] +#[case::missing_provider_request("provider_request:")] #[case::missing_request("litellm_request:")] #[case::transport_id("transport:unexpected")] #[case::gateway_attempt_separator("gateway_attempt")] diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 6b853ebf151..3ac9668e42a 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases( json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), )); round_trip::(request( - json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), )); } @@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() { json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), @@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() { json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}), ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), ); } diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index 641d15db5b9..8d65adb0c0f 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -1,5 +1,5 @@ use litellm_traces::{ - AgentNode, SpanStatus, iso_time, listed_summary, + AgentNode, SpanStatus, SpendMatch, iso_time, listed_summary, query::named::{ListTracesRow, SpendByResponseIdsRow, TraceSpansRow}, resolve_trace, }; @@ -11,6 +11,7 @@ const MS: i64 = 1_000_000; fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), @@ -32,8 +33,8 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac call_keys: Vec::new(), call_evidence: None, tool_call_id: String::new(), - team_id: String::new(), - api_key_hash: String::new(), + team_id: "team".into(), + api_key_hash: "key".into(), user_id: String::new(), } } @@ -55,31 +56,18 @@ fn llm(span_id: &str, parent: &str, agent: &str, response_id: &str) -> TraceSpan } } -fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { - span.team_id = team.into(); - span.user_id = user.into(); - span.api_key_hash = key.into(); - span -} - -fn spend( - request_id: &str, - response_id: &str, - team: &str, - user: &str, - key: &str, - cost: f64, -) -> SpendByResponseIdsRow { +fn spend(request_id: &str, response_id: &str, cost: f64) -> SpendByResponseIdsRow { SpendByResponseIdsRow { request_id: request_id.into(), - response_id: response_id.into(), litellm_call_id: String::new(), + response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), - team_id: team.into(), - api_key: key.into(), - user: user.into(), + api_key: "key".into(), + user: String::new(), + team_id: "team".into(), spend: Some(cost), start_ms: T0 / MS, } @@ -221,6 +209,7 @@ fn repeated_subagent_invocations_aggregate_into_one_node() { tool_calls: 1, duration_ms: 1000.0, spend: None, + priced_calls: 0, } ); let researcher = &trace.agents[1]; @@ -384,38 +373,15 @@ fn names_and_frameworks_are_sorted_and_distinct() { } #[rstest] -fn repeated_response_counts_once_and_other_owners_are_ignored() { +fn repeated_response_id_counts_once() { let rows = [ - owned( - row("root", "", "agent", "agent", "agent"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-1", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-2", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), + row("root", "", "agent", "agent", "agent"), + llm("llm-1", "root", "agent", "response-1"), + llm("llm-2", "root", "agent", "response-1"), ]; let spend = [ - spend("request-other", "response-1", "team-b", "", "key-b", 99.0), - spend("request-1", "response-1", "team-a", "", "key-a", 0.25), - spend( - "request-other-key", - "unrelated-response", - "team-a", - "", - "key-c", - 50.0, - ), + spend("request-1", "response-1", 0.25), + spend("request-other", "unrelated-response", 50.0), ]; let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); assert_eq!(trace.summary.spend, Some(0.25)); @@ -431,352 +397,363 @@ fn repeated_response_counts_once_and_other_owners_are_ignored() { } #[rstest] -fn ambiguous_response_id_keeps_cost_unknown() { - let rows = [owned( - llm("llm-1", "", "agent", "response-1"), - "", - "user", - "key-a", - )]; - let spend = [ - spend("response-1", "response-1", "", "user", "key-a", 0.25), - spend( - "response-1_cache_hit123", - "response-1", - "", - "user", - "key-a", - 0.0, - ), - ]; - let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); - assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); -} - -#[rstest] -#[case::key_differs("team", "", "export", "team", "", "request", false)] -#[case::shared_key("team", "", "export", "team", "", "export", true)] -#[case::shared_user("", "user", "export", "", "user", "request", true)] -#[case::teamless_key("", "", "key", "", "", "key", true)] -#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] -#[case::other_user("", "user", "export", "", "other-user", "request", false)] -#[case::no_shared_identity("", "", "export", "", "", "request", false)] -#[case::no_identity("", "", "", "", "", "", false)] -#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] -fn cost_requires_shared_ownership( - #[case] trace_team: &str, - #[case] trace_user: &str, - #[case] trace_key: &str, - #[case] spend_team: &str, - #[case] spend_user: &str, - #[case] spend_key: &str, - #[case] known: bool, -) { +fn model_calls_link_the_spend_log_they_were_priced_from() { let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - trace_team, - trace_user, - trace_key, - ), - owned( - llm("llm", "agent", "agent", "response"), - trace_team, - trace_user, - trace_key, - ), + row("agent", "", "agent", "agent", "agent"), + llm("matched", "agent", "agent", "matched"), + llm("no-log", "agent", "agent", "missing"), + llm("no-id", "agent", "agent", ""), + llm("ambiguous", "agent", "agent", "cached"), ]; - let spend = [spend( - "request", "response", spend_team, spend_user, spend_key, 0.25, - )]; - let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); - let expected = known.then_some(0.25); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[1].spend, expected); -} - -#[rstest] -#[case::missing_id("missing_id")] -#[case::missing_spend("missing_spend")] -#[case::duplicate_spend("duplicate_spend")] -fn incomplete_call_cost_never_becomes_a_partial_total(#[case] failure: &str) { - let second_id = if failure == "missing_id" { - "" - } else { - "second" - }; - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "export", - ), - owned( - llm("first", "agent", "agent", "first"), - "team", - "", - "export", - ), - owned( - llm("second", "agent", "agent", second_id), - "team", - "", - "export", - ), + let logs = [ + spend("request-matched", "matched", 0.25), + spend("request-cached-a", "cached", 0.25), + spend("request-cached-b", "cached", 0.0), ]; - let first = spend("first", "first", "team", "", "export", 0.25); - let second = spend("second", "second", "team", "", "export", 0.25); - let duplicate = spend("duplicate", "second", "team", "", "export", 0.25); - let spend = if failure == "duplicate_spend" { - vec![first, second, duplicate] - } else { - vec![first] - }; - let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); - assert_eq!(trace.spans[1].spend, Some(0.25)); - assert_eq!(trace.spans[2].spend, None); - assert_eq!(trace.summary.spend, None); - assert_eq!(trace.agents[0].spend, None); -} - -#[rstest] -fn transport_spans_complete_a_call_without_its_own_id() { - let mut transport = row("http", "llm", "POST", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec!["transport:".parse().unwrap()]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, Some(0.5)); -} - -#[rstest] -#[case::lone_call(1, Some(0.5))] -#[case::two_calls(2, None)] -fn sibling_transports_belong_to_the_only_model_call_under_their_parent( - #[case] calls: usize, - #[case] expected: Option, -) { - let mut transport = at( - row("http", "step", "gateway.request", "framework", ""), - 2, - 10, + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + let links: Vec<_> = trace + .spans + .iter() + .map(|span| (span.spend_log_request_id.as_deref(), span.spend_match)) + .collect(); + assert_eq!( + links, + [ + (None, None), + (Some("request-matched"), Some(SpendMatch::Matched)), + (None, Some(SpendMatch::NoSpendLog)), + (None, Some(SpendMatch::NoCallId)), + (None, Some(SpendMatch::Ambiguous)), + ] ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut rows = vec![ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - for index in 0..calls { - let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); - call.call_evidence = None; - rows.push(owned(call, "team", "", "key")); - } - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] -fn sibling_transport_does_not_lose_model_call_spend( - #[case] transport_timing: Option<(i64, u64)>, - #[case] unrelated_spend: bool, +fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_a_wrapper() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", ""), + ]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-request", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!( + ( + trace.spans[1].spend, + trace.spans[1].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn unpriced_call_leaves_a_partial_total_of_the_priced_calls() { + let rows = [ + row("agent", "", "agent", "agent", "agent"), + llm("first", "agent", "agent", "first"), + llm("second", "agent", "agent", "second"), + llm("third", "agent", "agent", ""), + ]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("first", "first", 0.25)]).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.summary.priced_calls, + trace.summary.llm_calls + ), + (Some(0.25), 1, 3) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), None, None] + ); +} + +#[rstest] +fn no_priced_call_leaves_cost_unknown() { + let rows = [llm("call", "", "agent", "response")]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("other", "other", 0.25)]).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +fn gateway_logged(request_id: &str, call_id: &str, cost: f64) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..spend(request_id, &format!("chatcmpl-{request_id}"), cost) + } +} + +#[rstest] +#[case::response_id(litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), Some(0.25))] +#[case::gateway_call_id(litellm_traces::CallKey::LiteLlmRequest("gateway".into()), Some(0.25))] +#[case::other_call_id(litellm_traces::CallKey::LiteLlmRequest("other".into()), None)] +#[case::request_id_is_not_a_call_id(litellm_traces::CallKey::LiteLlmRequest("request".into()), None)] +#[case::transport(litellm_traces::CallKey::Transport, None)] +#[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt, None)] +fn only_ids_litellm_assigned_join_spend( #[case] key: litellm_traces::CallKey, #[case] expected: Option, ) { - let call = owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![litellm_traces::CallKey::ProviderResponse( - "chatcmpl-1".into(), - )], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("chat", "step", "agent", "chatcmpl-1") - }, - "team", - "", - "key", + let rows = [TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![key], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "trace".into(), + ..gateway_logged("request", "gateway", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (expected, u64::from(expected.is_some())) ); - let base_rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - call, - ]; - let rows: Vec<_> = base_rows - .into_iter() - .chain(transport_timing.map(|(start, duration)| { - let mut transport = at( - row("tool-http", "step", "GET", "framework", ""), - start, - duration, - ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![key]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - owned(transport, "team", "", "key") - })) - .collect(); - let logs: Vec<_> = std::iter::once(spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5)) - .chain(unrelated_spend.then(|| SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "tool-http".into(), - ..spend("unrelated", "unrelated", "team", "", "key", 0.75) - })) - .collect(); - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::agreeing_ids( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] -#[case::conflicting_response_id( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-b"), - None -)] -#[case::conflicting_gateway_and_response( - litellm_traces::CallKey::Transport, - "call-b", - Some("response-b"), - None -)] -#[case::agreeing_gateway_attempt( - litellm_traces::CallKey::GatewayAttempt, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] -fn gateway_attempt_identifiers_must_match_one_spend_row( - #[case] transport: litellm_traces::CallKey, - #[case] call_id: &str, - #[case] response_id: Option<&str>, +#[case::same_team("team", Some(0.25), SpendMatch::Matched)] +#[case::other_team("other-team", None, SpendMatch::NoSpendLog)] +fn spend_logs_price_only_calls_in_their_own_team( + #[case] logged_team: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let keys = [ - transport, - litellm_traces::CallKey::LiteLlmRequest(call_id.into()), - ] - .into_iter() - .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) - .collect(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(llm("call", "agent", "agent", ""), "team", "", "key"), - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: keys, - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..row("attempt", "call", "gateway.request", "framework", "") - }, - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - litellm_call_id: "call-a".into(), - trace_id: "trace".into(), - span_id: "attempt".into(), - ..spend("request-a", "response-a", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - litellm_call_id: "call-b".into(), - trace_id: "trace".into(), - span_id: "other-attempt".into(), - ..spend("request-b", "response-b", "team", "", "key", 0.5) - }, - ]; + let rows = [TraceSpansRow { + team_id: "team".into(), + ..llm("call", "", "agent", "response") + }]; + let logs = [SpendByResponseIdsRow { + team_id: logged_team.into(), + ..spend("request", "response", 0.25) + }]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[2].spend, expected); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); } #[rstest] -#[case::legacy_row("", Some(0.5))] -#[case::other_call("other-call", None)] -fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( +#[case::legacy_row_without_call_id("", Some(0.25), SpendMatch::Matched)] +#[case::call_id_names_nothing("missing", None, SpendMatch::Ambiguous)] +fn recorded_gateway_id_must_agree_unless_spend_predates_gateway_ids( #[case] logged_call_id: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let mut transport = row("http", "llm", "gateway.request", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec![ - "transport:".parse().unwrap(), - "litellm_request:gateway-call".parse().unwrap(), - ]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [gateway_logged("request", logged_call_id, 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.spans[0].spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); +} + +#[rstest] +#[case::missing_cost(None, SpendMatch::Matched)] +#[case::finite_cost(Some(0.25), SpendMatch::Matched)] +fn a_single_matched_row_is_matched_whatever_its_cost( + #[case] cost: Option, + #[case] matched: SpendMatch, +) { + let rows = [llm("call", "", "agent", "response")]; + let logs = [SpendByResponseIdsRow { + spend: cost, + ..spend("request", "response", 0.0) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.spans[0].spend_match, + trace.spans[0].spend_log_request_id.as_deref() ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), + (cost, Some(matched), Some("request")) + ); + assert_eq!(trace.summary.priced_calls, u64::from(cost.is_some())); +} + +#[rstest] +#[case::same_request("gateway", Some(0.25), SpendMatch::Matched)] +#[case::conflicting_requests("other", None, SpendMatch::Ambiguous)] +fn response_id_and_call_id_must_name_the_same_request( + #[case] call_id: &str, + #[case] expected: Option, + #[case] matched: SpendMatch, +) { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + gateway_logged("request", "gateway", 0.25), + gateway_logged("other-request", "other", 0.5), ]; - let mut logged = spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - logged.litellm_call_id = logged_call_id.into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.summary.spend, + trace.spans[0].spend_match + ), + (expected, expected, Some(matched)) + ); +} + +#[rstest] +fn call_id_picks_the_call_when_its_response_id_has_a_cache_hit_twin() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-shared".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-shared", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "cache-hit".into(), + ..spend("request_cache_hit", "chatcmpl-shared", 0.0) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.spans[0].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn wrapper_recording_a_retry_and_its_final_attempt_prices_both() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-retry".into()), + litellm_traces::CallKey::ProviderResponse("chatcmpl-final".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + spend("retry", "chatcmpl-retry", 0.25), + spend("final", "chatcmpl-final", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (Some(0.75), Some(SpendMatch::Matched)) + ); +} + +#[rstest] +fn call_id_prices_a_legacy_spend_log_without_litellm_call_id() { + let rows = [TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("legacy".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [spend("legacy", "chatcmpl-legacy", 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn spend_lookup_collects_only_assigned_ids() { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..row("span", "", "operation", "llm", "") + }; + assert_eq!( + litellm_traces::SpendLookup::new(&[recorded]), + litellm_traces::SpendLookup { + response_ids: vec!["response".into()], + provider_request_ids: Vec::new(), + request_ids: vec!["request".into()], + trace_ids: vec!["trace".into()], + } + ); +} + +#[rstest] +fn upstream_response_id_inside_a_managed_id_joins_spend() { + let rows = [llm("call", "", "agent", "chatcmpl-upstream")]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "chatcmpl-upstream".into(), + ..spend("request", "resp_managed", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn conflicting_complete_wrapper_cannot_price_an_unrelated_call() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "retry-response".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", "response"), + ]; + let logs = [ + spend("retry", "retry-response", 0.25), + spend("final", "response", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +#[rstest] +fn ambiguous_response_id_keeps_cost_unknown() { + let rows = [llm("llm-1", "", "agent", "response-1")]; + let spend = [ + spend("response-1", "response-1", 0.25), + spend("response-1_cache_hit123", "response-1", 0.0), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); } #[rstest] @@ -827,37 +804,6 @@ fn iso_time_matches_python_isoformat(#[case] ms: i64, #[case] expected: &str) { assert_eq!(iso_time(ms), expected); } -#[rstest] -#[case::narrows_ambiguity("request-a", Some(0.25))] -#[case::conflicting_exact_request("request-c", None)] -fn complete_wrapper_reconciles_ambiguous_response( - #[case] exact_id: &str, - #[case] expected: Option, -) { - let wrapper = TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") - }; - let rows = [ - wrapper, - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("request-a", "response", "team", "", "key", 0.25), - spend("request-b", "response", "team", "", "key", 0.5), - spend("request-c", "other-response", "team", "", "key", 0.75), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - #[rstest] #[case::missing(None, None)] #[case::free(Some(0.0), Some(0.0))] @@ -868,15 +814,10 @@ fn complete_correlation_requires_known_finite_cost( #[case] cost: Option, #[case] expected: Option, ) { - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let logged = SpendByResponseIdsRow { spend: cost, - ..spend("request", "response", "team", "", "key", 0.25) + ..spend("request", "response", 0.25) }; let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); assert_eq!(trace.summary.spend, expected); @@ -884,160 +825,39 @@ fn complete_correlation_requires_known_finite_cost( assert_eq!(trace.spans[0].spend, expected); } -#[rstest] -#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] -#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] -#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] -#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] -fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( - #[case] retry_logged: bool, - #[case] retry_evidence: litellm_traces::CallEvidenceKind, - #[case] expected: Option, -) { - let transport = |id: &str| { - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec!["transport:".parse().unwrap()], - call_evidence: Some(if id == "first" { - retry_evidence - } else { - litellm_traces::CallEvidenceKind::Complete - }), - ..row(id, "call", "POST", "framework", "") - }, - "team", - "", - "key", - ) - }; - let rows = [ - owned( - llm("call", "", "agent", "final-response"), - "team", - "", - "key", - ), - transport("first"), - transport("second"), - ]; - let logs = [ - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "first".into(), - ..spend("retry", "retry-response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "second".into(), - ..spend("final", "final-response", "team", "", "key", 0.5) - }, - spend("cached", "final-response", "team", "", "key", 0.0), - ]; - let available = if retry_logged { &logs[..] } else { &logs[1..] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - -#[rstest] -#[case::same_request(false)] -#[case::ambiguous_response(true)] -fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![ - "provider_response:response".parse().unwrap(), - "litellm_request:request".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - spend("request", "response", "team", "", "key", 0.25), - spend("cached", "response", "team", "", "key", 0.5), - ]; - let available = if cached_row { &logs[..] } else { &logs[..1] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.agents[0].spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::finite(0.25, Some(0.5))] #[case::overflow(f64::MAX, None)] fn trace_cost_requires_a_finite_total(#[case] cost: f64, #[case] expected: Option) { let rows = [ - owned(llm("first", "", "agent", "response-a"), "team", "", "key"), - owned(llm("second", "", "agent", "response-b"), "team", "", "key"), + llm("first", "", "agent", "response-a"), + llm("second", "", "agent", "response-b"), ]; let logs = [ - spend("request-a", "response-a", "team", "", "key", cost), - spend("request-b", "response-b", "team", "", "key", cost), + spend("request-a", "response-a", cost), + spend("request-b", "response-b", cost), ]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.agents[0].spend, expected); } -#[rstest] -fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![ - "litellm_request:retry".parse().unwrap(), - "litellm_request:final".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("retry", "retry-response", "team", "", "key", 0.25), - spend("final", "response", "team", "", "key", 0.5), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.75)); - assert_eq!(trace.agents[0].spend, Some(0.75)); -} - #[rstest] #[case::legacy(None, Some(0.25))] #[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] #[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] #[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] -fn legacy_request_id_fallback_respects_recorded_evidence( +fn stored_response_id_requires_complete_call_evidence( #[case] evidence: Option, #[case] expected: Option, ) { - let span = owned( - TraceSpansRow { - call_evidence: evidence, - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - ); + let span = TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }; let stored = serde_json::to_value(span).unwrap(); let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); - let logs = [spend("request", "response", "team", "", "key", 0.25)]; + let logs = [spend("request", "response", 0.25)]; let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.spans[0].spend, expected); @@ -1058,28 +878,6 @@ fn malformed_stored_span_fields_are_rejected( assert!(serde_json::from_value::(encoded).is_err()); } -#[rstest] -#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] -#[case::complete(litellm_traces::CallEvidenceKind::Complete)] -fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( - #[case] evidence: litellm_traces::CallEvidenceKind, -) { - let recorded = TraceSpansRow { - trace_id: "trace".to_owned(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".to_owned()), - litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), - litellm_traces::CallKey::Transport, - ], - call_evidence: Some(evidence), - ..row("span", "", "operation", "llm", "") - }; - let lookup = litellm_traces::SpendLookup::new(&[recorded]); - assert_eq!(lookup.response_ids, ["response"]); - assert_eq!(lookup.request_ids, ["request"]); - assert_eq!(lookup.trace_ids, ["trace"]); -} - #[rstest] #[case::parent_first(false)] #[case::child_first(true)] @@ -1150,68 +948,23 @@ fn empty_root_preview_uses_the_earliest_agent_or_model_input() { fn repeated_request_ids_preserve_storage_identity(#[case] reverse: bool) { let first = SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) + ..spend("same", "response", 0.25) }; let second = SpendByResponseIdsRow { start_ms: 200, - ..spend("same", "response", "team", "", "key", 0.5) + ..spend("same", "response", 0.5) }; let logs = if reverse { [second, first] } else { [first, second] }; - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, None); assert_eq!(trace.spans[0].spend, None); } -#[rstest] -#[case::gateway(false)] -#[case::transport(true)] -fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { - let rows = [owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".into()), - if transport { - litellm_traces::CallKey::Transport - } else { - litellm_traces::CallKey::LiteLlmRequest("gateway".into()) - }, - ], - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - litellm_call_id: "gateway".into(), - trace_id: "trace".into(), - span_id: "call".into(), - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "other".into(), - ..spend("same", "response", "team", "", "key", 0.5) - }, - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::same_row(true, Some(0.25))] #[case::distinct_rows(false, Some(0.75))] @@ -1220,27 +973,19 @@ fn totals_deduplicate_only_equal_storage_identities( #[case] expected: Option, ) { let rows = [ - owned(llm("first", "", "agent", "a"), "team", "", "key"), - owned( - llm("second", "", "agent", if duplicate { "a" } else { "b" }), - "team", - "", - "key", - ), + llm("first", "", "agent", "a"), + llm("second", "", "agent", if duplicate { "a" } else { "b" }), ]; let logs = [ SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "a", "team", "", "key", 0.25) + ..spend("same", "a", 0.25) }, SpendByResponseIdsRow { start_ms: if duplicate { 100 } else { 200 }, ..spend( "same", if duplicate { "a" } else { "b" }, - "team", - "", - "key", if duplicate { 0.25 } else { 0.5 }, ) }, @@ -1254,80 +999,6 @@ fn totals_deduplicate_only_equal_storage_identities( ); } -#[rstest] -fn conflicting_keys_cannot_agree_on_request_id_alone() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "gateway".into(), - ..spend("same", "other", "team", "", "key", 0.5) - }, - ]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - None - ); -} - -#[rstest] -#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] -#[case::legacy("", "gateway", "team", "key", Some(0.25))] -#[case::conflict("other", "gateway", "team", "key", None)] -#[case::other_team("gateway", "provider-id", "other-team", "key", None)] -#[case::other_key("gateway", "provider-id", "team", "other-key", None)] -fn gateway_lookup_respects_legacy_fallback_and_ownership( - #[case] call_id: &str, - #[case] request_id: &str, - #[case] team: &str, - #[case] key: &str, - #[case] expected: Option, -) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("call", "", "agent", "") - }, - "team", - "", - "key", - )]; - let logs = [SpendByResponseIdsRow { - litellm_call_id: call_id.into(), - ..spend(request_id, "provider", team, "", key, 0.25) - }]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - expected - ); -} - #[rstest] #[case::matching("call-one", "claude_code.tool.execution", SpanStatus::Error)] #[case::other_tool("other-call", "claude_code.tool.execution", SpanStatus::Ok)] @@ -1389,3 +1060,790 @@ fn native_tool_failure_log_matches_by_call_id_without_double_counting( assert_eq!(trace.spans[0].status, expected); assert_eq!(trace.summary.error_count, 1); } + +fn owned_spend( + request_id: &str, + response_id: &str, + team: &str, + user: &str, + key: &str, + cost: f64, +) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + request_id: request_id.into(), + response_id: response_id.into(), + litellm_call_id: String::new(), + upstream_response_id: String::new(), + provider_request_id: String::new(), + trace_id: String::new(), + span_id: String::new(), + team_id: team.into(), + api_key: key.into(), + user: user.into(), + spend: Some(cost), + start_ms: T0 / MS, + } +} + +fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { + span.team_id = team.into(); + span.user_id = user.into(); + span.api_key_hash = key.into(); + span +} + +#[rstest] +fn repeated_response_counts_once_and_other_owners_are_ignored() { + let rows = [ + owned( + row("root", "", "agent", "agent", "agent"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-1", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-2", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + ]; + let spend = [ + owned_spend("request-other", "response-1", "team-b", "", "key-b", 99.0), + owned_spend("request-1", "response-1", "team-a", "", "key-a", 0.25), + owned_spend( + "request-other-key", + "unrelated-response", + "team-a", + "", + "key-c", + 50.0, + ), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), Some(0.25)] + ); +} + +#[rstest] +#[case::key_differs("team", "", "export", "team", "", "request", false)] +#[case::shared_key("team", "", "export", "team", "", "export", true)] +#[case::shared_user("", "user", "export", "", "user", "request", true)] +#[case::teamless_key("", "", "key", "", "", "key", true)] +#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] +#[case::other_user("", "user", "export", "", "other-user", "request", false)] +#[case::no_shared_identity("", "", "export", "", "", "request", false)] +#[case::no_identity("", "", "", "", "", "", false)] +#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] +fn cost_requires_shared_ownership( + #[case] trace_team: &str, + #[case] trace_user: &str, + #[case] trace_key: &str, + #[case] spend_team: &str, + #[case] spend_user: &str, + #[case] spend_key: &str, + #[case] known: bool, +) { + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + trace_team, + trace_user, + trace_key, + ), + owned( + llm("llm", "agent", "agent", "response"), + trace_team, + trace_user, + trace_key, + ), + ]; + let spend = [owned_spend( + "request", "response", spend_team, spend_user, spend_key, 0.25, + )]; + let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); + let expected = known.then_some(0.25); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[1].spend, expected); +} + +#[rstest] +#[case::missing_id("missing_id")] +#[case::missing_spend("missing_spend")] +#[case::duplicate_spend("duplicate_spend")] +fn partial_trace_total_counts_only_complete_calls(#[case] failure: &str) { + let second_id = if failure == "missing_id" { + "" + } else { + "second" + }; + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "export", + ), + owned( + llm("first", "agent", "agent", "first"), + "team", + "", + "export", + ), + owned( + llm("second", "agent", "agent", second_id), + "team", + "", + "export", + ), + ]; + let first = owned_spend("first", "first", "team", "", "export", 0.25); + let second = owned_spend("second", "second", "team", "", "export", 0.25); + let duplicate = owned_spend("duplicate", "second", "team", "", "export", 0.25); + let spend = if failure == "duplicate_spend" { + vec![first, second, duplicate] + } else { + vec![first] + }; + let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.spans[1].spend, Some(0.25)); + assert_eq!(trace.spans[2].spend, None); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); +} + +#[rstest] +fn transport_spans_complete_a_call_without_its_own_id() { + let mut transport = row("http", "llm", "POST", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec!["transport:".parse().unwrap()]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, Some(0.5)); +} + +#[rstest] +#[case::lone_call(1, Some(0.5))] +#[case::two_calls(2, None)] +fn sibling_transports_belong_to_the_only_model_call_under_their_parent( + #[case] calls: usize, + #[case] expected: Option, +) { + let mut transport = at( + row("http", "step", "gateway.request", "framework", ""), + 2, + 10, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut rows = vec![ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + for index in 0..calls { + let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); + call.call_evidence = None; + rows.push(owned(call, "team", "", "key")); + } + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] +fn sibling_transport_does_not_lose_model_call_spend( + #[case] transport_timing: Option<(i64, u64)>, + #[case] unrelated_spend: bool, + #[case] key: litellm_traces::CallKey, + #[case] expected: Option, +) { + let call = owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "chatcmpl-1".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("chat", "step", "agent", "chatcmpl-1") + }, + "team", + "", + "key", + ); + let base_rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + call, + ]; + let rows: Vec<_> = base_rows + .into_iter() + .chain(transport_timing.map(|(start, duration)| { + let mut transport = at( + row("tool-http", "step", "GET", "framework", ""), + start, + duration, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![key]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + owned(transport, "team", "", "key") + })) + .collect(); + let logs: Vec<_> = std::iter::once(owned_spend( + "chatcmpl-1", + "chatcmpl-1", + "team", + "", + "key", + 0.5, + )) + .chain(unrelated_spend.then(|| SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "tool-http".into(), + ..owned_spend("unrelated", "unrelated", "team", "", "key", 0.75) + })) + .collect(); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::agreeing_ids( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] +#[case::conflicting_response_id( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-b"), + None +)] +#[case::conflicting_gateway_and_response( + litellm_traces::CallKey::Transport, + "call-b", + Some("response-b"), + None +)] +#[case::agreeing_gateway_attempt( + litellm_traces::CallKey::GatewayAttempt, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] +fn gateway_attempt_identifiers_must_match_one_spend_row( + #[case] transport: litellm_traces::CallKey, + #[case] call_id: &str, + #[case] response_id: Option<&str>, + #[case] expected: Option, +) { + let keys = [ + transport, + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ] + .into_iter() + .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) + .collect(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(llm("call", "agent", "agent", ""), "team", "", "key"), + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: keys, + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..row("attempt", "call", "gateway.request", "framework", "") + }, + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "call-a".into(), + trace_id: "trace".into(), + span_id: "attempt".into(), + ..owned_spend("request-a", "response-a", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "call-b".into(), + trace_id: "trace".into(), + span_id: "other-attempt".into(), + ..owned_spend("request-b", "response-b", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[2].spend, expected); +} + +#[rstest] +#[case::legacy_row("", Some(0.5))] +#[case::other_call("other-call", None)] +fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( + #[case] logged_call_id: &str, + #[case] expected: Option, +) { + let mut transport = row("http", "llm", "gateway.request", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec![ + "transport:".parse().unwrap(), + "litellm_request:gateway-call".parse().unwrap(), + ]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + logged.litellm_call_id = logged_call_id.into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); +} + +#[rstest] +#[case::narrows_ambiguity("request-a", Some(0.25))] +#[case::conflicting_exact_request("request-c", None)] +fn complete_wrapper_reconciles_ambiguous_response( + #[case] exact_id: &str, + #[case] expected: Option, +) { + let wrapper = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") + }; + let rows = [ + wrapper, + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("request-a", "response", "team", "", "key", 0.25), + owned_spend("request-b", "response", "team", "", "key", 0.5), + owned_spend("request-c", "other-response", "team", "", "key", 0.75), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] +#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] +#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] +#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] +fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( + #[case] retry_logged: bool, + #[case] retry_evidence: litellm_traces::CallEvidenceKind, + #[case] expected: Option, +) { + let transport = |id: &str| { + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec!["transport:".parse().unwrap()], + call_evidence: Some(if id == "first" { + retry_evidence + } else { + litellm_traces::CallEvidenceKind::Complete + }), + ..row(id, "call", "POST", "framework", "") + }, + "team", + "", + "key", + ) + }; + let rows = [ + owned( + llm("call", "", "agent", "final-response"), + "team", + "", + "key", + ), + transport("first"), + transport("second"), + ]; + let logs = [ + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "first".into(), + ..owned_spend("retry", "retry-response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "second".into(), + ..owned_spend("final", "final-response", "team", "", "key", 0.5) + }, + owned_spend("cached", "final-response", "team", "", "key", 0.0), + ]; + let available = if retry_logged { &logs[..] } else { &logs[1..] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::same_request(false)] +#[case::ambiguous_response(true)] +fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![ + "provider_response:response".parse().unwrap(), + "litellm_request:request".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + owned_spend("request", "response", "team", "", "key", 0.25), + owned_spend("cached", "response", "team", "", "key", 0.5), + ]; + let available = if cached_row { &logs[..] } else { &logs[..1] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![ + "litellm_request:retry".parse().unwrap(), + "litellm_request:final".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("retry", "retry-response", "team", "", "key", 0.25), + owned_spend("final", "response", "team", "", "key", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.75)); + assert_eq!(trace.agents[0].spend, Some(0.75)); +} + +#[rstest] +#[case::legacy(None, Some(0.25))] +#[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] +#[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] +#[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] +fn legacy_request_id_fallback_respects_recorded_evidence( + #[case] evidence: Option, + #[case] expected: Option, +) { + let span = owned( + TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + ); + let stored = serde_json::to_value(span).unwrap(); + let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); + let logs = [owned_spend("request", "response", "team", "", "key", 0.25)]; + let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.spans[0].spend, expected); +} + +#[rstest] +#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] +#[case::complete(litellm_traces::CallEvidenceKind::Complete)] +fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( + #[case] evidence: litellm_traces::CallEvidenceKind, +) { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(evidence), + ..row("span", "", "operation", "llm", "") + }; + let lookup = litellm_traces::SpendLookup::new(&[recorded]); + assert_eq!(lookup.response_ids, ["response"]); + assert_eq!(lookup.request_ids, ["request"]); + assert_eq!(lookup.trace_ids, ["trace"]); +} + +#[rstest] +#[case::gateway(false)] +#[case::transport(true)] +fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { + let rows = [owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".into()), + if transport { + litellm_traces::CallKey::Transport + } else { + litellm_traces::CallKey::LiteLlmRequest("gateway".into()) + }, + ], + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + litellm_call_id: "gateway".into(), + trace_id: "trace".into(), + span_id: "call".into(), + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "other".into(), + ..owned_spend("same", "response", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn conflicting_keys_cannot_agree_on_request_id_alone() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "gateway".into(), + ..owned_spend("same", "other", "team", "", "key", 0.5) + }, + ]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + None + ); +} + +#[rstest] +#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] +#[case::legacy("", "gateway", "team", "key", Some(0.25))] +#[case::conflict("other", "gateway", "team", "key", None)] +#[case::other_team("gateway", "provider-id", "other-team", "key", None)] +#[case::other_key("gateway", "provider-id", "team", "other-key", None)] +fn gateway_lookup_respects_legacy_fallback_and_ownership( + #[case] call_id: &str, + #[case] request_id: &str, + #[case] team: &str, + #[case] key: &str, + #[case] expected: Option, +) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("call", "", "agent", "") + }, + "team", + "", + "key", + )]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..owned_spend(request_id, "provider", team, "", key, 0.25) + }]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + expected + ); +} + +#[rstest] +#[case::native_id("provider_request:req_native", false, true)] +#[case::historical_id("provider_response:req_native", false, true)] +#[case::legacy_message_id("provider_response:msg_native", false, true)] +#[case::transport("", true, true)] +#[case::no_correlation("", false, false)] +fn native_cost_uses_typed_ids_or_the_original_transport( + #[case] encoded: &str, + #[case] traceparent: bool, + #[case] priced: bool, + #[values(false, true)] grouped: bool, +) { + let call = TraceSpansRow { + trace_id: if grouped { + "session-trace" + } else { + "original-trace" + } + .into(), + original_trace_id: if grouped { "original-trace" } else { "" }.into(), + name: "claude_code.llm_request".into(), + framework: "claude-code".into(), + call_keys: encoded.parse().into_iter().collect(), + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..llm("native-call", "", "agent", "") + }; + let log = SpendByResponseIdsRow { + provider_request_id: "req_native".into(), + trace_id: if traceparent { "original-trace" } else { "" }.into(), + span_id: if traceparent { "native-call" } else { "" }.into(), + ..spend("gateway-log", "msg_native", 0.25) + }; + let lookup = litellm_traces::SpendLookup::new(std::slice::from_ref(&call)); + let trace = resolve_trace("trace", "ref", &[call], &[log]).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (priced.then_some(0.25), u64::from(priced)) + ); + assert_eq!(trace.spans[0].spend, priced.then_some(0.25)); + if encoded.is_empty() { + assert_eq!(lookup.trace_ids, ["original-trace"]); + } else if !encoded.ends_with("msg_native") { + assert_eq!(lookup.provider_request_ids, ["req_native"]); + assert!(lookup.response_ids.is_empty()); + } +} + +#[rstest] +fn provider_request_id_cannot_match_a_message_id_of_the_same_value() { + let call = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderRequest( + "req_native".into(), + )], + ..llm("call", "", "agent", "") + }; + let rows = [spend("unrelated", "req_native", 0.25)]; + let trace = resolve_trace("trace", "ref", &[call], &rows).unwrap(); + assert_eq!(trace.summary.spend, None); + assert_eq!(trace.spans[0].spend_match, Some(SpendMatch::NoSpendLog)); +} diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 588f81ec50a..6c9f726056b 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger from litellm.integrations.clickhouse.context import is_lens_analysis from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.tracing.types import SpendLogRecord from litellm.types.utils import StandardLoggingPayload @@ -155,6 +156,11 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[ return SpendLogRecord( request_id=request_id, response_id=strip_cache_hit_suffix(request_id), + provider_request_id=( + get_provider_request_id(kwargs.get("response_headers")) + or get_provider_request_id(hidden_params.get("additional_headers")) + or "" + ), litellm_call_id=payload.get("litellm_call_id") or "", call_type=payload.get("call_type") or "", api_key=metadata.get("user_api_key_hash") or "", diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4fa78640e24..76ea2c25159 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,6 +11,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -495,7 +496,8 @@ def process_response_headers( **processed_headers, **additional_headers, } - return additional_headers + request_id: Final = get_provider_request_id(response_headers) + return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} def preserve_upstream_non_openai_attributes( diff --git a/litellm/litellm_core_utils/llm_response_utils/get_headers.py b/litellm/litellm_core_utils/llm_response_utils/get_headers.py index d04abcb6e7b..b86cc954b0e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_headers.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_headers.py @@ -1,6 +1,20 @@ from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter, ValidationError + +_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id") + + +def get_provider_request_id(headers: object) -> str | None: + try: + values: Final = _HEADERS.validate_python(headers) + except ValidationError: + return None + normalized: Final = {name.lower(): value for name, value in values.items()} + return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None) + def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict: """ @@ -29,7 +43,9 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> if "x-ratelimit-remaining-tokens" in _response_headers: openai_headers["x-ratelimit-remaining-tokens"] = _response_headers["x-ratelimit-remaining-tokens"] llm_provider_headers: Final = _get_llm_provider_headers(_response_headers) - return {**llm_provider_headers, **openai_headers} + request_id: Final = get_provider_request_id(_response_headers) + native_headers: Final = {"request-id": request_id} if request_id is not None else {} + return {**llm_provider_headers, **openai_headers, **native_headers} def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 86c36a7b413..53ea85a2740 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2214,6 +2214,7 @@ class BaseLLMHTTPHandler: # used for logging + cost tracking logging_obj.model_call_details["httpx_response"] = response + logging_obj.model_call_details["response_headers"] = dict(response.headers) initial_response: AsyncIterator | AnthropicMessagesResponse if stream: diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index e011b795000..46af4e236eb 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -21,6 +21,8 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +from pydantic import TypeAdapter + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -337,9 +339,15 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t return hook(kwargs, error, call_type) -def stream_opened(logger: Logging) -> None: +_STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True + logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python( + head.get("additional_headers") or {} + ) def stream_success( diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 127e86e9160..4a09e731228 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -59,6 +59,7 @@ class AgentNode(typing_extensions.TypedDict): tool_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] duration_ms: ReadOnly[float] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] SpanType: TypeAlias = Literal[ @@ -77,6 +78,9 @@ SpanType: TypeAlias = Literal[ ] +SpendMatch: TypeAlias = Literal["matched", "no_call_id", "no_spend_log", "ambiguous", "incomplete_evidence"] + + class TraceScope(typing_extensions.TypedDict): all_teams: ReadOnly[Literal[0, 1]] user_id: ReadOnly[str] @@ -120,6 +124,7 @@ class TraceSummary(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] models: ReadOnly[tuple[str, ...]] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] class Span(typing_extensions.TypedDict): @@ -140,6 +145,8 @@ class Span(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=4294967295)]] litellm_request_id: ReadOnly[str | None] spend: ReadOnly[float | None] + spend_log_request_id: ReadOnly[str | None] + spend_match: ReadOnly[SpendMatch | None | None] class Trace(typing_extensions.TypedDict): diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index e9930dc7bf7..8bd119940c2 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -8,6 +8,7 @@ class SpendLogRecord(TypedDict): request_id: ReadOnly[str] response_id: ReadOnly[str] + provider_request_id: ReadOnly[str] litellm_call_id: ReadOnly[str] call_type: ReadOnly[str] api_key: ReadOnly[str] diff --git a/scripts/trace_codegen/schemas/traces/Trace.json b/scripts/trace_codegen/schemas/traces/Trace.json index dc66f77412c..ebd99880956 100644 --- a/scripts/trace_codegen/schemas/traces/Trace.json +++ b/scripts/trace_codegen/schemas/traces/Trace.json @@ -28,6 +28,12 @@ "null" ] }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "spend": { "format": "double", "type": [ @@ -49,7 +55,8 @@ "llm_calls", "tool_calls", "duration_ms", - "spend" + "spend", + "priced_calls" ], "type": "object" }, @@ -120,6 +127,22 @@ "null" ] }, + "spend_log_request_id": { + "type": [ + "string", + "null" + ] + }, + "spend_match": { + "anyOf": [ + { + "$ref": "#/$defs/SpendMatch" + }, + { + "type": "null" + } + ] + }, "start_offset_ms": { "format": "double", "type": "number" @@ -148,7 +171,9 @@ "input_tokens", "output_tokens", "litellm_request_id", - "spend" + "spend", + "spend_log_request_id", + "spend_match" ], "type": "object" }, @@ -177,6 +202,16 @@ ], "type": "string" }, + "SpendMatch": { + "enum": [ + "matched", + "no_call_id", + "no_spend_log", + "ambiguous", + "incomplete_evidence" + ], + "type": "string" + }, "TraceSummary": { "properties": { "agent_count": { @@ -245,6 +280,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -306,7 +347,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/scripts/trace_codegen/schemas/traces/TracePage.json b/scripts/trace_codegen/schemas/traces/TracePage.json index dfa02ba0d39..429635ec3b1 100644 --- a/scripts/trace_codegen/schemas/traces/TracePage.json +++ b/scripts/trace_codegen/schemas/traces/TracePage.json @@ -76,6 +76,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -137,7 +143,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/tests/llm_translation/test_llm_response_utils/test_get_headers.py b/tests/llm_translation/test_llm_response_utils/test_get_headers.py index 380f89bbdd4..60a6a4bebcd 100644 --- a/tests/llm_translation/test_llm_response_utils/test_get_headers.py +++ b/tests/llm_translation/test_llm_response_utils/test_get_headers.py @@ -8,6 +8,7 @@ import pytest from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, _get_llm_provider_headers, + get_provider_request_id, ) @@ -72,3 +73,17 @@ def test_get_llm_provider_headers(): } result = _get_llm_provider_headers(input_headers) assert result == expected_output, "Unexpected output for _get_llm_provider_headers" + + +@pytest.mark.parametrize("header", ("request-id", "Request-Id", "x-request-id", "llm_provider-request-id")) +def test_native_clients_receive_the_provider_request_id(header: str) -> None: + result = get_response_headers({header: "req_test", "unrelated": "value"}) + assert result["request-id"] == "req_test" + assert get_provider_request_id(result) == "req_test" + assert result["llm_provider-unrelated"] == "value" + assert "unrelated" not in result + + +@pytest.mark.parametrize("headers", (None, {}, {"request-id": ""}, {"request-id": 42})) +def test_invalid_provider_request_ids_remain_absent(headers: object) -> None: + assert get_provider_request_id(headers) is None diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py index 491f8651b52..fe47c1a59f3 100644 --- a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -163,6 +163,7 @@ def test_success_row_mapping(): assert row["request_id"] == "chatcmpl-abc123" assert row["response_id"] == "chatcmpl-abc123" assert row["litellm_call_id"] == "gateway-call" + assert row["provider_request_id"] == "" assert row["spend"] == 0.00042 assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30) assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7) @@ -186,6 +187,18 @@ def test_success_row_mapping(): assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key" +@pytest.mark.parametrize("header_source", ("response_headers", "additional_headers")) +def test_provider_request_id_stays_separate_from_message_and_gateway_ids(header_source: str) -> None: + headers: Final = {"request-id": "req_native"} + hidden: Final = {"additional_headers": headers} if header_source == "additional_headers" else {} + kwargs: Final = {"response_cost": 0.00042, "response_headers": headers if header_source == "response_headers" else None} + payload: Final = cast(StandardLoggingPayload, _payload(id="msg_native", hidden_params=hidden)) + row: Final = spend_log_row_from_payload(payload, kwargs) + assert (row["provider_request_id"], row["response_id"], row["litellm_call_id"]) == ( + "req_native", "msg_native", "gateway-call" + ) + + @pytest.mark.parametrize("status", ("success", "failure")) @pytest.mark.asyncio async def test_custom_request_metadata_is_redacted_before_clickhouse_logging( diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 2a6dd347d5f..9f48ea62a72 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,6 +1,7 @@ """Tests for litellm_core_utils.core_helpers module.""" import logging +from typing import Final import httpx import pytest @@ -16,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, map_finish_reason, normalize_drop_params, + process_response_headers, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -25,6 +27,14 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ImageResponse, TranscriptionResponse +@pytest.mark.parametrize("header", ("request-id", "x-request-id", "llm_provider-request-id")) +def test_native_request_id_survives_stream_header_processing(header: str) -> None: + processed: Final = process_response_headers(httpx.Headers({header: "req_native"})) + + assert processed["request-id"] == "req_native" + assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native" + + class TestBudgetReservationBinding: """The request-end release skips a reservation a cost callback has claimed, so the claim must land on the one dict auth stamped, through whichever metadata field or auth object diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 93d18727f22..2d59ab337d4 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -895,6 +895,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp additional_headers = result._hidden_params["additional_headers"] assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123" assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def" + assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers) collected = b"".join([chunk async for chunk in result]) assert b"message_start" in collected @@ -1601,8 +1602,7 @@ async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeyp expected_response = {"id": "msg_1", "content": []} mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response) - ok_response = Mock() - ok_response.raise_for_status = Mock(return_value=None) + ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) mock_client = AsyncMock(spec=AsyncHTTPHandler) mock_client.post = AsyncMock(return_value=ok_response) @@ -2072,8 +2072,7 @@ async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks( captured_kwargs.update(call_kwargs) return sentinel_response - mock_httpx_response = Mock() - mock_httpx_response.status_code = 200 + mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) with ( patch.object( diff --git a/tests/unit/rust_bridge/messages/test_route_host.py b/tests/unit/rust_bridge/messages/test_route_host.py index 1be42e2249d..7b15553a055 100644 --- a/tests/unit/rust_bridge/messages/test_route_host.py +++ b/tests/unit/rust_bridge/messages/test_route_host.py @@ -161,4 +161,4 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert isinstance(additional, dict) assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" - assert "request-id" not in additional + assert additional["request-id"] == additional["llm_provider-request-id"] diff --git a/tests/unit/rust_bridge/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/test_callbacks_legacy_python.py index 045dcb52ce9..c78fbb3f246 100644 --- a/tests/unit/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/test_callbacks_legacy_python.py @@ -34,6 +34,14 @@ def _supplied_logger() -> Logging: ) +def test_native_stream_headers_reach_spend_callbacks() -> None: + logger: Final = _supplied_logger() + legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}}) + + assert logger.stream is True + assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"} + + def test_setup_reuses_a_supplied_logger() -> None: supplied: Final = _supplied_logger() result: Final = setup( @@ -159,9 +167,7 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation pending.close() -CONTRACT_PATH: Final = ( - Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" -) +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" def test_the_rust_contract_matches_the_shim_signatures() -> None: diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts index 27c46a528e7..f33f86cd280 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts @@ -26,6 +26,8 @@ function makeTrace(scene: Scenario, index: number, now: number) { parent_span_id: null, span_id: spanId(0), spend: null, + spend_log_request_id: null, + spend_match: null, start_offset_ms: 0, status: "ok", type: "agent", @@ -53,6 +55,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { input_tokens: 520 + index * 41, output_tokens: 48 + index * 7, spend: 0.003 + index * 0.0002, + spend_match: "matched", }; const trace: Trace = { summary: { @@ -71,6 +74,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { service: "demo-agents", span_count: toolCount + 2, spend: model.spend, + priced_calls: 1, start_time: iso(now - (index + 1) * 35 * 60_000), status: scene.failed ? "error" : "ok", tool_calls: toolCount, @@ -85,6 +89,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { llm_calls: 1, tool_calls: toolCount, spend: model.spend, + priced_calls: 1, }, ], spans: [base, ...tools, model], diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts index 866f1042b73..44eac42755c 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts @@ -85,6 +85,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0), output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0), spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0), + priced_calls: models.length, }; const finalDetail = run.details.at(-1)!; const history = caseDetails.filter((_, index) => index % 3 === 2); @@ -102,6 +103,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { llm_calls: caseCount, tool_calls: caseCount, spend: summary.spend - (final.spend ?? 0), + priced_calls: caseCount, }, ], }, diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx index 16afa455be6..33fb073d0a9 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx @@ -10,8 +10,8 @@ import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; import type { TraceHandoff } from "../../api"; +import { runCost } from "../../list/AgentTracesTable"; import { traceRefOf, traceShareUrl } from "../../routing"; -import { formatCost } from "../../list/AgentTracesTable"; import { IdChip } from "../../ui/IdChip"; import { SpanIcon } from "../../ui/SpanIcon"; import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework"; @@ -89,6 +89,7 @@ interface RunHeaderProps { export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) { const { summary } = trace; const failed = summary.status === "error"; + const cost = runCost(summary); return (
@@ -127,7 +128,10 @@ export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) - + {summary.error_count > 0 && }
diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx index fb9285d0777..cf2c32ff5c7 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx @@ -13,10 +13,31 @@ import type { Span, SpanDetail, SpanErrorPage, Trace } from "../../types"; vi.mock("../../../../networking", () => ({ agentTraceSpanCall: vi.fn(), agentTraceSpanErrorCall: vi.fn(), + uiSpendLogsCall: vi.fn(), getProxyBaseUrl: () => "http://proxy.test/", })); -import { agentTraceSpanCall, agentTraceSpanErrorCall } from "../../../../networking"; +vi.mock("../../../../logs/detail", () => ({ + LogDetailsDrawer: ({ + open, + logEntry, + onClose, + backTo, + }: { + open: boolean; + logEntry: { request_id: string } | null; + onClose: () => void; + backTo?: { label: string }; + }) => + open && logEntry ? ( +
+ {logEntry.request_id} + {backTo && } +
+ ) : null, +})); + +import { agentTraceSpanCall, agentTraceSpanErrorCall, uiSpendLogsCall } from "../../../../networking"; type SpanFields = Partial & Pick; @@ -37,6 +58,8 @@ const span = (overrides: SpanFields): Span => ({ output_tokens: 0, litellm_request_id: null, spend: null, + spend_log_request_id: null, + spend_match: null, ...overrides, }); @@ -49,6 +72,9 @@ const llmFields: SpanFields = { input_tokens: 659, output_tokens: 60, litellm_request_id: "chatcmpl-abc", + spend: 0.0002, + spend_log_request_id: "req_7f3a9c2e1b44", + spend_match: "matched", }; const failedToolFields: SpanFields = { span_id: "tool1", @@ -244,13 +270,47 @@ describe("DetailPane", () => { expect(input).toHaveTextContent("acme-404"); }); - it("shows the LiteLLM request facts on the Request tab", async () => { + it("links a priced LLM step to its spend log in the header and opens it by that request id", async () => { const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); renderPane(spanRow(llm)); + const link = screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" }); + expect(link).toHaveTextContent("LiteLLM Spend Log$0.0002"); + await user.click(link); + expect(await screen.findByRole("dialog", { name: "Request log" })).toHaveTextContent("req_7f3a9c2e1b44"); + expect(vi.mocked(uiSpendLogsCall).mock.calls[0][0].params).toEqual({ request_id: "req_7f3a9c2e1b44" }); + }); + + it("returns from the spend log to the same agent trace step", async () => { + const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); + renderPane(spanRow(llm)); + await user.click(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })); + const drawer = await screen.findByRole("dialog", { name: "Request log" }); + await user.click(within(drawer).getByRole("button", { name: "Back to Lens trace" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Request log" })).not.toBeInTheDocument()); + expect(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })).toBeVisible(); + }); + + it.each([ + { spend_match: "no_call_id" as const, reason: "This step records no usable call identifier or transport context" }, + { spend_match: "no_spend_log" as const, reason: "No matching spend log belongs to this run" }, + { spend_match: "incomplete_evidence" as const, reason: "This step does not account for every request in the call" }, + ])("says why an unpriced LLM step has no cost ($spend_match)", async ({ spend_match, reason }) => { + const user = userEvent.setup(); + const unpriced: SpanFields = { ...llmFields, spend: null, spend_log_request_id: null, spend_match }; + renderPane(spanRow(span(unpriced))); + expect(screen.getByTitle(reason)).toHaveTextContent("Cost not matched"); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("tab", { name: "Request" })); - expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument(); - expect(screen.getByText("659")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument(); + expect(await screen.findByText(reason, { selector: "p" })).toBeInTheDocument(); + }); + + it("keeps the Cost fact for a priced step that has no single spend log to link", () => { + const pricedFromRetries: SpanFields = { ...llmFields, spend: 0.0005, spend_log_request_id: null }; + renderPane(spanRow(span(pricedFromRetries))); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); + expect(screen.getByText("Cost").parentElement).toHaveTextContent("Cost$0.0005"); }); it("summarizes a ×N group with its failure pattern", () => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx index 4757c55ab80..240ea149199 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx @@ -16,11 +16,12 @@ interface PaneHeaderProps { idValue?: string; facts: readonly (readonly [label: string, value: string])[]; actions?: React.ReactNode; + links?: React.ReactNode; onClose: () => void; } /** Selected step identity: tile, name, id, then its time and usage on one quiet line. */ -export function PaneHeader({ type, model, failed, title, idValue, facts, actions, onClose }: PaneHeaderProps) { +export function PaneHeader({ type, model, failed, title, idValue, facts, actions, links, onClose }: PaneHeaderProps) { return (
@@ -40,15 +41,18 @@ export function PaneHeader({ type, model, failed, title, idValue, facts, actions
- {facts.length > 0 && ( -
- {facts.map(([label, value]) => ( -
-
{label}
-
{value}
-
- ))} -
+ {(facts.length > 0 || links) && ( +
+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ {links} +
)} ); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx index 855a7060336..cd36d2686ef 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx @@ -1,19 +1,14 @@ "use client"; -import { ArrowUpRight } from "lucide-react"; -import { useState } from "react"; - import CopyButton from "@/components/shared/CopyButton"; -import { Button } from "@/components/ui/button"; -import { LogDetailsDrawer } from "../../../../logs/detail"; import { fieldEntries } from "../content/payload"; import { formatCost } from "../../list/AgentTracesTable"; import type { Span } from "../../types"; import { fmtMs, fmtTok } from "../../utils"; -import { useSpanRequestLog } from "../useSpanRequestLog"; import { FieldTree } from "../content/FieldTree"; import { DetailGroup } from "./DetailGroup"; +import { SpendLogLink, unmatchedReason } from "./SpendLogLink"; interface RequestTabProps { span: Span; @@ -22,17 +17,7 @@ interface RequestTabProps { traceStartMs: number; } -/** Usage and the linked LiteLLM request for an LLM span; "Open request log" opens the request drawer over the run. */ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) { - const [drawerOpen, setDrawerOpen] = useState(false); - const logQuery = useSpanRequestLog( - accessToken, - span.litellm_request_id, - traceStartMs + span.start_offset_ms, - drawerOpen, - ); - const logNotFound = drawerOpen && logQuery.isSuccess && logQuery.data === null; - if (span.type !== "llm") { return
This span is not a model request.
; } @@ -51,30 +36,22 @@ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) - - {span.litellm_request_id ? ( -
+ +
+ {span.spend_log_request_id && (
- +
- +
- - {logNotFound &&

No request log found for this call.

} + )} +
+
- ) : ( -

Not linked to a LiteLLM request

- )} + {unmatchedReason(span) &&

{unmatchedReason(span)}

} +
- setDrawerOpen(false)} - logEntry={logQuery.data ?? null} - accessToken={accessToken} - />
); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx index cfa92f67577..d74d7ad8ccd 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx @@ -12,6 +12,7 @@ import { ContentTab, useSpanDetail } from "../content/ContentTab"; import { AttributesTab } from "./AttributesTab"; import { PaneHeader } from "./PaneHeader"; import { RequestTab } from "./RequestTab"; +import { SpendLogLink } from "./SpendLogLink"; const TAB_LABELS: Record = { content: "Content", request: "Request", attributes: "Attributes" }; @@ -26,7 +27,7 @@ const spanFacts = (span: Span): readonly Fact[] => { const tokens = span.input_tokens + span.output_tokens; const optional: readonly (Fact | null)[] = [ tokens > 0 ? ["Tokens", fmtTok(tokens)] : null, - span.spend != null ? ["Cost", formatCost(span.spend)] : null, + span.spend != null && span.spend_log_request_id == null ? ["Cost", formatCost(span.spend)] : null, span.type === "llm" && span.model ? ["Step", span.name] : null, ]; return [["Duration", fmtMs(span.duration_ms)], ...optional.filter((fact): fact is Fact => fact !== null)]; @@ -42,6 +43,7 @@ export function SpanPane({ }: SpanTabProps & { trace: Trace; span: Span; accessToken: string; onClose: () => void }) { const { trace_id: traceId, trace_ref: traceRef, start_time: startTime } = trace.summary; const handoff = useTracesApi(accessToken).handoff(traceId, span.span_id, traceRef); + const traceStartMs = Date.parse(startTime); const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null, traceRef); return (