fix(tracing): preserve native actor identity across background work

This commit is contained in:
moe-berri 2026-10-06 11:54:07 -07:00
parent de984d6ebc
commit bead5162d4
27 changed files with 869 additions and 44 deletions

View file

@ -297,6 +297,10 @@ mod tests {
fn row(span_id: &str) -> TraceSpansRow {
TraceSpansRow {
native_agent_id: String::new(),
native_parent_agent_id: String::new(),
session_id: String::new(),
query_source: String::new(),
trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: String::new(),

View file

@ -259,6 +259,10 @@ fn access() -> ReadAccessParams {
fn span(index: usize) -> TraceSpansRow {
TraceSpansRow {
native_agent_id: String::new(),
native_parent_agent_id: String::new(),
session_id: String::new(),
query_source: String::new(),
trace_id: "trace".into(),
span_id: format!("span-{index}"),
parent_span_id: if index == 0 {

View file

@ -16,6 +16,10 @@ const TTL: Duration = Duration::from_secs(120);
fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
TraceSpansRow {
native_agent_id: String::new(),
native_parent_agent_id: String::new(),
session_id: String::new(),
query_source: String::new(),
trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: parent.into(),

View file

@ -1,4 +1,10 @@
WITH page AS (
WITH
(Framework = 'claude-code' AND (SpanAttributes['session.id'] != '' OR SpanAttributes['agent_id'] != '')) AS native,
(native AND SpanAttributes['agent_id'] != '') AS native_child,
(native AND (SpanName = 'claude_code.interaction' OR
coalesce(nullIf(SpanAttributes['query_source_safe'], ''), SpanAttributes['query_source']) IN
('repl_main_thread', 'sdk', 'sdk_main_thread', 'generate_session_title', 'prompt_suggestion'))) AS native_root,
page AS (
SELECT TraceId AS trace_id,
hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref,
if(length(groupUniqArrayArray(UserIds)) = 1, arrayElement(groupUniqArrayArray(UserIds), 1), '') AS user_id, TeamId AS team_id, ApiKeyHash AS api_key_hash,
@ -27,7 +33,8 @@ HAVING min(StartTs) >= fromUnixTimestamp64Milli({start_ms:Int64})
ORDER BY start_ms DESC, trace_ref DESC
LIMIT {limit:UInt32}
)
SELECT page.* EXCEPT (trace_start, trace_end),
SELECT page.* EXCEPT (trace_start, trace_end, agent_invocations),
if(identities.native_count > 0, identities.resolved_invocations, page.agent_invocations) AS agent_invocations,
identities.agent_names AS agent_names, identities.agent_count AS agent_count,
identities.frameworks AS frameworks
FROM page
@ -35,7 +42,13 @@ LEFT JOIN (
SELECT TeamId, ApiKeyHash, TraceId,
arraySort(groupUniqArrayIf(AgentName, AgentName != '')) AS agent_names,
arraySort(groupUniqArrayIf(toString(Framework), Framework != '')) AS frameworks,
uniqExactIf(if(AgentName = '', SpanName, AgentName), ObservationType = 'agent') AS agent_count
uniqExactIf((SpanAttributes['session.id'], SpanAttributes['agent_id']), native_child)
+ uniqExactIf(SpanAttributes['session.id'], native_root) AS native_count,
native_count + uniqExactIf(if(AgentName = '', SpanName, AgentName), ObservationType = 'agent' AND NOT native) AS agent_count,
uniqExactIf(SpanId, ObservationType = 'agent' AND NOT native)
+ uniqExactIf((SpanAttributes['session.id'], SpanAttributes['agent_id']), native_child)
+ greatest(uniqExactIf(SpanId, native AND SpanName = 'claude_code.interaction'),
uniqExactIf(SpanAttributes['session.id'], native_root)) AS resolved_invocations
FROM otel_traces
WHERE Timestamp >= (SELECT min(trace_start) FROM page)
AND Timestamp <= (SELECT max(trace_end) FROM page)

View file

@ -1,6 +1,10 @@
SELECT * FROM (
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.SpanAttributes['agent_id'] AS native_agent_id,
o.SpanAttributes['parent_agent_id'] AS native_parent_agent_id,
o.SpanAttributes['session.id'] AS session_id,
coalesce(nullIf(o.SpanAttributes['query_source_safe'], ''), o.SpanAttributes['query_source']) AS query_source,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,
lengthUTF8(o.StatusMessage) > 128 AS error_truncated,

View file

@ -1,5 +1,9 @@
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.SpanAttributes['agent_id'] AS native_agent_id,
o.SpanAttributes['parent_agent_id'] AS native_parent_agent_id,
o.SpanAttributes['session.id'] AS session_id,
coalesce(nullIf(o.SpanAttributes['query_source_safe'], ''), o.SpanAttributes['query_source']) AS query_source,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,
lengthUTF8(o.StatusMessage) > 128 AS error_truncated,

View file

@ -1,6 +1,10 @@
SELECT * FROM (
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.SpanAttributes['agent_id'] AS native_agent_id,
o.SpanAttributes['parent_agent_id'] AS native_parent_agent_id,
o.SpanAttributes['session.id'] AS session_id,
coalesce(nullIf(o.SpanAttributes['query_source_safe'], ''), o.SpanAttributes['query_source']) AS query_source,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,
lengthUTF8(o.StatusMessage) > 128 AS error_truncated,

View file

@ -1,5 +1,9 @@
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.SpanAttributes['agent_id'] AS native_agent_id,
o.SpanAttributes['parent_agent_id'] AS native_parent_agent_id,
o.SpanAttributes['session.id'] AS session_id,
coalesce(nullIf(o.SpanAttributes['query_source_safe'], ''), o.SpanAttributes['query_source']) AS query_source,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,
lengthUTF8(o.StatusMessage) > 128 AS error_truncated,

View file

@ -95,6 +95,14 @@ struct TraceSpansRowEncoding {
pub wrapper_candidate: bool,
pub agent: String,
#[serde(default)]
pub native_agent_id: String,
#[serde(default)]
pub native_parent_agent_id: String,
#[serde(default)]
pub session_id: String,
#[serde(default)]
pub query_source: String,
#[serde(default)]
pub framework: String,
#[serde(serialize_with = "litellm_traces::wire::serialize_status")]
pub status: litellm_traces::SpanStatus,
@ -344,7 +352,7 @@ mod tests {
quoted,
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "native_agent_id": "child", "native_parent_agent_id": "parent", "session_id": "session", "query_source": "agent.builtin.general-purpose", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
quoted,
);
round_trip::<SpanDetailRow>(

View file

@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use litellm_http::Client;
use litellm_traces::query::named::ReadAccessParams;
use litellm_traces_cache::{ReadError, TraceReader};
use litellm_traces_cache::{ReadError, TraceReader, TraceStore};
use litellm_traces_clickhouse::{
ClickHouseTraces, Connection, InsertTable, QueryScope, insert_rows,
};
@ -710,3 +710,105 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
}
Ok(())
}
#[rstest]
#[tokio::test]
async fn historical_native_actors_are_resolved_from_raw_attributes_with_tenant_isolation(
#[future(awt)] migrated_database: TestResult<SeededDatabase>,
) -> TestResult {
let fixture = migrated_database?;
let client = &fixture.database.client;
let writer = Connection::writer(&fixture.database.url)?;
let start_ms = 1_790_000_000_000_i64;
let records = [
("root", "", "agent", "", "", "team-a"),
("first", "root", "llm", "first", "", "team-a"),
("second", "root", "llm", "second", "", "team-a"),
("nested", "root", "llm", "nested", "first", "team-a"),
("tool", "root", "tool", "first", "", "team-a"),
("foreign", "root", "llm", "foreign", "", "team-b"),
];
let rows = records.iter().enumerate().map(|(index, (id, parent, kind, actor, parent_actor, team))| {
BTreeMap::from([
("Timestamp".into(), json!((start_ms + index as i64) * 1_000_000)),
("TraceId".into(), json!("native-run")),
("SpanId".into(), json!(id)),
("ParentSpanId".into(), json!(parent)),
("SpanName".into(), json!(if *kind == "agent" { "claude_code.interaction" } else { kind })),
("ObservationType".into(), json!(kind)),
("Framework".into(), json!("claude-code")),
("AgentName".into(), json!("claude-code")),
("SpanAttributes".into(), json!({
"session.id": "historical-session", "agent_id": actor, "parent_agent_id": parent_actor,
"query_source_safe": if actor.is_empty() { "repl_main_thread" } else { "agent.builtin.general-purpose" },
})),
("TeamId".into(), json!(team)),
("ApiKeyHash".into(), json!("key-a")),
("Duration".into(), json!(1_000_000)),
])
}).collect();
insert_rows(client, &writer, DATABASE, InsertTable::OtelTraces, rows).await?;
let connection = fixture
.readers
.connection(client, &QueryScope::All, "fixture-secret")
.await?;
let (reader, store) = make_reader(client, connection);
let access = ReadAccessParams {
all_teams: false,
user_id: String::new(),
team_ids: vec!["team-a".into()],
};
let listed = store
.list_runs(&litellm_traces::query::named::ListTracesParams {
access: access.clone(),
start_ms: 0,
end_ms: 2_000_000_000_000,
cursor_ms: 0,
cursor_trace_id: String::new(),
limit: 50,
})
.await?;
assert_eq!(listed[0].agent_count, 4);
assert_eq!(listed[0].agent_invocations, 4);
let page = reader
.list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
.await?;
assert_eq!(page.data.len(), 1);
assert_eq!(page.data[0].agent_count, 4);
let trace = reader
.get_trace(&store, &access, "native-run", &page.data[0].trace_ref)
.await?
.ok_or("missing trace")?;
assert_eq!(trace.summary.agent_count, 4);
assert_eq!(trace.summary.agent_invocations, 4);
assert_eq!(
trace
.agents
.iter()
.map(|actor| actor.llm_calls)
.sum::<u64>(),
3
);
assert_eq!(
trace
.agents
.iter()
.map(|actor| actor.tool_calls)
.sum::<u64>(),
1
);
assert!(!trace.spans.iter().any(|span| span.span_id == "foreign"));
let first = trace
.spans
.iter()
.find(|span| span.span_id == "first")
.ok_or("missing first actor")?;
let nested = trace
.spans
.iter()
.find(|span| span.span_id == "nested")
.ok_or("missing nested actor")?;
assert!(first.actor_id.is_some());
assert_eq!(nested.parent_actor_id, first.actor_id);
Ok(())
}

View file

@ -45,12 +45,15 @@ fn span_type(name: &str, attributes: &BTreeMap<String, String>) -> SpanType {
}
}
/// `agent:custom:search_agent` -> `search_agent`: the subagent a request ran for.
fn subagent(attributes: &BTreeMap<String, String>) -> Option<&str> {
let mut parts = attr(attributes, "query_source")
.strip_prefix("agent:")?
.splitn(2, ':');
let (_kind, name) = (parts.next()?, parts.next()?);
let source =
super::super::select_attribute(attributes, &["query_source", "query_source_safe"])?;
let (rest, separator) = if let Some(rest) = source.text.strip_prefix("agent:") {
(rest, ':')
} else {
(source.text.strip_prefix("agent.")?, '.')
};
let (_, name) = rest.split_once(separator)?;
(!name.is_empty()).then_some(name)
}
@ -199,7 +202,7 @@ impl Format for ClaudeCode {
let kind = span_type(context.name, attributes);
let base = SpanFacts {
role: Some(RoleEvidence::Declared(ObservationType::Framework)),
agent_name: Some(CLAUDE_CODE_AGENT.to_owned()),
agent_name: Some(subagent(attributes).unwrap_or(CLAUDE_CODE_AGENT).to_owned()),
tool_call_id: present(attributes, &["gen_ai.tool.call.id"]),
..SpanFacts::default()
};
@ -312,6 +315,32 @@ mod tests {
otlp::DecodedEvent,
};
#[rstest]
#[case::log_source("query_source", "agent:builtin:general-purpose")]
#[case::span_source("query_source_safe", "agent.builtin.general-purpose")]
fn native_actor_metadata_does_not_require_session_capture(
#[case] key: &str,
#[case] source: &str,
) {
let attributes = BTreeMap::from([
("span.type".into(), "llm_request".into()),
("session.id".into(), "session".into()),
("agent_id".into(), "child".into()),
("parent_agent_id".into(), "parent".into()),
(key.into(), source.into()),
]);
let span = normalization("claude_code.llm_request", &attributes, &[])
.unwrap()
.span;
assert_eq!(span.agent_name.as_deref(), Some("general-purpose"));
assert_eq!(span.agent_metadata.agent_id.as_deref(), Some("child"));
assert_eq!(
span.agent_metadata.parent_agent_id.as_deref(),
Some("parent")
);
assert_eq!(span.agent_metadata.session_id.as_deref(), Some("session"));
}
fn normalization(
name: &str,
attributes: &BTreeMap<String, String>,

View file

@ -70,6 +70,12 @@ impl From<Integration> for String {
#[derive(Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(default)]
pub struct AgentMetadata {
#[serde(deserialize_with = "optional")]
pub agent_id: Option<String>,
#[serde(deserialize_with = "optional")]
pub parent_agent_id: Option<String>,
#[serde(deserialize_with = "optional")]
pub session_id: Option<String>,
#[serde(deserialize_with = "optional")]
pub lc_agent_name: Option<String>,
#[serde(deserialize_with = "optional")]
@ -109,6 +115,9 @@ pub struct AgentMetadata {
impl AgentMetadata {
pub(crate) fn byte_len(&self) -> usize {
let strings = [
&self.agent_id,
&self.parent_agent_id,
&self.session_id,
&self.lc_agent_name,
&self.ls_agent_purpose,
&self.ls_agent_runtime,
@ -140,6 +149,10 @@ impl AgentMetadata {
#[derive(strum::EnumString, strum::IntoStaticStr)]
#[strum(serialize_all = "snake_case")]
enum MetadataField {
AgentId,
ParentAgentId,
#[strum(serialize = "session.id", to_string = "session_id")]
SessionId,
LcAgentName,
LsIntegration,
LsAgentType,

View file

@ -81,6 +81,14 @@ pub struct TraceSpansRow {
pub wrapper_candidate: bool,
pub agent: String,
#[serde(default)]
pub native_agent_id: String,
#[serde(default)]
pub native_parent_agent_id: String,
#[serde(default)]
pub session_id: String,
#[serde(default)]
pub query_source: String,
#[serde(default)]
pub framework: String,
#[serde(serialize_with = "crate::wire::serialize_status")]
pub status: crate::SpanStatus,

View file

@ -0,0 +1,153 @@
use std::collections::{BTreeMap, BTreeSet, HashMap};
use crate::query::named::TraceSpansRow;
use super::graph::Graph;
pub(super) fn native(row: &TraceSpansRow) -> bool {
row.framework == "claude-code"
&& (!row.session_id.is_empty() || !row.native_agent_id.is_empty())
}
fn identity(row: &TraceSpansRow, agent: &str) -> String {
let session = if row.session_id.is_empty() {
&row.trace_id
} else {
&row.session_id
};
format!("claude-code:{}:{session}:{agent}", session.len())
}
fn root(row: &TraceSpansRow) -> String {
identity(row, "root")
}
fn explicit(row: &TraceSpansRow) -> Option<String> {
if !native(row) {
return None;
}
if !row.native_agent_id.is_empty() {
return Some(identity(row, &format!("agent:{}", row.native_agent_id)));
}
(row.name == "claude_code.interaction"
|| matches!(
row.query_source.as_str(),
"repl_main_thread"
| "sdk"
| "sdk_main_thread"
| "generate_session_title"
| "prompt_suggestion"
))
.then(|| root(row))
}
fn child_source(row: &TraceSpansRow) -> bool {
row.query_source == "agent"
|| row.query_source.starts_with("agent:")
|| row.query_source.starts_with("agent.")
}
pub(super) struct Actor {
pub(super) id: String,
pub(super) parent: Option<String>,
pub(super) name: String,
pub(super) spans: Vec<usize>,
}
pub(super) struct Actors {
owners: Vec<Option<String>>,
pub(super) entries: BTreeMap<String, Actor>,
}
impl Actors {
pub(super) fn new(graph: &Graph<'_>) -> Self {
let rows = graph.rows;
let direct: Vec<_> = rows.iter().map(explicit).collect();
let mut boundaries: HashMap<usize, BTreeSet<&str>> = HashMap::new();
let mut tools: HashMap<&str, BTreeSet<&str>> = HashMap::new();
for (index, owner) in direct.iter().enumerate() {
let Some(owner) = owner.as_deref() else {
continue;
};
let row = &rows[index];
if let Some(parent) = graph.parent(index)
&& rows[parent].name == "claude_code.tool.execution"
{
boundaries.entry(parent).or_default().insert(owner);
}
if !row.tool_call_id.is_empty() {
tools.entry(&row.tool_call_id).or_default().insert(owner);
}
}
let unique = |set: Option<&BTreeSet<&str>>| {
set.filter(|set| set.len() == 1)
.and_then(|set| set.first().copied())
.map(str::to_owned)
};
let owners: Vec<_> = rows
.iter()
.enumerate()
.map(|(index, row)| {
if !native(row) {
return None;
}
direct[index]
.clone()
.or_else(|| unique(boundaries.get(&index)))
.or_else(|| unique(tools.get(row.tool_call_id.as_str())))
.or_else(|| {
graph.ancestors(index).into_iter().find_map(|ancestor| {
let candidate = unique(boundaries.get(&ancestor))
.or_else(|| direct[ancestor].clone())?;
(!child_source(row) || candidate != root(row)).then_some(candidate)
})
})
})
.collect();
let mut entries: BTreeMap<String, Actor> = BTreeMap::new();
for (index, owner) in owners.iter().enumerate() {
let Some(id) = owner else { continue };
let row = &rows[index];
let actor = entries.entry(id.clone()).or_insert_with(|| Actor {
id: id.clone(),
parent: (id != &root(row)).then(|| root(row)),
name: if id == &root(row) {
row.agent.clone()
} else {
"Subagent".to_owned()
},
spans: Vec::new(),
});
actor.spans.push(index);
if !row.native_parent_agent_id.is_empty() {
actor.parent = Some(identity(
row,
&format!("agent:{}", row.native_parent_agent_id),
));
}
if child_source(row) {
let name = row
.query_source
.strip_prefix("agent:")
.and_then(|source| source.split_once(':'))
.or_else(|| {
row.query_source
.strip_prefix("agent.")
.and_then(|source| source.split_once('.'))
});
if let Some((_, name)) = name
&& !name.is_empty()
{
actor.name = name.to_owned();
}
}
}
Self { owners, entries }
}
pub(super) fn owner(&self, index: usize) -> Option<&Actor> {
self.owners[index]
.as_ref()
.and_then(|id| self.entries.get(id))
}
}

View file

@ -1,3 +1,4 @@
mod actors;
mod graph;
mod resolution;
mod spend;

View file

@ -8,6 +8,7 @@ use crate::{
};
use super::{
actors::Actors,
graph::Graph,
spend::{self, Ownership, Requests, SpendEvidence},
};
@ -22,6 +23,7 @@ pub(super) fn agent_label(row: &TraceSpansRow) -> &str {
pub(super) struct Resolution<'a> {
pub(super) graph: Graph<'a>,
pub(super) actors: Actors,
ownership: Ownership<'a>,
spend: &'a [SpendRow],
types: HashMap<&'a str, ObservationType>,
@ -46,6 +48,7 @@ impl<'a> Resolution<'a> {
})
.collect();
Self {
actors: Actors::new(&graph),
ownership: Ownership {
team_id: &rows[0].team_id,
api_key_hash: &rows[0].api_key_hash,

View file

@ -29,7 +29,15 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span
parent_span_id: optional(&row.parent_span_id),
name: row.name.clone(),
kind: resolution.kind(index),
agent: row.agent.clone(),
actor_id: resolution.actors.owner(index).map(|actor| actor.id.clone()),
parent_actor_id: resolution
.actors
.owner(index)
.and_then(|actor| actor.parent.clone()),
agent: resolution
.actors
.owner(index)
.map_or_else(|| row.agent.clone(), |actor| actor.name.clone()),
framework: row.framework.clone(),
start_offset_ms: (i128::from(row.start_ns) - i128::from(trace_start_ns)) as f64
/ NANOS_PER_MS,
@ -51,7 +59,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span
fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
let graph = &resolution.graph;
let mut entries: IndexMap<&str, Vec<usize>> = IndexMap::new();
for index in (0..graph.rows.len()).filter(|index| resolution.is_agent(*index)) {
for index in (0..graph.rows.len()).filter(|index| {
resolution.is_agent(*index) && !super::actors::native(resolution.row(*index))
}) {
entries
.entry(agent_label(resolution.row(index)))
.or_default()
@ -62,7 +72,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
let parent_agent = graph
.parent(index)
.map(|parent| graph.rows[parent].agent.as_str());
if !row.agent.is_empty()
if !super::actors::native(row)
&& !row.agent.is_empty()
&& !explicit.contains(row.agent.as_str())
&& parent_agent != Some(row.agent.as_str())
{
@ -72,38 +83,99 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
let calls: Vec<(&str, Option<Requests<'_>>)> = resolution
.model_calls
.iter()
.filter(|call| !super::actors::native(resolution.row(**call)))
.map(|call| (resolution.owner(*call), resolution.call_requests(*call)))
.collect();
let tools = resolution.unique_tools();
entries
.into_iter()
.map(|(name, spans)| {
let parent_agent = graph.ancestors(spans[0]).into_iter().find_map(|ancestor| {
let label = agent_label(resolution.row(ancestor));
(resolution.is_agent(ancestor) && label != name).then(|| label.to_owned())
});
let owned_calls: Vec<Option<Requests<'_>>> = calls
let legacy = entries.into_iter().map(|(name, spans)| {
let parent_agent = graph.ancestors(spans[0]).into_iter().find_map(|ancestor| {
let label = agent_label(resolution.row(ancestor));
(resolution.is_agent(ancestor) && label != name).then(|| label.to_owned())
});
let owned_calls: Vec<Option<Requests<'_>>> = calls
.iter()
.filter(|(owner, _)| *owner == name)
.map(|(_, requests)| requests.clone())
.collect();
AgentNode {
actor_id: None,
parent_actor_id: None,
name: name.to_owned(),
parent_agent,
invocations: spans.len() as u64,
llm_calls: owned_calls.len() as u64,
tool_calls: tools
.iter()
.filter(|(owner, _)| *owner == name)
.map(|(_, requests)| requests.clone())
.filter(|tool| {
!super::actors::native(resolution.row(**tool))
&& resolution.owner(**tool) == name
})
.count() as u64,
duration_ms: spans
.iter()
.map(|span| graph.rows[*span].duration_ns)
.sum::<u64>() as f64
/ NANOS_PER_MS,
spend: total(&owned_calls),
}
});
legacy
.chain(resolution.actors.entries.values().map(|actor| {
let calls: Vec<_> = resolution
.model_calls
.iter()
.filter(|index| {
resolution
.actors
.owner(**index)
.is_some_and(|owner| owner.id == actor.id)
})
.map(|index| resolution.call_requests(*index))
.collect();
let start = actor
.spans
.iter()
.map(|index| i128::from(graph.rows[*index].start_ns))
.min()
.unwrap_or_default();
let end = actor
.spans
.iter()
.map(|index| {
i128::from(graph.rows[*index].start_ns)
+ i128::from(graph.rows[*index].duration_ns)
})
.max()
.unwrap_or(start);
AgentNode {
name: name.to_owned(),
parent_agent,
invocations: spans.len() as u64,
llm_calls: owned_calls.len() as u64,
actor_id: Some(actor.id.clone()),
parent_actor_id: actor.parent.clone(),
name: actor.name.clone(),
parent_agent: actor
.parent
.as_ref()
.and_then(|id| resolution.actors.entries.get(id))
.map(|parent| parent.name.clone()),
invocations: actor
.spans
.iter()
.filter(|index| graph.rows[**index].name == "claude_code.interaction")
.count()
.max(1) as u64,
llm_calls: calls.len() as u64,
tool_calls: tools
.iter()
.filter(|tool| resolution.owner(**tool) == name)
.filter(|index| {
resolution
.actors
.owner(**index)
.is_some_and(|owner| owner.id == actor.id)
})
.count() as u64,
duration_ms: spans
.iter()
.map(|span| graph.rows[*span].duration_ns)
.sum::<u64>() as f64
/ NANOS_PER_MS,
spend: total(&owned_calls),
duration_ms: (end - start) as f64 / NANOS_PER_MS,
spend: total(&calls),
}
})
}))
.collect()
}

View file

@ -25,6 +25,10 @@ pub struct Span {
#[serde(rename = "type")]
pub kind: crate::ObservationType,
pub agent: String,
#[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))]
pub actor_id: Option<String>,
#[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))]
pub parent_actor_id: Option<String>,
pub framework: String,
pub start_offset_ms: f64,
pub duration_ms: f64,
@ -43,6 +47,10 @@ pub struct Span {
#[macro_rules_attribute::apply(response_type)]
#[derive(Clone, Debug, PartialEq)]
pub struct AgentNode {
#[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))]
pub actor_id: Option<String>,
#[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))]
pub parent_actor_id: Option<String>,
pub name: String,
pub parent_agent: Option<String>,
pub invocations: u64,

View file

@ -176,6 +176,23 @@ fn trace_span(span: DecodedSpan) -> TraceSpansRow {
})
.unwrap_or_default();
TraceSpansRow {
native_agent_id: span.attributes.get("agent_id").cloned().unwrap_or_default(),
native_parent_agent_id: span
.attributes
.get("parent_agent_id")
.cloned()
.unwrap_or_default(),
session_id: span
.attributes
.get("session.id")
.cloned()
.unwrap_or_default(),
query_source: span
.attributes
.get("query_source_safe")
.or_else(|| span.attributes.get("query_source"))
.cloned()
.unwrap_or_default(),
trace_id: span.trace_id,
span_id: span.span_id,
parent_span_id: span.parent_span_id,
@ -266,6 +283,10 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow {
i64::try_from(i128::from(call.start_ns) + i128::from(call.duration_ns) + 1_000_000)
.expect("valid unrelated transport timestamp");
TraceSpansRow {
native_agent_id: String::new(),
native_parent_agent_id: String::new(),
session_id: String::new(),
query_source: String::new(),
trace_id: call.trace_id.clone(),
span_id: format!("unrelated-transport-{}", call.span_id),
parent_span_id: call.parent_span_id.clone(),

View file

@ -53,7 +53,11 @@ fn result_contracts_preserve_public_field_names() {
json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent",
"native_agent_id": "child",
"native_parent_agent_id": "parent",
"session_id": "session",
"query_source": "agent.builtin.general-purpose", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
);
round_trip::<SpanDetailRow>(
json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}),

View file

@ -10,6 +10,10 @@ const MS: i64 = 1_000_000;
fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
TraceSpansRow {
native_agent_id: String::new(),
native_parent_agent_id: String::new(),
session_id: String::new(),
query_source: String::new(),
trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: parent.into(),
@ -214,6 +218,8 @@ fn repeated_subagent_invocations_aggregate_into_one_node() {
assert_eq!(
trace.agents[0],
AgentNode {
actor_id: None,
parent_actor_id: None,
name: "deep_research_agent".into(),
parent_agent: None,
invocations: 1,
@ -1389,3 +1395,282 @@ fn native_tool_failure_log_matches_by_call_id_without_double_counting(
assert_eq!(trace.spans[0].status, expected);
assert_eq!(trace.summary.error_count, 1);
}
#[rstest]
#[case::root_recorded(true)]
#[case::root_pending(false)]
fn native_actor_identity_survives_same_names_nested_and_resumed_work(#[case] root_recorded: bool) {
let native = |mut row: TraceSpansRow, actor: &str, parent: &str, source: &str| {
row.framework = "claude-code".into();
row.session_id = "session-one".into();
row.native_agent_id = actor.into();
row.native_parent_agent_id = parent.into();
row.query_source = source.into();
row
};
let mut rows = vec![
native(
at(
row(
"root-call",
"root",
"claude_code.llm_request",
"llm",
"claude-code",
),
0,
10,
),
"",
"",
"repl_main_thread",
),
native(
at(
row(
"first-call",
"launch-execution",
"claude_code.llm_request",
"llm",
"claude-code",
),
10,
10,
),
"first",
"",
"agent.builtin.general-purpose",
),
native(
at(
row(
"second-call",
"root",
"claude_code.llm_request",
"llm",
"claude-code",
),
20,
10,
),
"second",
"",
"agent.builtin.general-purpose",
),
native(
at(
row(
"nested-call",
"root",
"claude_code.llm_request",
"llm",
"claude-code",
),
30,
10,
),
"nested",
"first",
"agent.builtin.general-purpose",
),
native(
at(
row(
"resumed-call",
"new-root",
"claude_code.llm_request",
"llm",
"claude-code",
),
100,
10,
),
"first",
"",
"agent.builtin.general-purpose",
),
native(
at(row("tool", "root", "Bash", "tool", "claude-code"), 105, 2),
"first",
"",
"",
),
native(
at(
row(
"launch-execution",
"root",
"claude_code.tool.execution",
"framework",
"claude-code",
),
9,
1,
),
"",
"",
"",
),
native(
at(
row(
"reply",
"launch-execution",
"claude_code.assistant_response",
"chain",
"claude-code",
),
21,
0,
),
"",
"",
"agent:builtin:general-purpose",
),
native(
at(
row(
"unassigned",
"new-root",
"claude_code.assistant_response",
"chain",
"claude-code",
),
115,
0,
),
"",
"",
"agent:builtin:general-purpose",
),
];
if root_recorded {
rows.push(native(
at(
row(
"root",
"",
"claude_code.interaction",
"agent",
"claude-code",
),
0,
90,
),
"",
"",
"",
));
rows.push(native(
at(
row(
"new-root",
"",
"claude_code.interaction",
"agent",
"claude-code",
),
95,
25,
),
"",
"",
"",
));
}
let trace = resolve_trace("trace", "", &rows, &[]).unwrap();
assert_eq!(trace.summary.agent_count, 4);
assert_eq!(
trace
.agents
.iter()
.map(|actor| actor.llm_calls)
.sum::<u64>(),
trace.summary.llm_calls
);
assert_eq!(
trace
.agents
.iter()
.map(|actor| actor.tool_calls)
.sum::<u64>(),
trace.summary.tool_calls
);
let first_id = trace
.spans
.iter()
.find(|span| span.span_id == "first-call")
.unwrap()
.actor_id
.as_ref()
.unwrap();
let first = trace
.agents
.iter()
.find(|actor| actor.actor_id.as_ref() == Some(first_id))
.unwrap();
assert_eq!(first.name, "general-purpose");
assert_eq!(
(
first.invocations,
first.llm_calls,
first.tool_calls,
first.duration_ms
),
(1, 2, 1, 101.0)
);
for id in ["resumed-call", "tool", "reply"] {
assert_eq!(
trace
.spans
.iter()
.find(|span| span.span_id == id)
.unwrap()
.actor_id
.as_ref(),
Some(first_id)
);
}
let second = trace
.spans
.iter()
.find(|span| span.span_id == "second-call")
.unwrap();
assert_ne!(second.actor_id.as_ref(), Some(first_id));
let nested = trace
.spans
.iter()
.find(|span| span.span_id == "nested-call")
.unwrap();
assert_eq!(nested.parent_actor_id.as_ref(), Some(first_id));
assert!(
trace
.spans
.iter()
.find(|span| span.span_id == "unassigned")
.unwrap()
.actor_id
.is_none()
);
assert_eq!(
trace.summary.agent_invocations,
if root_recorded { 5 } else { 4 }
);
}
#[rstest]
fn native_actor_identifiers_are_scoped_to_their_session() {
let rows = ["session-one", "session-two"].map(|session| TraceSpansRow {
session_id: session.into(),
native_agent_id: "shared-child-id".into(),
framework: "claude-code".into(),
..row(session, "", "claude_code.llm_request", "llm", "claude-code")
});
let trace = resolve_trace("trace", "", &rows, &[]).unwrap();
assert_eq!(trace.summary.agent_count, 2);
assert_ne!(trace.spans[0].actor_id, trace.spans[1].actor_id);
assert_ne!(
trace.spans[0].parent_actor_id,
trace.spans[1].parent_actor_id
);
}

View file

@ -52,6 +52,8 @@ SpanStatus: TypeAlias = Literal["ok", "error", "unset"]
class AgentNode(typing_extensions.TypedDict):
actor_id: ReadOnly[NotRequired[str | None]]
parent_actor_id: ReadOnly[NotRequired[str | None]]
name: ReadOnly[str]
parent_agent: ReadOnly[str | None]
invocations: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]]
@ -128,6 +130,8 @@ class Span(typing_extensions.TypedDict):
name: ReadOnly[str]
type: ReadOnly[SpanType]
agent: ReadOnly[str]
actor_id: ReadOnly[NotRequired[str | None]]
parent_actor_id: ReadOnly[NotRequired[str | None]]
framework: ReadOnly[str]
start_offset_ms: ReadOnly[float]
duration_ms: ReadOnly[float]

View file

@ -3,6 +3,13 @@
"AgentNode": {
"description": "One distinct agent in a trace: 200 invocations of `researcher` are one node.",
"properties": {
"actor_id": {
"type": [
"string",
"null"
],
"x-python-optional": true
},
"duration_ms": {
"format": "double",
"type": "number"
@ -22,6 +29,13 @@
"name": {
"type": "string"
},
"parent_actor_id": {
"type": [
"string",
"null"
],
"x-python-optional": true
},
"parent_agent": {
"type": [
"string",
@ -43,6 +57,8 @@
}
},
"required": [
"actor_id",
"parent_actor_id",
"name",
"parent_agent",
"invocations",
@ -55,6 +71,13 @@
},
"Span": {
"properties": {
"actor_id": {
"type": [
"string",
"null"
],
"x-python-optional": true
},
"agent": {
"type": "string"
},
@ -104,6 +127,13 @@
"minimum": 0,
"type": "integer"
},
"parent_actor_id": {
"type": [
"string",
"null"
],
"x-python-optional": true
},
"parent_span_id": {
"type": [
"string",
@ -137,6 +167,8 @@
"name",
"type",
"agent",
"actor_id",
"parent_actor_id",
"framework",
"start_offset_ms",
"duration_ms",

View file

@ -615,3 +615,26 @@ describe("coding sessions", () => {
expect(conversationWarnings(details, true)).toEqual([]);
});
});
it("keeps explicit native actors separate across workflow branches and background resumes", () => {
const span = (id: string, actor: string, parentActor: string | null, offset: number): Span => ({
...root, span_id: id, name: "claude_code.llm_request", type: "llm", framework: "claude-code",
parent_span_id: "root", actor_id: actor, parent_actor_id: parentActor, agent: "general-purpose", start_offset_ms: offset,
});
const spans = [
{ ...root, framework: "claude-code", actor_id: "root-actor", parent_actor_id: null, agent: "claude-code" },
span("first", "actor-one", "root-actor", 1),
span("second", "actor-two", "root-actor", 2),
span("nested", "actor-three", "actor-one", 3),
{ ...span("resumed", "actor-one", "root-actor", 4), parent_span_id: null },
];
const details = new Map(spans.map((span) => [span.span_id, detail(span.span_id, "", `Reply ${span.span_id}`)]));
const items = buildConversation(spans, details, true);
expect(items.filter((item) => item.span.span_id === "resumed").map((item) => item.agentId)).toEqual(["actor-one"]);
const groups = groupConversation(items, spans);
const branches = groups.filter((group) => group.kind === "branch");
expect(branches.map((group) => group.id)).toEqual(["actor-one", "actor-two"]);
expect(branches[0].children.map((group) => group.kind === "branch" ? group.id : group.item.span.span_id)).toEqual(["first", "actor-three", "resumed"]);
expect(items.filter((item) => ["first", "resumed"].includes(item.span.span_id)).map((item) => item.agentName)).toEqual(["general-purpose (1)", "general-purpose (1)"]);
});

View file

@ -209,6 +209,8 @@ function isNativeAgent(span: Span): boolean {
}
function conversationBranch(span: Span, byId: ReadonlyMap<string, Span>): string {
if (span.actor_id) return span.actor_id;
if (span.framework === "claude-code" && span.actor_id === null) return `unassigned:${span.span_id}`;
if (span.type === "agent" || span.parent_span_id === null) return span.span_id;
let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined;
const visited = new Set<string>();
@ -221,7 +223,7 @@ function conversationBranch(span: Span, byId: ReadonlyMap<string, Span>): string
}
function agentIdentity(span: Span, details: ReadonlyMap<string, SpanDetail>): string {
return isNativeAgent(span) ? span.span_id : details.get(span.span_id)?.attributes["gen_ai.agent.id"] || span.span_id;
return span.actor_id || (isNativeAgent(span) ? span.span_id : details.get(span.span_id)?.attributes["gen_ai.agent.id"] || span.span_id);
}
function agentLabels(agents: readonly Span[], details: ReadonlyMap<string, SpanDetail>): ReadonlyMap<string, string> {
@ -234,7 +236,7 @@ function agentLabels(agents: readonly Span[], details: ReadonlyMap<string, SpanD
const args = detail && isNativeAgent(agent) ? toolInput(detail.input, detail.input_ui) : undefined;
const description = args && typeof args === "object" && "description" in args ? args.description : undefined;
const name =
agent.framework === "claude-code" && !isNativeAgent(agent)
agent.framework === "claude-code" && (agent.actor_id || !isNativeAgent(agent))
? agent.agent || agent.name || "Agent"
: agent.name || agent.agent || "Agent";
return { agent, name: typeof description === "string" && description ? description : name };
@ -325,22 +327,28 @@ export function buildConversation(
};
if (combined.length || item.showError) items.push(item);
}
const byBranch = new Map([...spans].sort((a, b) => b.start_offset_ms - a.start_offset_ms).map((span) => [branch(span), span]));
const agents = [...new Set(conversationSteps(spans).map(branch))].flatMap((id) => {
const span = byId.get(id);
const span = byId.get(id) ?? byBranch.get(id);
return span ? [span] : [];
});
const labels = agentLabels(agents, details);
const actor = (id: string): string => {
const span = byId.get(id);
const span = byId.get(id) ?? byBranch.get(id);
if (span?.framework === "claude-code" && span.actor_id === null) return id;
return span ? agentIdentity(span, details) : id;
};
return items
.map((item) => ({
...item,
agentId: actor(branch(item.span)),
agentName: labels.get(actor(branch(item.span))) || item.span.agent,
agentName: item.span.framework === "claude-code" && item.span.actor_id === null
? "Unassigned"
: labels.get(actor(branch(item.span))) || item.span.agent,
branchId: branch(item.span),
parentBranchId: (() => {
if (item.span.actor_id) return item.span.parent_actor_id || undefined;
if (item.span.framework === "claude-code" && item.span.actor_id === null) return undefined;
const parentId = byId.get(branch(item.span))?.parent_span_id;
const parent = parentId ? byId.get(parentId) : undefined;
return parent ? branch(parent) : undefined;
@ -358,6 +366,8 @@ export function groupConversation(items: readonly ConversationItem[], spans: rea
const byId = new Map(spans.map((span) => [span.span_id, span]));
const parentById = new Map(
conversationSteps(spans).flatMap((span) => {
if (span.actor_id) return [[span.actor_id, span.parent_actor_id || undefined] as const];
if (span.framework === "claude-code" && span.actor_id === null) return [[conversationBranch(span, byId), undefined] as const];
const branch = byId.get(conversationBranch(span, byId));
if (!branch) return [];
const parent = branch.parent_span_id ? byId.get(branch.parent_span_id) : undefined;

View file

@ -113,7 +113,7 @@ export function errorSource(span: Span): ErrorSource | null {
type GroupOrSpan = Span | { group: Span[] };
const groupKey = (span: Pick<Span, "name" | "type" | "agent">): string => `${span.agent}|${span.name}|${span.type}`;
const groupKey = (span: Pick<Span, "name" | "type" | "agent" | "actor_id">): string => `${span.actor_id ?? span.agent}|${span.name}|${span.type}`;
/** Siblings sharing agent + name + type fold into one group once there are enough of them (or enough failures). */
function groupChildren(children: readonly Span[]): GroupOrSpan[] {
@ -142,7 +142,7 @@ function groupChildren(children: readonly Span[]): GroupOrSpan[] {
return out;
}
export const groupRowId = (parentKey: string, span: Pick<Span, "name" | "type" | "agent">): string =>
export const groupRowId = (parentKey: string, span: Pick<Span, "name" | "type" | "agent" | "actor_id">): string =>
`grp::${parentKey}::${groupKey(span)}`;
interface RowContext {

View file

@ -25792,6 +25792,8 @@ export interface components {
};
/** AgentNode */
AgentNode: {
/** Actor Id */
actor_id?: string | null;
/** Duration Ms */
duration_ms: number;
/** Invocations */
@ -25800,6 +25802,8 @@ export interface components {
llm_calls: number;
/** Name */
name: string;
/** Parent Actor Id */
parent_actor_id?: string | null;
/** Parent Agent */
parent_agent: string | null;
/** Spend */
@ -45075,6 +45079,8 @@ export interface components {
};
/** Span */
Span: {
/** Actor Id */
actor_id?: string | null;
/** Agent */
agent: string;
/** Duration Ms */
@ -45097,6 +45103,8 @@ export interface components {
name: string;
/** Output Tokens */
output_tokens: number;
/** Parent Actor Id */
parent_actor_id?: string | null;
/** Parent Span Id */
parent_span_id: string | null;
/** Span Id */