mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
feat(tracing): support claude agent sdk traces with agent name, logo and chat content (#44248)
* feat(traces): add Framework column to otel_traces * feat(traces): pass span events to normalizers and add framework field * feat(traces): add Claude Code and Agent SDK span normalizer * feat(traces): decode events before normalizing and apply tool span names * feat(traces): list distinct frameworks per trace * feat(traces): return span framework in trace spans query * test(traces): add scrubbed Claude Agent SDK OTLP fixtures * test(traces): cover Claude Agent SDK normalization from real exports * test(traces): assert trace list frameworks stay scoped per trace * feat(tracing): validate framework in native normalized spans * feat(tracing): add framework to Span and frameworks to TraceSummary * feat(tracing): store normalized framework on span rows * feat(tracing): surface span framework and trace frameworks * test(tracing): cover framework aggregation in trace summaries * test(tracing): decode Claude Agent SDK rows with framework and tool args * chore(ui): regenerate API types for trace frameworks * feat(ui): add trace framework registry for Claude Agent SDK and Claude Code * feat(ui): show SDK logo and label in the runs list Agent column * feat(ui): show SDK logo and label in the run header * test(ui): cover SDK label and logo in the runs list * test(ui): cover SDK label and logo in the run header * feat(tracing): show the agent's final answer as claude agent span output * feat(tracing): name claude code agents after their otel service * test(tracing): cover claude code agent naming from the service * fix(tracing): mark the span row framework field read-only * test(tracing): scrub host os details from the claude sdk fixture * test(tracing): scrub host os details from the detailed claude sdk fixture * fix(ui): hide the decorative sdk logo from screen readers * feat(ui): show the agent name with the sdk logo in the runs list * feat(ui): show the agent name with the sdk logo in the run header * test(ui): cover agent names beside the sdk logo in the runs list * test(ui): cover the agent name in the run header
This commit is contained in:
parent
626357549f
commit
481a403090
26 changed files with 3764 additions and 47 deletions
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE {database}.otel_traces ADD COLUMN IF NOT EXISTS Framework LowCardinality(String) AFTER AgentName
|
||||
|
|
@ -25,11 +25,13 @@ ORDER BY start_ms DESC, trace_ref DESC
|
|||
LIMIT {limit:UInt32}
|
||||
)
|
||||
SELECT page.* EXCEPT (trace_start, trace_end),
|
||||
identities.agent_names AS agent_names, identities.agent_count AS agent_count
|
||||
identities.agent_names AS agent_names, identities.agent_count AS agent_count,
|
||||
identities.frameworks AS frameworks
|
||||
FROM page
|
||||
LEFT JOIN (
|
||||
SELECT TeamId, ApiKeyHash, TraceId,
|
||||
arraySort(groupUniqArrayIf(AgentName, AgentName != '')) AS agent_names,
|
||||
arraySort(groupUniqArrayIf(toString(Framework), Framework != '')) AS frameworks,
|
||||
uniqExactIf(if(AgentName = '', SpanName, AgentName), ObservationType = 'agent') AS agent_count
|
||||
FROM otel_traces
|
||||
WHERE Timestamp >= (SELECT min(trace_start) FROM page)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,19 @@
|
|||
SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes
|
||||
FROM otel_traces
|
||||
WHERE TraceId = {trace_id:String} AND SpanId = {span_id:String}
|
||||
AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})
|
||||
AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})
|
||||
SELECT o.SpanId AS span_id, o.Input AS input,
|
||||
if(o.Output = '' AND o.ObservationType = 'agent', answer.output, o.Output) AS output,
|
||||
o.SpanAttributes AS attributes
|
||||
FROM otel_traces AS o
|
||||
LEFT JOIN (
|
||||
SELECT ParentSpanId AS parent_span_id, argMax(Output, Timestamp) AS output
|
||||
FROM otel_traces
|
||||
WHERE TraceId = {trace_id:String} AND ParentSpanId = {span_id:String}
|
||||
AND ObservationType = 'llm' AND Output != ''
|
||||
AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})
|
||||
AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})
|
||||
GROUP BY ParentSpanId
|
||||
) AS answer ON answer.parent_span_id = o.SpanId
|
||||
WHERE o.TraceId = {trace_id:String} AND o.SpanId = {span_id:String}
|
||||
AND (empty({team_ids:Array(String)}) OR o.TeamId IN {team_ids:Array(String)})
|
||||
AND ({api_key_hash:String} = '' OR o.ApiKeyHash = {api_key_hash:String})
|
||||
AND ({trace_ref:String} = '' OR
|
||||
hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) = {trace_ref:String})
|
||||
hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) = {trace_ref:String})
|
||||
LIMIT 1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
|
||||
o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status,
|
||||
o.ObservationType AS type, o.AgentName AS agent,
|
||||
o.Framework AS framework, o.StatusCode AS status,
|
||||
substringUTF8(o.StatusMessage, 1, 128) AS status_message,
|
||||
lengthUTF8(o.StatusMessage) > 128 AS error_truncated,
|
||||
toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns,
|
||||
|
|
|
|||
367
litellm-rust/crates/traces/src/normalize/claude_code.rs
Normal file
367
litellm-rust/crates/traces/src/normalize/claude_code.rs
Normal file
|
|
@ -0,0 +1,367 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use super::{NormalizedSpan, ObservationType, SpanNormalizer, attr, first, tokens};
|
||||
use crate::{DecodeError, otlp::DecodedEvent};
|
||||
|
||||
pub(crate) const CLAUDE_CODE_SCOPE: &str = "com.anthropic.claude_code.tracing";
|
||||
pub(crate) const CLAUDE_CODE_AGENT: &str = "claude-code";
|
||||
const AGENT_SDK_FRAMEWORK: &str = "claude-agent-sdk";
|
||||
|
||||
pub(super) struct ClaudeCodeNormalizer;
|
||||
|
||||
enum SpanType {
|
||||
Interaction,
|
||||
LlmRequest,
|
||||
Tool,
|
||||
Other,
|
||||
}
|
||||
|
||||
fn span_type(name: &str, attributes: &BTreeMap<String, String>) -> SpanType {
|
||||
let kind = attr(attributes, "span.type");
|
||||
let kind = if kind.is_empty() {
|
||||
name.strip_prefix("claude_code.").unwrap_or(name)
|
||||
} else {
|
||||
kind
|
||||
};
|
||||
match kind {
|
||||
"interaction" => SpanType::Interaction,
|
||||
"llm_request" => SpanType::LlmRequest,
|
||||
"tool" => SpanType::Tool,
|
||||
_ => SpanType::Other,
|
||||
}
|
||||
}
|
||||
|
||||
fn framework(attributes: &BTreeMap<String, String>) -> &'static str {
|
||||
if attr(attributes, "query_source_safe") == "sdk"
|
||||
|| attr(attributes, "system_prompt_preview").contains("cc_entrypoint=sdk")
|
||||
{
|
||||
AGENT_SDK_FRAMEWORK
|
||||
} else {
|
||||
CLAUDE_CODE_AGENT
|
||||
}
|
||||
}
|
||||
|
||||
fn split_header(text: &str) -> Option<(&str, &str)> {
|
||||
let (header, body) = text.strip_prefix('[')?.split_once("]\n")?;
|
||||
Some((header, body))
|
||||
}
|
||||
|
||||
fn without_header<'a>(text: &'a str, prefix: &str) -> &'a str {
|
||||
split_header(text)
|
||||
.filter(|(header, _)| header.starts_with(prefix))
|
||||
.map_or(text, |(_, body)| body)
|
||||
}
|
||||
|
||||
fn tool_arguments(attributes: &BTreeMap<String, String>) -> Option<&str> {
|
||||
let arguments = without_header(attr(attributes, "tool_input"), "TOOL INPUT");
|
||||
serde_json::from_str::<Map<String, Value>>(arguments)
|
||||
.is_ok()
|
||||
.then_some(arguments)
|
||||
}
|
||||
|
||||
fn tool_input(attributes: &BTreeMap<String, String>) -> String {
|
||||
if let Some(arguments) = tool_arguments(attributes) {
|
||||
return arguments.to_owned();
|
||||
}
|
||||
let fields: Map<String, Value> = [
|
||||
("command", "full_command"),
|
||||
("file_path", "file_path"),
|
||||
("bash_argv0", "bash_argv0"),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(key, source)| {
|
||||
let value = attr(attributes, source);
|
||||
(!value.is_empty()).then(|| (key.to_owned(), Value::String(value.to_owned())))
|
||||
})
|
||||
.collect();
|
||||
if fields.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
Value::Object(fields).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_output(attributes: &BTreeMap<String, String>, events: &[DecodedEvent]) -> String {
|
||||
events
|
||||
.iter()
|
||||
.filter(|event| event.name == "tool.output")
|
||||
.flat_map(|event| {
|
||||
["output", "content", "diff"]
|
||||
.into_iter()
|
||||
.map(|key| attr(&event.attributes, key))
|
||||
})
|
||||
.find(|value| !value.is_empty())
|
||||
.unwrap_or_else(|| without_header(attr(attributes, "new_context"), "TOOL RESULT"))
|
||||
.to_owned()
|
||||
}
|
||||
|
||||
fn context_message(context: &str) -> Value {
|
||||
let (role, content) = match split_header(context) {
|
||||
Some(("USER" | "USER PROMPT", body)) => ("user", body),
|
||||
Some(("ASSISTANT", body)) => ("assistant", body),
|
||||
Some((header, body)) if header.starts_with("TOOL RESULT") => ("tool", body),
|
||||
_ => ("user", context),
|
||||
};
|
||||
json!({"role": role, "content": content})
|
||||
}
|
||||
|
||||
fn user_prompt(attributes: &BTreeMap<String, String>) -> String {
|
||||
let prompt = attr(attributes, "user_prompt");
|
||||
if prompt.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
json!([{"role": "user", "content": prompt}]).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn llm_input(attributes: &BTreeMap<String, String>) -> String {
|
||||
let messages: Vec<Value> = [
|
||||
Some(attr(attributes, "system_prompt_preview"))
|
||||
.filter(|system| !system.is_empty())
|
||||
.map(|system| json!({"role": "system", "content": system})),
|
||||
Some(attr(attributes, "new_context"))
|
||||
.filter(|context| !context.is_empty())
|
||||
.map(context_message),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
if messages.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
Value::Array(messages).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn llm_output(attributes: &BTreeMap<String, String>) -> String {
|
||||
let output = attr(attributes, "response.model_output");
|
||||
if output.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
json!({"role": "assistant", "content": output}).to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn input_tokens(attributes: &BTreeMap<String, String>) -> Result<u32, DecodeError> {
|
||||
["input_tokens", "cache_read_tokens", "cache_creation_tokens"]
|
||||
.into_iter()
|
||||
.try_fold(0u32, |total, key| {
|
||||
total
|
||||
.checked_add(tokens(attributes, key)?)
|
||||
.ok_or(DecodeError::TokenCountOutOfRange)
|
||||
})
|
||||
}
|
||||
|
||||
impl SpanNormalizer for ClaudeCodeNormalizer {
|
||||
fn matches(&self, scope_name: &str, _attributes: &BTreeMap<String, String>) -> bool {
|
||||
scope_name == CLAUDE_CODE_SCOPE
|
||||
}
|
||||
|
||||
fn consumed_attributes(&self, attributes: &BTreeMap<String, String>) -> [&'static str; 2] {
|
||||
match span_type("", attributes) {
|
||||
SpanType::Interaction => ["user_prompt", ""],
|
||||
SpanType::LlmRequest => ["new_context", "response.model_output"],
|
||||
SpanType::Tool if tool_arguments(attributes).is_some() => ["tool_input", ""],
|
||||
SpanType::Tool | SpanType::Other => ["", ""],
|
||||
}
|
||||
}
|
||||
|
||||
fn display_name(&self, attributes: &BTreeMap<String, String>) -> Option<String> {
|
||||
let tool_name = attr(attributes, "tool_name");
|
||||
(matches!(span_type("", attributes), SpanType::Tool) && !tool_name.is_empty())
|
||||
.then(|| tool_name.to_owned())
|
||||
}
|
||||
|
||||
fn normalize(
|
||||
&self,
|
||||
name: &str,
|
||||
_parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
events: &[DecodedEvent],
|
||||
) -> Result<NormalizedSpan, DecodeError> {
|
||||
let base = NormalizedSpan {
|
||||
observation_type: ObservationType::Framework,
|
||||
agent_name: CLAUDE_CODE_AGENT.to_owned(),
|
||||
framework: framework(attributes).to_owned(),
|
||||
litellm_request_id: String::new(),
|
||||
model: String::new(),
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
input: String::new(),
|
||||
output: String::new(),
|
||||
};
|
||||
Ok(match span_type(name, attributes) {
|
||||
SpanType::Interaction => NormalizedSpan {
|
||||
observation_type: ObservationType::Agent,
|
||||
input: user_prompt(attributes),
|
||||
..base
|
||||
},
|
||||
SpanType::LlmRequest => NormalizedSpan {
|
||||
observation_type: ObservationType::Llm,
|
||||
litellm_request_id: first(attributes, "gen_ai.response.id", "request_id")
|
||||
.to_owned(),
|
||||
model: first(attributes, "model", "gen_ai.request.model").to_owned(),
|
||||
input_tokens: input_tokens(attributes)?,
|
||||
output_tokens: tokens(attributes, "output_tokens")?,
|
||||
input: llm_input(attributes),
|
||||
output: llm_output(attributes),
|
||||
..base
|
||||
},
|
||||
SpanType::Tool => NormalizedSpan {
|
||||
observation_type: ObservationType::Tool,
|
||||
input: tool_input(attributes),
|
||||
output: tool_output(attributes, events),
|
||||
..base
|
||||
},
|
||||
SpanType::Other => base,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use rstest::rstest;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{CLAUDE_CODE_SCOPE, ClaudeCodeNormalizer, SpanNormalizer};
|
||||
use crate::{DecodeError, normalize::ObservationType, otlp::DecodedEvent};
|
||||
|
||||
fn attributes(pairs: &[(&str, &str)]) -> BTreeMap<String, String> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(key, value)| ((*key).to_owned(), (*value).to_owned()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn tool_without_detailed_input_lists_known_arguments() {
|
||||
let span = ClaudeCodeNormalizer
|
||||
.normalize(
|
||||
"claude_code.tool",
|
||||
"parent",
|
||||
&attributes(&[
|
||||
("span.type", "tool"),
|
||||
("tool_name", "Bash"),
|
||||
("full_command", "git status"),
|
||||
("bash_argv0", "git"),
|
||||
]),
|
||||
&[],
|
||||
)
|
||||
.expect("valid span");
|
||||
let input: Value = serde_json::from_str(&span.input).expect("argument object");
|
||||
assert_eq!(input["command"], "git status");
|
||||
assert_eq!(input["bash_argv0"], "git");
|
||||
assert!(input.get("file_path").is_none());
|
||||
assert!(input.get("role").is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn malformed_tool_input_falls_back_and_stays_in_attributes() {
|
||||
let attrs = attributes(&[
|
||||
("span.type", "tool"),
|
||||
("tool_input", "[TOOL INPUT: Read]\nnot json"),
|
||||
("file_path", "/workspace/a.py"),
|
||||
]);
|
||||
let span = ClaudeCodeNormalizer
|
||||
.normalize("claude_code.tool", "parent", &attrs, &[])
|
||||
.expect("valid span");
|
||||
let input: Value = serde_json::from_str(&span.input).expect("argument object");
|
||||
assert_eq!(input["file_path"], "/workspace/a.py");
|
||||
assert!(
|
||||
!ClaudeCodeNormalizer
|
||||
.consumed_attributes(&attrs)
|
||||
.contains(&"tool_input")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::event_output(
|
||||
vec![DecodedEvent { name: "tool.output".to_owned(), attributes: attributes(&[("output", "stdout text")]) }],
|
||||
"stdout text"
|
||||
)]
|
||||
#[case::event_diff(
|
||||
vec![DecodedEvent { name: "tool.output".to_owned(), attributes: attributes(&[("diff", "+line")]) }],
|
||||
"+line"
|
||||
)]
|
||||
#[case::other_event_ignored(
|
||||
vec![DecodedEvent { name: "other".to_owned(), attributes: attributes(&[("output", "nope")]) }],
|
||||
"{\"stdout\":\"ctx\"}"
|
||||
)]
|
||||
fn tool_output_prefers_event_then_context(
|
||||
#[case] events: Vec<DecodedEvent>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let span = ClaudeCodeNormalizer
|
||||
.normalize(
|
||||
"claude_code.tool",
|
||||
"parent",
|
||||
&attributes(&[
|
||||
("span.type", "tool"),
|
||||
("new_context", "[TOOL RESULT: Bash]\n{\"stdout\":\"ctx\"}"),
|
||||
]),
|
||||
&events,
|
||||
)
|
||||
.expect("valid span");
|
||||
assert_eq!(span.output, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn llm_tool_result_context_becomes_tool_message() {
|
||||
let span = ClaudeCodeNormalizer
|
||||
.normalize(
|
||||
"claude_code.llm_request",
|
||||
"parent",
|
||||
&attributes(&[
|
||||
("span.type", "llm_request"),
|
||||
("new_context", "[TOOL RESULT: toolu_1]\n1\timport os"),
|
||||
]),
|
||||
&[],
|
||||
)
|
||||
.expect("valid span");
|
||||
let input: Value = serde_json::from_str(&span.input).expect("messages");
|
||||
assert_eq!(input[0]["role"], "tool");
|
||||
assert_eq!(input[0]["content"], "1\timport os");
|
||||
assert_eq!(span.output, "");
|
||||
assert_eq!(span.framework, "claude-code");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn llm_token_sum_overflow_is_rejected() {
|
||||
let result = ClaudeCodeNormalizer.normalize(
|
||||
"claude_code.llm_request",
|
||||
"parent",
|
||||
&attributes(&[
|
||||
("span.type", "llm_request"),
|
||||
("input_tokens", "4294967295"),
|
||||
("cache_read_tokens", "1"),
|
||||
]),
|
||||
&[],
|
||||
);
|
||||
assert!(matches!(result, Err(DecodeError::TokenCountOutOfRange)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::span_type_wins("claude_code.tool", "hook", ObservationType::Framework)]
|
||||
#[case::name_fallback("claude_code.interaction", "", ObservationType::Agent)]
|
||||
#[case::unknown("claude_code.something_new", "", ObservationType::Framework)]
|
||||
fn span_type_attribute_then_name_select_the_observation(
|
||||
#[case] name: &str,
|
||||
#[case] kind: &str,
|
||||
#[case] expected: ObservationType,
|
||||
) {
|
||||
let attrs = if kind.is_empty() {
|
||||
BTreeMap::new()
|
||||
} else {
|
||||
attributes(&[("span.type", kind)])
|
||||
};
|
||||
let span = ClaudeCodeNormalizer
|
||||
.normalize(name, "parent", &attrs, &[])
|
||||
.expect("valid span");
|
||||
assert_eq!(span.observation_type, expected);
|
||||
assert!(ClaudeCodeNormalizer.matches(CLAUDE_CODE_SCOPE, &attrs));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use super::{NormalizedSpan, ObservationType, SpanNormalizer, attr, first, usage_tokens};
|
||||
use crate::DecodeError;
|
||||
use crate::{DecodeError, otlp::DecodedEvent};
|
||||
|
||||
pub(super) struct GenAiNormalizer;
|
||||
|
||||
|
|
@ -30,6 +30,7 @@ impl SpanNormalizer for GenAiNormalizer {
|
|||
_name: &str,
|
||||
parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
_events: &[DecodedEvent],
|
||||
) -> Result<NormalizedSpan, DecodeError> {
|
||||
let (input_tokens, output_tokens) = usage_tokens(attributes)?;
|
||||
let observation_type = match attr(attributes, "gen_ai.operation.name") {
|
||||
|
|
@ -42,6 +43,7 @@ impl SpanNormalizer for GenAiNormalizer {
|
|||
Ok(NormalizedSpan {
|
||||
observation_type,
|
||||
agent_name: attr(attributes, "gen_ai.agent.name").to_owned(),
|
||||
framework: String::new(),
|
||||
litellm_request_id: attr(attributes, "gen_ai.response.id").to_owned(),
|
||||
model: first(attributes, "gen_ai.request.model", "gen_ai.response.model").to_owned(),
|
||||
input_tokens,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned};
|
|||
use serde_json::{Value, ser::Formatter};
|
||||
|
||||
use super::{NormalizedSpan, ObservationType, SpanNormalizer, attr, usage_tokens};
|
||||
use crate::DecodeError;
|
||||
use crate::{DecodeError, otlp::DecodedEvent};
|
||||
|
||||
pub(super) struct LangSmithNormalizer;
|
||||
|
||||
|
|
@ -404,6 +404,7 @@ impl SpanNormalizer for LangSmithNormalizer {
|
|||
name: &str,
|
||||
parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
_events: &[DecodedEvent],
|
||||
) -> Result<NormalizedSpan, DecodeError> {
|
||||
let (input_tokens, output_tokens) = usage_tokens(attributes)?;
|
||||
let observation_type = span_type(name, parent_span_id, attributes);
|
||||
|
|
@ -411,6 +412,7 @@ impl SpanNormalizer for LangSmithNormalizer {
|
|||
Ok(NormalizedSpan {
|
||||
observation_type,
|
||||
agent_name: attr(attributes, "langsmith.metadata.lc_agent_name").to_owned(),
|
||||
framework: String::new(),
|
||||
litellm_request_id: io.request_id,
|
||||
model: attr(attributes, "gen_ai.request.model").to_owned(),
|
||||
input_tokens,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::DecodeError;
|
||||
use crate::{DecodeError, otlp::DecodedEvent};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
|
||||
|
|
@ -17,6 +17,7 @@ pub enum ObservationType {
|
|||
pub struct NormalizedSpan {
|
||||
pub observation_type: ObservationType,
|
||||
pub agent_name: String,
|
||||
pub framework: String,
|
||||
pub litellm_request_id: String,
|
||||
pub model: String,
|
||||
pub input_tokens: u32,
|
||||
|
|
@ -27,6 +28,7 @@ pub struct NormalizedSpan {
|
|||
|
||||
pub(crate) struct Normalization {
|
||||
pub span: NormalizedSpan,
|
||||
pub display_name: Option<String>,
|
||||
pub consumed_attributes: [&'static str; 2],
|
||||
}
|
||||
|
||||
|
|
@ -38,7 +40,7 @@ pub struct NormalizedFieldDefinition {
|
|||
pub meaning: &'static str,
|
||||
}
|
||||
|
||||
pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 8] = [
|
||||
pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 9] = [
|
||||
NormalizedFieldDefinition {
|
||||
name: "observation_type",
|
||||
clickhouse_column: "ObservationType",
|
||||
|
|
@ -51,6 +53,12 @@ pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 8] = [
|
|||
clickhouse_type: "LowCardinality(String)",
|
||||
meaning: "Agent associated with this span",
|
||||
},
|
||||
NormalizedFieldDefinition {
|
||||
name: "framework",
|
||||
clickhouse_column: "Framework",
|
||||
clickhouse_type: "LowCardinality(String)",
|
||||
meaning: "Agent framework or SDK that emitted this span, e.g. claude-agent-sdk",
|
||||
},
|
||||
NormalizedFieldDefinition {
|
||||
name: "litellm_request_id",
|
||||
clickhouse_column: "LiteLLMRequestId",
|
||||
|
|
@ -97,13 +105,20 @@ trait SpanNormalizer {
|
|||
name: &str,
|
||||
parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
events: &[DecodedEvent],
|
||||
) -> Result<NormalizedSpan, DecodeError>;
|
||||
fn display_name(&self, _attributes: &BTreeMap<String, String>) -> Option<String> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
mod claude_code;
|
||||
mod genai;
|
||||
mod langsmith;
|
||||
mod openinference;
|
||||
|
||||
use claude_code::ClaudeCodeNormalizer;
|
||||
pub(crate) use claude_code::{CLAUDE_CODE_AGENT, CLAUDE_CODE_SCOPE};
|
||||
use genai::GenAiNormalizer;
|
||||
use langsmith::LangSmithNormalizer;
|
||||
use openinference::OpenInferenceNormalizer;
|
||||
|
|
@ -207,8 +222,10 @@ pub fn normalize(
|
|||
name: &str,
|
||||
parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
events: &[DecodedEvent],
|
||||
) -> Result<Normalization, DecodeError> {
|
||||
let normalizers: [&dyn SpanNormalizer; 3] = [
|
||||
let normalizers: [&dyn SpanNormalizer; 4] = [
|
||||
&ClaudeCodeNormalizer,
|
||||
&LangSmithNormalizer,
|
||||
&OpenInferenceNormalizer,
|
||||
&GenAiNormalizer,
|
||||
|
|
@ -217,7 +234,7 @@ pub fn normalize(
|
|||
.into_iter()
|
||||
.find(|normalizer| normalizer.matches(scope_name, attributes))
|
||||
.expect("GenAI fallback always matches");
|
||||
let span = normalizer.normalize(name, parent_span_id, attributes)?;
|
||||
let span = normalizer.normalize(name, parent_span_id, attributes, events)?;
|
||||
let agent_name = recorded_agent_name(name, attributes, &span);
|
||||
let observation_type = if !parent_span_id.is_empty()
|
||||
&& scope_name == "openinference.instrumentation.langchain"
|
||||
|
|
@ -233,6 +250,7 @@ pub fn normalize(
|
|||
observation_type,
|
||||
..span
|
||||
},
|
||||
display_name: normalizer.display_name(attributes),
|
||||
consumed_attributes: normalizer.consumed_attributes(attributes),
|
||||
})
|
||||
}
|
||||
|
|
@ -249,6 +267,7 @@ mod tests {
|
|||
#[case::langsmith("langsmith", [("langsmith.span.kind", "llm"), ("openinference.span.kind", "TOOL")], ObservationType::Llm)]
|
||||
#[case::openinference("other", [("openinference.span.kind", "LLM"), ("gen_ai.operation.name", "execute_tool")], ObservationType::Llm)]
|
||||
#[case::genai("other", [("gen_ai.operation.name", "execute_tool"), ("gen_ai.usage.input_tokens", "7")], ObservationType::Tool)]
|
||||
#[case::claude_code("com.anthropic.claude_code.tracing", [("span.type", "llm_request"), ("openinference.span.kind", "TOOL")], ObservationType::Llm)]
|
||||
fn convention_dispatch_preserves_precedence(
|
||||
#[case] scope: &str,
|
||||
#[case] attributes: [(&str, &str); 2],
|
||||
|
|
@ -258,7 +277,7 @@ mod tests {
|
|||
.into_iter()
|
||||
.map(|(key, value)| (key.to_owned(), value.to_owned()))
|
||||
.collect();
|
||||
let fields = normalize(scope, "step", "parent", &attributes)
|
||||
let fields = normalize(scope, "step", "parent", &attributes, &[])
|
||||
.expect("valid tokens")
|
||||
.span;
|
||||
assert_eq!(fields.observation_type, expected);
|
||||
|
|
@ -269,7 +288,7 @@ mod tests {
|
|||
|
||||
#[rstest]
|
||||
fn field_definitions_match_serialized_normalized_span() {
|
||||
let fields = normalize("", "root", "", &BTreeMap::new())
|
||||
let fields = normalize("", "root", "", &BTreeMap::new(), &[])
|
||||
.expect("valid tokens")
|
||||
.span;
|
||||
let serialized = serde_json::to_value(fields).expect("serializable fields");
|
||||
|
|
@ -290,7 +309,7 @@ mod tests {
|
|||
fn token_counts_accept_surrounding_whitespace() {
|
||||
let attributes =
|
||||
BTreeMap::from([("gen_ai.usage.input_tokens".to_owned(), " 7 ".to_owned())]);
|
||||
let fields = normalize("", "root", "", &attributes)
|
||||
let fields = normalize("", "root", "", &attributes, &[])
|
||||
.expect("valid tokens")
|
||||
.span;
|
||||
assert_eq!(fields.input_tokens, 7);
|
||||
|
|
@ -302,6 +321,6 @@ mod tests {
|
|||
fn token_counts_outside_storage_range_are_rejected(#[case] value: &str) {
|
||||
let attributes =
|
||||
BTreeMap::from([("gen_ai.usage.input_tokens".to_owned(), value.to_owned())]);
|
||||
assert!(normalize("", "root", "", &attributes).is_err());
|
||||
assert!(normalize("", "root", "", &attributes, &[]).is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use super::{NormalizedSpan, ObservationType, SpanNormalizer, attr, tokens, usage_tokens};
|
||||
use crate::DecodeError;
|
||||
use crate::{DecodeError, otlp::DecodedEvent};
|
||||
|
||||
pub(super) struct OpenInferenceNormalizer;
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ impl SpanNormalizer for OpenInferenceNormalizer {
|
|||
_name: &str,
|
||||
parent_span_id: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
_events: &[DecodedEvent],
|
||||
) -> Result<NormalizedSpan, DecodeError> {
|
||||
let (usage_input, usage_output) = usage_tokens(attributes)?;
|
||||
let observation_type = match attr(attributes, "openinference.span.kind")
|
||||
|
|
@ -34,6 +35,7 @@ impl SpanNormalizer for OpenInferenceNormalizer {
|
|||
Ok(NormalizedSpan {
|
||||
observation_type,
|
||||
agent_name: attr(attributes, "agent.name").to_owned(),
|
||||
framework: String::new(),
|
||||
litellm_request_id: String::new(),
|
||||
model: attr(attributes, "llm.model_name").to_owned(),
|
||||
input_tokens: if attributes.contains_key("llm.token_count.prompt") {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,10 @@ use super::{
|
|||
attributes::attributes,
|
||||
limits::{Budget, MAX_ATTRIBUTES, MAX_DECODED_SPAN_BYTES, MAX_EVENTS, MAX_SPANS},
|
||||
};
|
||||
use crate::{DecodeError, Shared, normalize::normalize};
|
||||
use crate::{
|
||||
DecodeError, Shared,
|
||||
normalize::{CLAUDE_CODE_AGENT, CLAUDE_CODE_SCOPE, normalize},
|
||||
};
|
||||
|
||||
pub(super) fn flatten(request: ExportTraceServiceRequest) -> Result<Vec<DecodedSpan>, DecodeError> {
|
||||
let mut budget = Budget::new(MAX_DECODED_SPAN_BYTES);
|
||||
|
|
@ -127,11 +130,23 @@ fn decoded_span(
|
|||
let status = span.status.unwrap_or_default();
|
||||
let parent_span_id = hex_bytes(&span.parent_span_id);
|
||||
let span_attributes = attributes(span.attributes, budget)?;
|
||||
let events = span
|
||||
.events
|
||||
.into_iter()
|
||||
.map(|event| {
|
||||
budget.consume(event.name.len() + 96)?;
|
||||
Ok(DecodedEvent {
|
||||
name: event.name,
|
||||
attributes: attributes(event.attributes, budget)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, DecodeError>>()?;
|
||||
let normalization = normalize(
|
||||
scope_name.as_ref(),
|
||||
&span.name,
|
||||
&parent_span_id,
|
||||
&span_attributes,
|
||||
&events,
|
||||
)?;
|
||||
let resource_agent_name = resource_attributes
|
||||
.get("gen_ai.agent.name")
|
||||
|
|
@ -139,6 +154,13 @@ fn decoded_span(
|
|||
let agent_name = match (resource_agent_name, normalization.span.agent_name.as_str()) {
|
||||
(Some(name), "") => name.clone(),
|
||||
(Some(name), "hermes-agent") if scope_name.as_ref() == "hermes-otel-plugin" => name.clone(),
|
||||
(Some(name), CLAUDE_CODE_AGENT) if scope_name.as_ref() == CLAUDE_CODE_SCOPE => name.clone(),
|
||||
(None, CLAUDE_CODE_AGENT) if scope_name.as_ref() == CLAUDE_CODE_SCOPE => {
|
||||
resource_attributes
|
||||
.get("service.name")
|
||||
.filter(|name| !name.is_empty())
|
||||
.map_or_else(|| CLAUDE_CODE_AGENT.to_owned(), Clone::clone)
|
||||
}
|
||||
(_, name) => name.to_owned(),
|
||||
};
|
||||
let normalized = crate::normalize::NormalizedSpan {
|
||||
|
|
@ -149,15 +171,17 @@ fn decoded_span(
|
|||
normalized.input.len()
|
||||
+ normalized.output.len()
|
||||
+ normalized.agent_name.len()
|
||||
+ normalized.framework.len()
|
||||
+ normalized.litellm_request_id.len()
|
||||
+ normalized.model.len(),
|
||||
+ normalized.model.len()
|
||||
+ normalization.display_name.as_ref().map_or(0, String::len),
|
||||
)?;
|
||||
Ok(DecodedSpan {
|
||||
trace_id: hex_bytes(&span.trace_id),
|
||||
span_id: hex_bytes(&span.span_id),
|
||||
parent_span_id,
|
||||
trace_state: span.trace_state,
|
||||
name: span.name,
|
||||
name: normalization.display_name.unwrap_or(span.name),
|
||||
kind: SpanKind::try_from(span.kind)
|
||||
.unwrap_or(SpanKind::Unspecified)
|
||||
.as_str_name()
|
||||
|
|
@ -178,17 +202,7 @@ fn decoded_span(
|
|||
.as_str_name()
|
||||
.to_owned(),
|
||||
status_message: status.message,
|
||||
events: span
|
||||
.events
|
||||
.into_iter()
|
||||
.map(|event| {
|
||||
budget.consume(event.name.len() + 96)?;
|
||||
Ok(DecodedEvent {
|
||||
name: event.name,
|
||||
attributes: attributes(event.attributes, budget)?,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, DecodeError>>()?,
|
||||
events,
|
||||
normalized,
|
||||
consumed_attributes: normalization.consumed_attributes,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -371,14 +371,54 @@ async fn listed_agent_names_preserve_scope_and_cursor(
|
|||
let writer = Connection::writer(&database.url)?;
|
||||
ensure_schema(&database.client, &writer, "trace_test", 7).await?;
|
||||
let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
|
||||
for (team, key, trace, agent, span, parent) in [
|
||||
("alpha", "one", "shared", "research_agent", "root", ""),
|
||||
("alpha", "one", "shared", "reviewer", "child", "root"),
|
||||
("alpha", "one", "shared", "reviewer", "repeated", "root"),
|
||||
("alpha", "one", "shared", "", "unnamed", "root"),
|
||||
("alpha", "one", "second", "support_agent", "root", ""),
|
||||
("alpha", "two", "shared", "private_agent", "root", ""),
|
||||
("beta", "one", "shared", "other_agent", "root", ""),
|
||||
for (team, key, trace, agent, span, parent, framework) in [
|
||||
(
|
||||
"alpha",
|
||||
"one",
|
||||
"shared",
|
||||
"research_agent",
|
||||
"root",
|
||||
"",
|
||||
"claude-code",
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"one",
|
||||
"shared",
|
||||
"reviewer",
|
||||
"child",
|
||||
"root",
|
||||
"claude-agent-sdk",
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"one",
|
||||
"shared",
|
||||
"reviewer",
|
||||
"repeated",
|
||||
"root",
|
||||
"claude-agent-sdk",
|
||||
),
|
||||
("alpha", "one", "shared", "", "unnamed", "root", ""),
|
||||
("alpha", "one", "second", "support_agent", "root", "", ""),
|
||||
(
|
||||
"alpha",
|
||||
"two",
|
||||
"shared",
|
||||
"private_agent",
|
||||
"root",
|
||||
"",
|
||||
"private-sdk",
|
||||
),
|
||||
(
|
||||
"beta",
|
||||
"one",
|
||||
"shared",
|
||||
"other_agent",
|
||||
"root",
|
||||
"",
|
||||
"other-sdk",
|
||||
),
|
||||
] {
|
||||
insert_rows(
|
||||
&database,
|
||||
|
|
@ -386,7 +426,7 @@ async fn listed_agent_names_preserve_scope_and_cursor(
|
|||
vec![serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent,
|
||||
"ServiceName": "shared-app", "SpanName": span, "AgentName": agent,
|
||||
"ObservationType": "agent",
|
||||
"Framework": framework, "ObservationType": "agent",
|
||||
"ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": key}
|
||||
}))?],
|
||||
)
|
||||
|
|
@ -477,6 +517,15 @@ async fn listed_agent_names_preserve_scope_and_cursor(
|
|||
serde_json::json!(["research_agent", "reviewer"])
|
||||
);
|
||||
assert_eq!(names["second"], serde_json::json!(["support_agent"]));
|
||||
let frameworks = [&first["data"][0], &second["data"][0]]
|
||||
.into_iter()
|
||||
.map(|row| (row["trace_id"].as_str().unwrap(), row["frameworks"].clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(
|
||||
frameworks["shared"],
|
||||
serde_json::json!(["claude-agent-sdk", "claude-code"])
|
||||
);
|
||||
assert_eq!(frameworks["second"], serde_json::json!([]));
|
||||
let counts = [&first["data"][0], &second["data"][0]]
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
|
|
|
|||
|
|
@ -386,3 +386,223 @@ fn normalizes_langsmith_fixture() {
|
|||
assert_eq!(tool.normalized.observation_type, ObservationType::Tool);
|
||||
assert!(tool.normalized.output.starts_with("Based on my research"));
|
||||
}
|
||||
|
||||
const CLAUDE_AGENT_SDK_FIXTURE: &[u8] =
|
||||
include_bytes!("../../../../tests/test_litellm/tracing/fixtures/claude_agent_sdk_export.json");
|
||||
const CLAUDE_AGENT_SDK_DETAILED_FIXTURE: &[u8] = include_bytes!(
|
||||
"../../../../tests/test_litellm/tracing/fixtures/claude_agent_sdk_detailed_export.json"
|
||||
);
|
||||
|
||||
fn raw_spans(fixture: &[u8]) -> Vec<serde_json::Value> {
|
||||
let export: serde_json::Value = serde_json::from_slice(fixture).expect("fixture JSON");
|
||||
export["resourceSpans"][0]["scopeSpans"][0]["spans"]
|
||||
.as_array()
|
||||
.expect("spans")
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn raw_attribute(span: &serde_json::Value, key: &str) -> Option<serde_json::Value> {
|
||||
span["attributes"]
|
||||
.as_array()
|
||||
.expect("attributes")
|
||||
.iter()
|
||||
.find(|attribute| attribute["key"] == key)
|
||||
.map(|attribute| attribute["value"].clone())
|
||||
}
|
||||
|
||||
fn raw_string(span: &serde_json::Value, key: &str) -> String {
|
||||
raw_attribute(span, key)
|
||||
.and_then(|value| value["stringValue"].as_str().map(str::to_owned))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn raw_int(span: &serde_json::Value, key: &str) -> u64 {
|
||||
raw_attribute(span, key).map_or(0, |value| match &value["intValue"] {
|
||||
serde_json::Value::String(text) => text.parse().expect("integer"),
|
||||
number => number.as_u64().expect("integer"),
|
||||
})
|
||||
}
|
||||
|
||||
fn raw_span<'a>(raw: &'a [serde_json::Value], span_id: &str) -> &'a serde_json::Value {
|
||||
raw.iter()
|
||||
.find(|span| {
|
||||
span["spanId"]
|
||||
.as_str()
|
||||
.is_some_and(|id| id.eq_ignore_ascii_case(span_id))
|
||||
})
|
||||
.expect("raw span")
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_telemetry(CLAUDE_AGENT_SDK_FIXTURE)]
|
||||
#[case::detailed_telemetry(CLAUDE_AGENT_SDK_DETAILED_FIXTURE)]
|
||||
fn normalizes_claude_agent_sdk_fixture(#[case] fixture: &[u8]) {
|
||||
let spans = decode_otlp(fixture, Some("application/json")).expect("valid OTLP export");
|
||||
let raw = raw_spans(fixture);
|
||||
let types: std::collections::BTreeSet<_> = spans
|
||||
.iter()
|
||||
.map(|span| format!("{:?}", span.normalized.observation_type))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
types,
|
||||
["Agent", "Framework", "Llm", "Tool"]
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
);
|
||||
|
||||
let root = spans
|
||||
.iter()
|
||||
.find(|span| span.normalized.observation_type == ObservationType::Agent)
|
||||
.expect("interaction root");
|
||||
assert!(root.parent_span_id.is_empty());
|
||||
let root_input: serde_json::Value =
|
||||
serde_json::from_str(&root.normalized.input).expect("root input messages");
|
||||
assert_eq!(root_input[0]["role"], "user");
|
||||
assert_eq!(
|
||||
root_input[0]["content"],
|
||||
raw_string(raw_span(&raw, &root.span_id), "user_prompt")
|
||||
);
|
||||
assert!(root.consumed_attributes.contains(&"user_prompt"));
|
||||
|
||||
let tools: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|span| span.normalized.observation_type == ObservationType::Tool)
|
||||
.collect();
|
||||
assert_eq!(tools.len(), 2);
|
||||
for tool in &tools {
|
||||
assert_eq!(
|
||||
tool.name,
|
||||
raw_string(raw_span(&raw, &tool.span_id), "tool_name")
|
||||
);
|
||||
let input: serde_json::Value =
|
||||
serde_json::from_str(&tool.normalized.input).expect("tool argument object");
|
||||
assert!(input.is_object());
|
||||
assert!(input.get("role").is_none());
|
||||
let event = tool
|
||||
.events
|
||||
.iter()
|
||||
.find(|event| event.name == "tool.output")
|
||||
.expect("tool output event");
|
||||
let expected_output = ["output", "content", "diff"]
|
||||
.into_iter()
|
||||
.filter_map(|key| event.attributes.get(key))
|
||||
.find(|value| !value.is_empty())
|
||||
.expect("event output");
|
||||
assert_eq!(&tool.normalized.output, expected_output);
|
||||
}
|
||||
let bash = tools
|
||||
.iter()
|
||||
.find(|tool| tool.name == "Bash")
|
||||
.expect("Bash tool");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&bash.normalized.input).unwrap()["command"],
|
||||
raw_string(raw_span(&raw, &bash.span_id), "full_command")
|
||||
);
|
||||
|
||||
let llms: Vec<_> = spans
|
||||
.iter()
|
||||
.filter(|span| span.normalized.observation_type == ObservationType::Llm)
|
||||
.collect();
|
||||
assert!(!llms.is_empty());
|
||||
for llm in &llms {
|
||||
let raw_llm = raw_span(&raw, &llm.span_id);
|
||||
let expected = raw_int(raw_llm, "input_tokens")
|
||||
+ raw_int(raw_llm, "cache_read_tokens")
|
||||
+ raw_int(raw_llm, "cache_creation_tokens");
|
||||
assert_eq!(u64::from(llm.normalized.input_tokens), expected);
|
||||
assert_eq!(
|
||||
u64::from(llm.normalized.output_tokens),
|
||||
raw_int(raw_llm, "output_tokens")
|
||||
);
|
||||
assert_eq!(llm.normalized.model, raw_string(raw_llm, "model"));
|
||||
if raw_string(raw_llm, "query_source_safe") == "sdk" {
|
||||
assert_eq!(llm.normalized.framework, "claude-agent-sdk");
|
||||
}
|
||||
}
|
||||
assert!(spans.iter().all(|span| {
|
||||
span.normalized.agent_name == span.resource_attributes["service.name"].as_str()
|
||||
}));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn claude_agent_sdk_detailed_fixture_keeps_full_tool_arguments_and_llm_messages() {
|
||||
let spans = decode_otlp(CLAUDE_AGENT_SDK_DETAILED_FIXTURE, Some("application/json"))
|
||||
.expect("valid OTLP export");
|
||||
let raw = raw_spans(CLAUDE_AGENT_SDK_DETAILED_FIXTURE);
|
||||
let bash = spans
|
||||
.iter()
|
||||
.find(|span| span.name == "Bash")
|
||||
.expect("Bash tool");
|
||||
let tool_input = raw_string(raw_span(&raw, &bash.span_id), "tool_input");
|
||||
let (_, arguments) = tool_input.split_once('\n').expect("tool input header");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&bash.normalized.input).unwrap(),
|
||||
serde_json::from_str::<serde_json::Value>(arguments).unwrap()
|
||||
);
|
||||
assert!(bash.consumed_attributes.contains(&"tool_input"));
|
||||
|
||||
let answer = spans
|
||||
.iter()
|
||||
.find(|span| {
|
||||
span.normalized.observation_type == ObservationType::Llm
|
||||
&& span.attributes.get("query_source_safe").map(String::as_str) == Some("sdk")
|
||||
&& !span.normalized.output.is_empty()
|
||||
})
|
||||
.expect("final SDK answer");
|
||||
let raw_answer = raw_span(&raw, &answer.span_id);
|
||||
let input: serde_json::Value =
|
||||
serde_json::from_str(&answer.normalized.input).expect("llm input messages");
|
||||
assert_eq!(input[0]["role"], "system");
|
||||
assert_eq!(
|
||||
input[0]["content"],
|
||||
raw_string(raw_answer, "system_prompt_preview")
|
||||
);
|
||||
let output: serde_json::Value =
|
||||
serde_json::from_str(&answer.normalized.output).expect("llm output message");
|
||||
assert_eq!(output["role"], "assistant");
|
||||
assert_eq!(
|
||||
output["content"],
|
||||
raw_string(raw_answer, "response.model_output")
|
||||
);
|
||||
|
||||
let title = spans
|
||||
.iter()
|
||||
.find(|span| {
|
||||
span.attributes.get("query_source_safe").map(String::as_str)
|
||||
== Some("generate_session_title")
|
||||
})
|
||||
.expect("side query");
|
||||
assert_eq!(title.normalized.framework, "claude-agent-sdk");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn claude_code_scope_takes_precedence_over_openinference_attributes(
|
||||
mut span: opentelemetry_proto::tonic::trace::v1::Span,
|
||||
) {
|
||||
use opentelemetry_proto::tonic::common::v1::{
|
||||
AnyValue, InstrumentationScope, KeyValue, any_value::Value,
|
||||
};
|
||||
let string = |key: &str, value: &str| KeyValue {
|
||||
key: key.to_owned(),
|
||||
value: Some(AnyValue {
|
||||
value: Some(Value::StringValue(value.to_owned())),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
span.attributes = vec![
|
||||
string("span.type", "tool"),
|
||||
string("tool_name", "Grep"),
|
||||
string("openinference.span.kind", "LLM"),
|
||||
];
|
||||
let mut request = request_with(span);
|
||||
request.resource_spans[0].scope_spans[0].scope = Some(InstrumentationScope {
|
||||
name: "com.anthropic.claude_code.tracing".to_owned(),
|
||||
..Default::default()
|
||||
});
|
||||
let spans = decode_otlp(&prost::Message::encode_to_vec(&request), None).expect("valid span");
|
||||
assert_eq!(spans[0].normalized.observation_type, ObservationType::Tool);
|
||||
assert_eq!(spans[0].name, "Grep");
|
||||
assert_eq!(spans[0].normalized.framework, "claude-code");
|
||||
assert_eq!(spans[0].normalized.agent_name, "claude-code");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ class NormalizedSpan(BaseModel):
|
|||
|
||||
observation_type: Literal["agent", "llm", "tool", "chain", "framework"]
|
||||
agent_name: str
|
||||
framework: str
|
||||
litellm_request_id: str
|
||||
model: str
|
||||
input_tokens: int = Field(ge=0, le=2**32 - 1)
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ def _span_row(span: DecodedSpan) -> SpanRow:
|
|||
ApiKeyHash="",
|
||||
ObservationType=normalized.observation_type,
|
||||
AgentName=normalized.agent_name,
|
||||
Framework=normalized.framework,
|
||||
Model=normalized.model,
|
||||
LiteLLMRequestId=attributes.get("gen_ai.response.id") or normalized.litellm_request_id,
|
||||
InputTokens=normalized.input_tokens,
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ def trace_summary_from_row(row: dict[str, Any], spend_rows: Sequence[_SpendRow]
|
|||
name=row["name"],
|
||||
service=row["service"],
|
||||
agent_names=tuple(row.get("agent_names") or ()),
|
||||
frameworks=tuple(row.get("frameworks") or ()),
|
||||
input_preview=row["input_preview"],
|
||||
start_time=_iso(int(row["start_ms"])),
|
||||
duration_ms=float(row["duration_ms"]),
|
||||
|
|
@ -146,6 +147,7 @@ def span_from_row(row: dict[str, Any], trace_start_ns: int, spend_rows: Sequence
|
|||
name=row["name"],
|
||||
type=row["type"],
|
||||
agent=row["agent"],
|
||||
framework=row.get("framework") or "",
|
||||
start_offset_ms=(int(row["start_ns"]) - trace_start_ns) / NANOS_PER_MS,
|
||||
duration_ms=int(row["duration_ns"]) / NANOS_PER_MS,
|
||||
status=_status(row["status"]),
|
||||
|
|
@ -252,6 +254,7 @@ def trace_from_rows(
|
|||
name=root["name"],
|
||||
service=rows[0]["service"],
|
||||
agent_names=tuple(sorted(frozenset(s["agent"] for s in spans if s["agent"]))),
|
||||
frameworks=tuple(sorted(frozenset(s["framework"] for s in spans if s["framework"]))),
|
||||
input_preview=root["input_preview"],
|
||||
start_time=_iso(trace_start_ns // NANOS_PER_MS),
|
||||
duration_ms=(trace_end_ns - trace_start_ns) / NANOS_PER_MS,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class Span(TypedDict):
|
|||
name: ReadOnly[str]
|
||||
type: ReadOnly[SpanType]
|
||||
agent: ReadOnly[str] # the agent this span runs inside, e.g. "researcher"
|
||||
framework: ReadOnly[str] # SDK that emitted the span, e.g. "claude-agent-sdk"; "" when unknown
|
||||
start_offset_ms: ReadOnly[float] # relative to trace start
|
||||
duration_ms: ReadOnly[float]
|
||||
status: ReadOnly[SpanStatus]
|
||||
|
|
@ -57,6 +58,7 @@ class TraceSummary(TypedDict):
|
|||
name: ReadOnly[str]
|
||||
service: ReadOnly[str]
|
||||
agent_names: ReadOnly[NotRequired[tuple[str, ...]]]
|
||||
frameworks: ReadOnly[NotRequired[tuple[str, ...]]]
|
||||
input_preview: ReadOnly[str]
|
||||
start_time: ReadOnly[str] # ISO 8601
|
||||
duration_ms: ReadOnly[float]
|
||||
|
|
@ -129,6 +131,7 @@ class SpanRow(TypedDict):
|
|||
ApiKeyHash: ReadOnly[str]
|
||||
ObservationType: SpanType
|
||||
AgentName: str
|
||||
Framework: ReadOnly[str]
|
||||
LiteLLMRequestId: str
|
||||
Model: str
|
||||
InputTokens: int
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
1051
tests/test_litellm/tracing/fixtures/claude_agent_sdk_export.json
Normal file
1051
tests/test_litellm/tracing/fixtures/claude_agent_sdk_export.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -579,3 +579,17 @@ def test_token_counts_outside_storage_range_are_rejected(count):
|
|||
exported = _span("root", b"\x01" * 8, gen_ai__usage__input_tokens=count)
|
||||
with pytest.raises(decode.InvalidOTLPPayloadError, match="storage range"):
|
||||
decode_otlp(_export(exported))
|
||||
|
||||
|
||||
def test_claude_agent_sdk_rows_carry_framework_tool_names_and_arguments():
|
||||
fixture = Path(__file__).parent / "fixtures" / "claude_agent_sdk_detailed_export.json"
|
||||
rows = decode_otlp(fixture.read_bytes(), "application/json")
|
||||
sdk_llms = [r for r in rows if r["ObservationType"] == "llm" and r["SpanAttributes"]["query_source_safe"] == "sdk"]
|
||||
assert sdk_llms and {r["Framework"] for r in sdk_llms} == {"claude-agent-sdk"}
|
||||
tools = {r["SpanName"]: r for r in rows if r["ObservationType"] == "tool"}
|
||||
assert set(tools) == {"Bash", "Read"}
|
||||
assert json.loads(tools["Bash"]["Input"])["command"] == tools["Bash"]["SpanAttributes"]["full_command"]
|
||||
assert "tool_input" not in tools["Bash"]["SpanAttributes"]
|
||||
root = next(r for r in rows if r["ObservationType"] == "agent")
|
||||
assert "user_prompt" not in root["SpanAttributes"]
|
||||
assert json.loads(root["Input"])[0]["role"] == "user"
|
||||
|
|
|
|||
|
|
@ -238,6 +238,29 @@ def test_trace_groups_normalized_names_and_preserves_span_labels():
|
|||
assert agents["researcher"]["llm_calls"] == 1
|
||||
|
||||
|
||||
def test_trace_frameworks_are_the_sorted_distinct_span_frameworks():
|
||||
rows = [
|
||||
_row("root", "", "claude_code.interaction", "agent", "claude-code", framework="claude-code"),
|
||||
_llm_row("llm", "root", "claude-code", "msg_1", framework="claude-agent-sdk"),
|
||||
_row("tool", "root", "Bash", "tool", "claude-code", framework="claude-code"),
|
||||
_row("other", "root", "step", "chain", "claude-code", framework=""),
|
||||
]
|
||||
trace = trace_from_rows("t1", rows)
|
||||
assert trace is not None
|
||||
assert trace["summary"]["frameworks"] == ("claude-agent-sdk", "claude-code")
|
||||
spans = {span["span_id"]: span for span in trace["spans"]}
|
||||
assert (spans["llm"]["framework"], spans["other"]["framework"]) == ("claude-agent-sdk", "")
|
||||
assert trace["agents"][0]["llm_calls"] == 1
|
||||
assert trace["agents"][0]["tool_calls"] == 1
|
||||
|
||||
|
||||
def test_spans_without_a_framework_column_report_none():
|
||||
trace = trace_from_rows("t1", _deep_agent_rows())
|
||||
assert trace is not None
|
||||
assert trace["summary"]["frameworks"] == ()
|
||||
assert {span["framework"] for span in trace["spans"]} == {""}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- list helpers
|
||||
|
||||
|
||||
|
|
@ -272,8 +295,10 @@ def test_trace_summary_from_row():
|
|||
"input_tokens": "30175",
|
||||
"output_tokens": "2620",
|
||||
"models": ["claude-sonnet-4-5"],
|
||||
"frameworks": ["claude-agent-sdk", "claude-code"],
|
||||
}
|
||||
)
|
||||
assert summary["frameworks"] == ("claude-agent-sdk", "claude-code")
|
||||
assert summary["status"] == "ok"
|
||||
assert (summary["span_count"], summary["error_count"]) == (126, 1)
|
||||
assert summary["start_time"] == "2026-09-30T04:36:29.377000+00:00"
|
||||
|
|
|
|||
|
|
@ -300,6 +300,30 @@ describe("AgentTracesSection", () => {
|
|||
expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length);
|
||||
});
|
||||
|
||||
it("shows each run's agent name with the logo of the SDK that produced it", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({
|
||||
...(traceList as TracePage),
|
||||
data: [
|
||||
{ ...runs[0], agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] },
|
||||
{ ...runs[1], agent_names: [], frameworks: ["claude-code"] },
|
||||
{ ...runs[2], frameworks: [] },
|
||||
],
|
||||
});
|
||||
renderSection();
|
||||
const [sdkRun, cliRun, plainRun] = await screen.findAllByTestId("agent-trace-row");
|
||||
const agentCell = (row: HTMLElement) => within(row).getAllByRole("cell")[1];
|
||||
|
||||
expect(agentCell(sdkRun)).toHaveTextContent(/^research-bot$/);
|
||||
expect(agentCell(sdkRun)).toHaveAttribute("title", "research-bot · Claude Agent SDK");
|
||||
expect(within(sdkRun).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toHaveAttribute(
|
||||
"src",
|
||||
expect.stringContaining("anthropic.svg"),
|
||||
);
|
||||
expect(agentCell(cliRun)).toHaveTextContent(/^Claude Code$/);
|
||||
expect(within(plainRun).queryByRole("img", { hidden: true })).not.toBeInTheDocument();
|
||||
expect(agentCell(plainRun)).toHaveTextContent((runs[2].agent_names ?? [runs[2].service]).join(", "));
|
||||
});
|
||||
|
||||
it("status filter 'Failed' keeps only runs with errors", () => {
|
||||
const failed = filterRuns(runs, "", "all", "error");
|
||||
expect(failed.length).toBeGreaterThan(0);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { formatActivityTimestamp } from "@/utils/activityTimestamp";
|
|||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import { StatusMark } from "./StatusMark";
|
||||
import { FrameworkLogo, traceFramework } from "./TraceFramework";
|
||||
import type { TraceSummary } from "./traceTypes";
|
||||
import { fmtMs, previewText, traceDisplayName, traceAgentNames } from "./traceUtils";
|
||||
|
||||
|
|
@ -32,6 +33,20 @@ const TH = "px-3 font-medium";
|
|||
const TH_NUM = "px-3 text-right font-medium";
|
||||
const TD_NUM = "px-3 text-right font-mono tabular-nums text-muted-foreground";
|
||||
|
||||
function AgentCell({ run }: { run: TraceSummary }) {
|
||||
const framework = traceFramework(run);
|
||||
const agents = traceAgentNames(run).join(", ");
|
||||
const title = [agents, framework?.label].filter(Boolean).join(" · ");
|
||||
return (
|
||||
<td className="px-3 text-muted-foreground" title={title}>
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
{framework && <FrameworkLogo framework={framework} />}
|
||||
<span className="truncate">{agents || framework?.label || "—"}</span>
|
||||
</div>
|
||||
</td>
|
||||
);
|
||||
}
|
||||
|
||||
/** Devtool-dense runs list: one row per agent run, newest first. */
|
||||
export function AgentTracesTable({
|
||||
traces,
|
||||
|
|
@ -83,9 +98,7 @@ export function AgentTracesTable({
|
|||
>
|
||||
{formatActivityTimestamp(run.start_time)}
|
||||
</td>
|
||||
<td className="truncate px-3 text-muted-foreground" title={traceAgentNames(run).join(", ")}>
|
||||
{traceAgentNames(run).join(", ") || "—"}
|
||||
</td>
|
||||
<AgentCell run={run} />
|
||||
<td className="px-3">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<StatusMark status={run.error_count > 0 ? "error" : "ok"} subtle />
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { screen } from "@testing-library/react";
|
||||
import { screen, within } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
|
|
@ -60,6 +60,27 @@ describe("RunView", () => {
|
|||
expect(header).not.toHaveTextContent("failed");
|
||||
});
|
||||
|
||||
it("shows the agent name with the SDK logo in the run header instead of the generic agent icon", async () => {
|
||||
renderRun({
|
||||
...research,
|
||||
summary: { ...research.summary, agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] },
|
||||
});
|
||||
|
||||
const header = await screen.findByRole("banner");
|
||||
expect(within(header).getByTestId("run-framework")).toHaveTextContent(/^research-bot$/);
|
||||
expect(within(header).getByTestId("run-framework")).toHaveAttribute("title", "Claude Agent SDK");
|
||||
expect(within(header).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toBeInTheDocument();
|
||||
expect(within(header).queryByTestId("span-icon")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the generic agent icon when the trace has no known SDK", async () => {
|
||||
renderRun({ ...research, summary: { ...research.summary, frameworks: ["some-other-sdk"] } });
|
||||
|
||||
const header = await screen.findByRole("banner");
|
||||
expect(within(header).getByTestId("span-icon")).toBeInTheDocument();
|
||||
expect(within(header).queryByTestId("run-framework")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("folds researcher ×12 in the span tree", async () => {
|
||||
renderRun(swarm);
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { IdChip } from "./IdChip";
|
|||
import { formatCost } from "./AgentTracesTable";
|
||||
import { SpanIcon } from "./SpanIcon";
|
||||
import { SpanTree } from "./SpanTree";
|
||||
import { FrameworkLogo, traceFramework } from "./TraceFramework";
|
||||
import type { SpanTreeState, TreeRow } from "./traceTree";
|
||||
import type { Trace } from "./traceTypes";
|
||||
import {
|
||||
|
|
@ -25,6 +26,7 @@ import {
|
|||
isFrameworkSpan,
|
||||
nearestVisibleSpanId,
|
||||
revealSpanInState,
|
||||
traceAgentNames,
|
||||
traceDisplayName,
|
||||
} from "./traceUtils";
|
||||
|
||||
|
|
@ -107,6 +109,21 @@ function Stat({ label, value, error = false }: { label: string; value: string; e
|
|||
);
|
||||
}
|
||||
|
||||
function RunIcon({ summary, failed }: { summary: Trace["summary"]; failed: boolean }) {
|
||||
const framework = traceFramework(summary);
|
||||
if (!framework) return <SpanIcon type="agent" error={failed} size="lg" />;
|
||||
return (
|
||||
<span
|
||||
className="inline-flex shrink-0 items-center gap-1 rounded-[5px] border border-border bg-card px-1.5 py-px text-[12px] text-foreground"
|
||||
data-testid="run-framework"
|
||||
title={framework.label}
|
||||
>
|
||||
<FrameworkLogo framework={framework} />
|
||||
{traceAgentNames(summary).join(", ") || framework.label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function RunHeader({ trace, onBack, embedded }: { trace: Trace; onBack: () => void; embedded: boolean }) {
|
||||
const { summary } = trace;
|
||||
const failed = summary.error_count > 0;
|
||||
|
|
@ -125,7 +142,7 @@ function RunHeader({ trace, onBack, embedded }: { trace: Trace; onBack: () => vo
|
|||
<span className="mx-1 h-[18px] w-px bg-border" />
|
||||
</>
|
||||
)}
|
||||
<SpanIcon type="agent" error={failed} size="lg" />
|
||||
<RunIcon summary={summary} failed={failed} />
|
||||
<h1 className="min-w-0 truncate text-[14px] font-medium text-foreground">{traceDisplayName(summary)}</h1>
|
||||
<IdChip value={summary.trace_id} label="Copy trace ID" showValue />
|
||||
<div className="flex min-w-0 flex-wrap items-center gap-1.5">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
import anthropicLogo from "../../../../public/assets/logos/anthropic.svg";
|
||||
import { Logo } from "@/components/molecules/logo/Logo";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import type { TraceSummary } from "./traceTypes";
|
||||
|
||||
export interface TraceFramework {
|
||||
readonly id: string;
|
||||
readonly label: string;
|
||||
readonly logo: string;
|
||||
}
|
||||
|
||||
const FRAMEWORKS: readonly TraceFramework[] = [
|
||||
{ id: "claude-agent-sdk", label: "Claude Agent SDK", logo: anthropicLogo.src },
|
||||
{ id: "claude-code", label: "Claude Code", logo: anthropicLogo.src },
|
||||
];
|
||||
|
||||
/**
|
||||
* The SDK that produced the trace. A Claude Agent SDK trace also carries "claude-code" (its root span has no
|
||||
* SDK marker), so registry order decides: the more specific framework wins.
|
||||
*/
|
||||
export const traceFramework = (summary: Pick<TraceSummary, "frameworks">): TraceFramework | null =>
|
||||
FRAMEWORKS.find((framework) => summary.frameworks?.includes(framework.id)) ?? null;
|
||||
|
||||
export function FrameworkLogo({ framework, className }: { framework: TraceFramework; className?: string }) {
|
||||
return (
|
||||
<span aria-hidden className="contents">
|
||||
<Logo src={framework.logo} label={framework.label} className={cn("size-3.5 shrink-0", className)} />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -44995,6 +44995,8 @@ export interface components {
|
|||
error: string | null;
|
||||
/** Error Truncated */
|
||||
error_truncated: boolean;
|
||||
/** Framework */
|
||||
framework: string;
|
||||
/** Input Preview */
|
||||
input_preview: string;
|
||||
/** Input Tokens */
|
||||
|
|
@ -46910,6 +46912,8 @@ export interface components {
|
|||
duration_ms: number;
|
||||
/** Error Count */
|
||||
error_count: number;
|
||||
/** Frameworks */
|
||||
frameworks?: string[];
|
||||
/** Input Preview */
|
||||
input_preview: string;
|
||||
/** Input Tokens */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue