mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(lens): preserve framework agent names and GenAI message content (#44218)
* fix(lens): use recorded agent identities across framework traces * fix(lens): tighten agent identity and bound trace lookups * style(tracing): wrap framework agent identity test case
This commit is contained in:
moe-berri
GitHub
parent
8aaa766734
commit
2584721ca3
14 changed files with 419 additions and 26 deletions
|
|
@ -1,11 +1,13 @@
|
|||
WITH page AS (
|
||||
SELECT TraceId AS trace_id,
|
||||
hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref,
|
||||
TeamId AS team_id, ApiKeyHash AS api_key_hash,
|
||||
ifNull(any(RootName), '') AS name, any(ServiceName) AS service,
|
||||
ifNull(any(RootInput), '') AS input_preview, ifNull(any(RootStatus), '') AS status,
|
||||
toUnixTimestamp64Milli(min(StartTs)) AS start_ms,
|
||||
min(StartTs) AS trace_start, max(EndTs) AS trace_end,
|
||||
dateDiff('millisecond', min(StartTs), max(EndTs)) AS duration_ms,
|
||||
sum(SpanCount) AS span_count, length(groupUniqArrayArray(AgentNames)) AS agent_count,
|
||||
sum(SpanCount) AS span_count,
|
||||
sum(AgentCount) AS agent_invocations,
|
||||
sum(LlmCount) AS llm_calls, sum(ToolCount) AS tool_calls,
|
||||
sum(InputTokens) AS input_tokens, sum(OutputTokens) AS output_tokens,
|
||||
|
|
@ -21,3 +23,21 @@ HAVING min(StartTs) >= fromUnixTimestamp64Milli({start_ms:Int64})
|
|||
< ({cursor_ms:Int64}, {cursor_trace_id:String}))
|
||||
ORDER BY start_ms DESC, trace_ref DESC
|
||||
LIMIT {limit:UInt32}
|
||||
)
|
||||
SELECT page.* EXCEPT (trace_start, trace_end),
|
||||
identities.agent_names AS agent_names, identities.agent_count AS agent_count
|
||||
FROM page
|
||||
LEFT JOIN (
|
||||
SELECT TeamId, ApiKeyHash, TraceId,
|
||||
arraySort(groupUniqArrayIf(AgentName, AgentName != '')) AS agent_names,
|
||||
uniqExactIf(if(AgentName = '', SpanName, AgentName), ObservationType = 'agent') AS agent_count
|
||||
FROM otel_traces
|
||||
WHERE Timestamp >= (SELECT min(trace_start) FROM page)
|
||||
AND Timestamp <= (SELECT max(trace_end) FROM page)
|
||||
AND TraceId IN (SELECT trace_id FROM page)
|
||||
AND (TeamId, ApiKeyHash, TraceId) IN (SELECT team_id, api_key_hash, trace_id FROM page)
|
||||
GROUP BY TeamId, ApiKeyHash, TraceId
|
||||
) AS identities
|
||||
ON page.team_id = identities.TeamId AND page.api_key_hash = identities.ApiKeyHash
|
||||
AND page.trace_id = identities.TraceId
|
||||
ORDER BY page.start_ms DESC, page.trace_ref DESC
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::DecodeError;
|
||||
use serde::Serialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
|
|
@ -148,6 +148,60 @@ fn usage_tokens(attributes: &BTreeMap<String, String>) -> Result<(u32, u32), Dec
|
|||
))
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
struct AgentMetadata {
|
||||
#[serde(default)]
|
||||
lc_agent_name: String,
|
||||
#[serde(default)]
|
||||
ls_integration: String,
|
||||
}
|
||||
|
||||
fn recorded_agent_name(
|
||||
name: &str,
|
||||
attributes: &BTreeMap<String, String>,
|
||||
span: &NormalizedSpan,
|
||||
) -> String {
|
||||
let explicit = [
|
||||
span.agent_name.as_str(),
|
||||
attr(attributes, "gen_ai.agent.name"),
|
||||
attr(attributes, "agent.name"),
|
||||
attr(attributes, "openclaw.agent"),
|
||||
]
|
||||
.into_iter()
|
||||
.find(|value| !value.is_empty());
|
||||
if let Some(value) = explicit {
|
||||
return value.to_owned();
|
||||
}
|
||||
let metadata =
|
||||
serde_json::from_str::<AgentMetadata>(attr(attributes, "metadata")).unwrap_or_default();
|
||||
if !metadata.lc_agent_name.is_empty() {
|
||||
return metadata.lc_agent_name;
|
||||
}
|
||||
if span.observation_type == ObservationType::Agent {
|
||||
let node = attr(attributes, "graph.node.id");
|
||||
if !node.is_empty() {
|
||||
return node.to_owned();
|
||||
}
|
||||
if metadata.ls_integration == "langgraph" && name != "LangGraph" && !is_middleware(name) {
|
||||
return name.to_owned();
|
||||
}
|
||||
}
|
||||
String::new()
|
||||
}
|
||||
|
||||
fn is_middleware(name: &str) -> bool {
|
||||
[
|
||||
".wrap_model_call",
|
||||
".wrap_tool_call",
|
||||
".before_agent",
|
||||
".after_agent",
|
||||
".before_model",
|
||||
".after_model",
|
||||
]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
}
|
||||
|
||||
pub fn normalize(
|
||||
scope_name: &str,
|
||||
name: &str,
|
||||
|
|
@ -163,8 +217,22 @@ pub fn normalize(
|
|||
.into_iter()
|
||||
.find(|normalizer| normalizer.matches(scope_name, attributes))
|
||||
.expect("GenAI fallback always matches");
|
||||
let span = normalizer.normalize(name, parent_span_id, attributes)?;
|
||||
let agent_name = recorded_agent_name(name, attributes, &span);
|
||||
let observation_type = if !parent_span_id.is_empty()
|
||||
&& scope_name == "openinference.instrumentation.langchain"
|
||||
&& is_middleware(name)
|
||||
{
|
||||
ObservationType::Framework
|
||||
} else {
|
||||
span.observation_type
|
||||
};
|
||||
Ok(Normalization {
|
||||
span: normalizer.normalize(name, parent_span_id, attributes)?,
|
||||
span: NormalizedSpan {
|
||||
agent_name,
|
||||
observation_type,
|
||||
..span
|
||||
},
|
||||
consumed_attributes: normalizer.consumed_attributes(attributes),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -133,7 +133,18 @@ fn decoded_span(
|
|||
&parent_span_id,
|
||||
&span_attributes,
|
||||
)?;
|
||||
let normalized = normalization.span;
|
||||
let resource_agent_name = resource_attributes
|
||||
.get("gen_ai.agent.name")
|
||||
.filter(|name| !name.is_empty());
|
||||
let agent_name = match (resource_agent_name, normalization.span.agent_name.as_str()) {
|
||||
(Some(name), "") => name.clone(),
|
||||
(Some(name), "hermes-agent") if scope_name.as_ref() == "hermes-otel-plugin" => name.clone(),
|
||||
(_, name) => name.to_owned(),
|
||||
};
|
||||
let normalized = crate::normalize::NormalizedSpan {
|
||||
agent_name,
|
||||
..normalization.span
|
||||
};
|
||||
budget.consume(
|
||||
normalized.input.len()
|
||||
+ normalized.output.len()
|
||||
|
|
|
|||
|
|
@ -362,6 +362,143 @@ async fn keyed_rollup_keeps_same_trace_ids_separate_by_api_key(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn listed_agent_names_preserve_scope_and_cursor(
|
||||
#[future(awt)] database: TestResult<ClickHouseDatabase>,
|
||||
) -> TestResult {
|
||||
let database = database?;
|
||||
let writer = Connection::writer(&database.url)?;
|
||||
ensure_schema(&database.client, &writer, "trace_test", 7).await?;
|
||||
let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
|
||||
for (team, key, trace, agent, span, parent) in [
|
||||
("alpha", "one", "shared", "research_agent", "root", ""),
|
||||
("alpha", "one", "shared", "reviewer", "child", "root"),
|
||||
("alpha", "one", "shared", "reviewer", "repeated", "root"),
|
||||
("alpha", "one", "shared", "", "unnamed", "root"),
|
||||
("alpha", "one", "second", "support_agent", "root", ""),
|
||||
("alpha", "two", "shared", "private_agent", "root", ""),
|
||||
("beta", "one", "shared", "other_agent", "root", ""),
|
||||
] {
|
||||
insert_rows(
|
||||
&database,
|
||||
"otel_traces",
|
||||
vec![serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent,
|
||||
"ServiceName": "shared-app", "SpanName": span, "AgentName": agent,
|
||||
"ObservationType": "agent",
|
||||
"ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": key}
|
||||
}))?],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let historical_rows = (0..5000)
|
||||
.map(|index| {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": timestamp - 86_400_000_000_000_i64,
|
||||
"TraceId": "shared", "SpanId": format!("historical-{index}"),
|
||||
"ParentSpanId": "", "SpanName": "historical", "AgentName": "private_agent",
|
||||
"ObservationType": "agent", "ServiceName": "shared-app",
|
||||
"ResourceAttributes": {"litellm.team_id": "alpha", "litellm.api_key_hash": "history"}
|
||||
}))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
insert_rows(&database, "otel_traces", historical_rows).await?;
|
||||
let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
|
||||
let parameters = BTreeMap::from([
|
||||
("team_ids".into(), Parameter::Strings(vec!["alpha".into()])),
|
||||
("api_key_hash".into(), Parameter::Text("one".into())),
|
||||
(
|
||||
"start_ms".into(),
|
||||
Parameter::Integer(timestamp / 1_000_000 - 1000),
|
||||
),
|
||||
(
|
||||
"end_ms".into(),
|
||||
Parameter::Integer(timestamp / 1_000_000 + 1000),
|
||||
),
|
||||
("cursor_ms".into(), Parameter::Integer(0)),
|
||||
("cursor_trace_id".into(), Parameter::Text(String::new())),
|
||||
("limit".into(), Parameter::Integer(1)),
|
||||
]);
|
||||
let first: serde_json::Value = serde_json::from_str(
|
||||
&execute_named_read(
|
||||
&database.client,
|
||||
&connection,
|
||||
ReadQuery::ListTraces,
|
||||
¶meters,
|
||||
)
|
||||
.await?,
|
||||
)?;
|
||||
let cursor = first["data"][0]["trace_ref"]
|
||||
.as_str()
|
||||
.ok_or("missing cursor")?;
|
||||
let next_parameters = parameters
|
||||
.into_iter()
|
||||
.chain([
|
||||
(
|
||||
"cursor_ms".into(),
|
||||
Parameter::Integer(timestamp / 1_000_000),
|
||||
),
|
||||
("cursor_trace_id".into(), Parameter::Text(cursor.into())),
|
||||
])
|
||||
.collect();
|
||||
let second: serde_json::Value = serde_json::from_str(
|
||||
&execute_named_read(
|
||||
&database.client,
|
||||
&connection,
|
||||
ReadQuery::ListTraces,
|
||||
&next_parameters,
|
||||
)
|
||||
.await?,
|
||||
)?;
|
||||
assert_eq!(
|
||||
first["data"].as_array().ok_or("missing first page")?.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
second["data"]
|
||||
.as_array()
|
||||
.ok_or("missing second page")?
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_ne!(first["data"][0]["trace_id"], second["data"][0]["trace_id"]);
|
||||
let names = [&first["data"][0], &second["data"][0]]
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
(
|
||||
row["trace_id"].as_str().unwrap(),
|
||||
row["agent_names"].clone(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(
|
||||
names["shared"],
|
||||
serde_json::json!(["research_agent", "reviewer"])
|
||||
);
|
||||
assert_eq!(names["second"], serde_json::json!(["support_agent"]));
|
||||
let counts = [&first["data"][0], &second["data"][0]]
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
(
|
||||
row["trace_id"].as_str().unwrap(),
|
||||
row["agent_count"].as_u64(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
assert_eq!(counts["shared"], Some(3));
|
||||
assert_eq!(counts["second"], Some(1));
|
||||
for page in [&first, &second] {
|
||||
assert!(
|
||||
page["statistics"]["rows_read"]
|
||||
.as_u64()
|
||||
.ok_or("missing read statistics")?
|
||||
< 5000
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rollup_merges_spans_across_days_without_losing_root_fields(
|
||||
|
|
@ -376,6 +513,7 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields(
|
|||
let root = serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": day_start - 1_000_000_000, "TraceId": "cross-day", "SpanId": "span-root",
|
||||
"ParentSpanId": "", "ServiceName": "proxy", "SpanName": "root", "Input": "root input",
|
||||
"AgentName": "lead", "ObservationType": "agent",
|
||||
"StatusCode": "STATUS_CODE_ERROR",
|
||||
"ResourceAttributes": {"litellm.team_id": "team-1"}
|
||||
}))?;
|
||||
|
|
@ -383,6 +521,7 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields(
|
|||
let child = serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": day_start + 1_000_000_000, "TraceId": "cross-day", "SpanId": "span-child",
|
||||
"ParentSpanId": "span-root", "ServiceName": "proxy", "SpanName": "child",
|
||||
"AgentName": "researcher", "ObservationType": "agent",
|
||||
"StatusCode": "STATUS_CODE_UNSET",
|
||||
"ResourceAttributes": {"litellm.team_id": "team-1"}
|
||||
}))?;
|
||||
|
|
@ -406,6 +545,33 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields(
|
|||
"RootStatus": "STATUS_CODE_ERROR", "SpanCount": 2
|
||||
}])
|
||||
);
|
||||
let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
|
||||
let parameters = BTreeMap::from([
|
||||
("team_ids".into(), Parameter::Strings(vec!["team-1".into()])),
|
||||
("api_key_hash".into(), Parameter::Text(String::new())),
|
||||
(
|
||||
"start_ms".into(),
|
||||
Parameter::Integer(day_start / 1_000_000 - 2000),
|
||||
),
|
||||
("end_ms".into(), Parameter::Integer(day_start / 1_000_000)),
|
||||
("cursor_ms".into(), Parameter::Integer(0)),
|
||||
("cursor_trace_id".into(), Parameter::Text(String::new())),
|
||||
("limit".into(), Parameter::Integer(10)),
|
||||
]);
|
||||
let listed: serde_json::Value = serde_json::from_str(
|
||||
&execute_named_read(
|
||||
&database.client,
|
||||
&connection,
|
||||
ReadQuery::ListTraces,
|
||||
¶meters,
|
||||
)
|
||||
.await?,
|
||||
)?;
|
||||
assert_eq!(
|
||||
listed["data"][0]["agent_names"],
|
||||
serde_json::json!(["lead", "researcher"])
|
||||
);
|
||||
assert_eq!(listed["data"][0]["agent_count"], 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ def trace_summary_from_row(row: dict[str, Any], spend_rows: Sequence[_SpendRow]
|
|||
trace_ref=row.get("trace_ref", ""),
|
||||
name=row["name"],
|
||||
service=row["service"],
|
||||
agent_names=tuple(row.get("agent_names") or ()),
|
||||
input_preview=row["input_preview"],
|
||||
start_time=_iso(int(row["start_ms"])),
|
||||
duration_ms=float(row["duration_ms"]),
|
||||
|
|
@ -169,8 +170,8 @@ def _parent_agent_of(span: Span, by_id: Mapping[str, Span]) -> str | None:
|
|||
if parent_id is None or parent_id not in by_id or parent_id == span["span_id"]:
|
||||
return None
|
||||
parent = by_id[parent_id]
|
||||
if parent["type"] == "agent" and parent["name"] != span["name"]:
|
||||
return parent["name"]
|
||||
if parent["type"] == "agent" and (parent["agent"] or parent["name"]) != (span["agent"] or span["name"]):
|
||||
return parent["agent"] or parent["name"]
|
||||
parent_id = parent["parent_span_id"]
|
||||
return None
|
||||
|
||||
|
|
@ -183,9 +184,9 @@ def agent_nodes(spans: Sequence[Span]) -> tuple[AgentNode, ...]:
|
|||
if span["type"] != "agent":
|
||||
continue
|
||||
node = agents.setdefault(
|
||||
span["name"],
|
||||
span["agent"] or span["name"],
|
||||
AgentNode(
|
||||
name=span["name"],
|
||||
name=span["agent"] or span["name"],
|
||||
parent_agent=_parent_agent_of(span, by_id),
|
||||
invocations=0,
|
||||
llm_calls=0,
|
||||
|
|
@ -250,6 +251,7 @@ def trace_from_rows(
|
|||
trace_ref=trace_ref,
|
||||
name=root["name"],
|
||||
service=rows[0]["service"],
|
||||
agent_names=tuple(sorted(frozenset(s["agent"] for s in spans if s["agent"]))),
|
||||
input_preview=root["input_preview"],
|
||||
start_time=_iso(trace_start_ns // NANOS_PER_MS),
|
||||
duration_ms=(trace_end_ns - trace_start_ns) / NANOS_PER_MS,
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ class TraceSummary(TypedDict):
|
|||
trace_ref: ReadOnly[NotRequired[str]]
|
||||
name: ReadOnly[str]
|
||||
service: ReadOnly[str]
|
||||
agent_names: ReadOnly[NotRequired[tuple[str, ...]]]
|
||||
input_preview: ReadOnly[str]
|
||||
start_time: ReadOnly[str] # ISO 8601
|
||||
duration_ms: ReadOnly[float]
|
||||
|
|
|
|||
|
|
@ -55,13 +55,94 @@ def _kv(key: str, value: str | int) -> KeyValue:
|
|||
return KeyValue(key=key, value=AnyValue(string_value=value))
|
||||
|
||||
|
||||
def _export(*spans: Span, service: str = "svc", scope: str = "test") -> bytes:
|
||||
def _export(*spans: Span, service: str = "svc", scope: str = "test", agent_name: str = "") -> bytes:
|
||||
resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=list(spans))])
|
||||
resource_spans.resource.attributes.append(_kv("service.name", service))
|
||||
if agent_name:
|
||||
resource_spans.resource.attributes.append(_kv("gen_ai.agent.name", agent_name))
|
||||
resource_spans.scope_spans[0].scope.name = scope
|
||||
return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "attributes"),
|
||||
[
|
||||
("research_agent", {"openinference.span.kind": "AGENT", "metadata": '{"lc_agent_name":"research_agent"}'}),
|
||||
("research_agent", {"openinference.span.kind": "AGENT", "metadata": '{"ls_integration":"langgraph"}'}),
|
||||
("research_agent._execute_core", {"openinference.span.kind": "AGENT", "graph.node.id": "research_agent"}),
|
||||
("agent", {"openinference.span.kind": "AGENT", "gen_ai.agent.name": "research_agent"}),
|
||||
("openclaw.harness.run", {"openclaw.agent": "research_agent"}),
|
||||
(
|
||||
"invoke_agent research_agent",
|
||||
{"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "research_agent"},
|
||||
),
|
||||
],
|
||||
ids=["deepagents", "langgraph", "crewai", "hermes", "openclaw", "genai"],
|
||||
)
|
||||
def test_framework_agent_identity_is_independent_of_service(name: str, attributes: dict[str, str]):
|
||||
span = _span(name, b"\x02" * 8, **attributes)
|
||||
row = decode_otlp(_export(span, service="shared-deployment"), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == "research_agent"
|
||||
assert row["ServiceName"] == "shared-deployment"
|
||||
assert row["SpanName"] == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["ClaudeAgentSDK.query", "FunctionAgent.run"])
|
||||
def test_resource_agent_name_labels_instrumentors_without_an_agent_attribute(name: str):
|
||||
span = _span(name, b"\x02" * 8, openinference__span__kind="AGENT")
|
||||
row = decode_otlp(_export(span, agent_name="research_agent"), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == "research_agent"
|
||||
|
||||
|
||||
def test_span_agent_name_takes_precedence_over_resource_default():
|
||||
span = _span("invoke_agent child", b"\x02" * 8, gen_ai__agent__name="child")
|
||||
row = decode_otlp(_export(span, agent_name="research_agent"), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == "child"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("scope", "span_name", "configured_name", "expected"),
|
||||
[
|
||||
("hermes-otel-plugin", "hermes-agent", "research_agent", "research_agent"),
|
||||
("hermes-otel-plugin", "child", "research_agent", "child"),
|
||||
("hermes-otel-plugin", "hermes-agent", "", "hermes-agent"),
|
||||
("other-plugin", "hermes-agent", "research_agent", "hermes-agent"),
|
||||
],
|
||||
)
|
||||
def test_hermes_resource_name_replaces_only_its_plugin_default(
|
||||
scope: str, span_name: str, configured_name: str, expected: str
|
||||
):
|
||||
span = _span("agent", b"\x02" * 8, gen_ai__agent__name=span_name)
|
||||
row = decode_otlp(_export(span, scope=scope, agent_name=configured_name), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("agent_name", ["research_agent", ""])
|
||||
def test_openinference_middleware_is_not_a_separate_agent(agent_name: str):
|
||||
span = _span(
|
||||
"PatchToolCallsMiddleware.before_agent", b"\x02" * 8, b"\x01" * 8,
|
||||
openinference__span__kind="AGENT", metadata=json.dumps({"lc_agent_name": agent_name}),
|
||||
)
|
||||
row = decode_otlp(_export(span, scope="openinference.instrumentation.langchain"), "application/x-protobuf")[0]
|
||||
assert (row["ObservationType"], row["AgentName"]) == ("framework", agent_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", ["test", "openinference.instrumentation.langchain"])
|
||||
@pytest.mark.parametrize("kind", ["CHAIN", "AGENT"])
|
||||
@pytest.mark.parametrize("metadata", ["not json", "[]", '{"lc_agent_name":null}', "{}"])
|
||||
def test_unnamed_framework_does_not_invent_an_agent_from_service(metadata: str, scope: str, kind: str):
|
||||
span = _span("workflow", b"\x02" * 8, openinference__span__kind=kind, metadata=metadata)
|
||||
row = decode_otlp(_export(span, scope=scope), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,expected", [("support", "support"), ("LangGraph", "")])
|
||||
def test_langgraph_distinguishes_configured_graph_name_from_default(name: str, expected: str):
|
||||
span = _span(name, b"\x02" * 8, openinference__span__kind="CHAIN", metadata='{"ls_integration":"langgraph"}')
|
||||
row = decode_otlp(_export(span, scope="openinference.instrumentation.langchain"), "application/x-protobuf")[0]
|
||||
assert row["AgentName"] == expected
|
||||
|
||||
|
||||
def _span(name: str, span_id: bytes, parent: bytes = b"", **attributes: str | int) -> Span:
|
||||
return Span(
|
||||
trace_id=bytes.fromhex(TRACE_ID),
|
||||
|
|
|
|||
|
|
@ -221,6 +221,23 @@ def test_agent_nodes_ignores_spans_of_unknown_agents():
|
|||
assert agent_nodes(spans) == ()
|
||||
|
||||
|
||||
def test_trace_groups_normalized_names_and_preserves_span_labels():
|
||||
rows = [
|
||||
_row("root", "", "invoke_agent research_agent", "agent", "research_agent"),
|
||||
_row("r1", "root", "researcher._execute_core", "agent", "researcher"),
|
||||
_row("r2", "r1", "invoke_agent researcher", "agent", "researcher"),
|
||||
_row("llm", "r2", "chat", "llm", "researcher"),
|
||||
]
|
||||
result = trace_from_rows("t1", rows)
|
||||
assert result is not None
|
||||
assert result["summary"]["agent_names"] == ("research_agent", "researcher")
|
||||
assert result["summary"]["name"] == "invoke_agent research_agent"
|
||||
agents = {agent["name"]: agent for agent in result["agents"]}
|
||||
assert agents["researcher"]["parent_agent"] == "research_agent"
|
||||
assert agents["researcher"]["invocations"] == 2
|
||||
assert agents["researcher"]["llm_calls"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- list helpers
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -271,10 +271,13 @@ describe("AgentTracesSection", () => {
|
|||
expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans");
|
||||
});
|
||||
|
||||
it("labels the OTEL service as the agent and filters runs by it", async () => {
|
||||
it("uses recorded agent names for the column and filter even when services are shared", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({
|
||||
...(traceList as TracePage),
|
||||
data: [...runs.slice(1), { ...runs[0], service: "billing-agent" }],
|
||||
data: [
|
||||
...runs.slice(1).map((run) => ({ ...run, service: "shared-app", agent_names: ["research-agent"] })),
|
||||
{ ...runs[0], service: "shared-app", agent_names: ["billing-agent", "review-agent"] },
|
||||
],
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
renderSection();
|
||||
|
|
@ -288,6 +291,10 @@ describe("AgentTracesSection", () => {
|
|||
const rows = screen.getAllByTestId("agent-trace-row");
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toHaveTextContent("billing-agent");
|
||||
expect(rows[0]).not.toHaveTextContent("shared-app");
|
||||
|
||||
await chooseSelectOption(user, agentFilter, "review-agent");
|
||||
expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1);
|
||||
|
||||
await chooseSelectOption(user, agentFilter, "All agents");
|
||||
expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { AgentTracesTable } from "./AgentTracesTable";
|
|||
import { RunDrawer } from "./RunDrawer";
|
||||
import { ALL_AGENTS, RunsToolbar, type RunStatusFilter } from "./RunsToolbar";
|
||||
import type { TraceSummary } from "./traceTypes";
|
||||
import { previewText } from "./traceUtils";
|
||||
import { previewText, traceAgentNames } from "./traceUtils";
|
||||
import { TimeRangeControls } from "./TimeRangeControls";
|
||||
import { TracesTimeline, type TimeWindow } from "./TracesTimeline";
|
||||
import { ActiveDot } from "./ActiveDot";
|
||||
|
|
@ -27,7 +27,7 @@ export function filterRuns(
|
|||
return runs.filter((run) => {
|
||||
const haystack = [run.trace_id, previewText(run.input_preview), run.name].map((s) => s.toLowerCase());
|
||||
const matchesQuery = !q || haystack.some((text) => text.includes(q));
|
||||
const matchesAgent = agent === ALL_AGENTS || run.service === agent;
|
||||
const matchesAgent = agent === ALL_AGENTS || traceAgentNames(run).includes(agent);
|
||||
const failed = run.error_count > 0;
|
||||
const matchesStatus = status === "all" || (status === "error" ? failed : !failed);
|
||||
return matchesQuery && matchesAgent && matchesStatus;
|
||||
|
|
@ -121,7 +121,7 @@ export function AgentTracesSection({
|
|||
if (setup.disabledDetail == null) void history.refetch();
|
||||
};
|
||||
|
||||
const agents = useMemo(() => Array.from(new Set(traces.traces.map((t) => t.service))).sort(), [traces.traces]);
|
||||
const agents = useMemo(() => Array.from(new Set(traces.traces.flatMap(traceAgentNames))).sort(), [traces.traces]);
|
||||
// Relative ranges end "now" (the list query uses Date.now() too); round to the minute so the histogram is stable.
|
||||
const endMs = isCustomDate ? moment(endTime).valueOf() : moment().endOf("minute").valueOf();
|
||||
const range = useMemo(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { cn } from "@/lib/cva.config";
|
|||
|
||||
import { StatusMark } from "./StatusMark";
|
||||
import type { TraceSummary } from "./traceTypes";
|
||||
import { fmtMs, previewText, traceDisplayName } from "./traceUtils";
|
||||
import { fmtMs, previewText, traceDisplayName, traceAgentNames } from "./traceUtils";
|
||||
|
||||
interface AgentTracesTableProps {
|
||||
traces: TraceSummary[];
|
||||
|
|
@ -83,8 +83,8 @@ export function AgentTracesTable({
|
|||
>
|
||||
{formatActivityTimestamp(run.start_time)}
|
||||
</td>
|
||||
<td className="truncate px-3 text-muted-foreground" title={run.service}>
|
||||
{run.service}
|
||||
<td className="truncate px-3 text-muted-foreground" title={traceAgentNames(run).join(", ")}>
|
||||
{traceAgentNames(run).join(", ") || "—"}
|
||||
</td>
|
||||
<td className="px-3">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
|
|
|
|||
|
|
@ -287,6 +287,19 @@ describe("payload helpers", () => {
|
|||
expect(messageText(image)).toBe(image);
|
||||
expect(messageText("[not json")).toBe("[not json");
|
||||
});
|
||||
|
||||
it("reads GenAI message parts and native content arrays without crashing previews", () => {
|
||||
const question = "What is an agent trace?";
|
||||
const parts = [{ type: "text", content: question }];
|
||||
const input = JSON.stringify([{ role: "user", parts }]);
|
||||
expect(parseMessages(input)).toEqual([{ role: "user", parts, content: question }]);
|
||||
expect(previewText(input)).toBe(question);
|
||||
expect(
|
||||
parseMessages(JSON.stringify({ role: "assistant", content: [{ type: "text", text: "An execution record" }] })),
|
||||
).toEqual([{ role: "assistant", content: "An execution record" }]);
|
||||
expect(parseMessages('[{"role":"assistant","tool_calls":[]}]')).toBeNull();
|
||||
expect(parseMessages('[{"role":"user","content":42}]')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("treeGuides", () => {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@ import type { Span, TraceMessage, TraceSummary } from "./traceTypes";
|
|||
/* Formatting */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export const traceAgentNames = (trace: TraceSummary): readonly string[] =>
|
||||
trace.agent_names ?? (trace.service ? [trace.service] : []);
|
||||
|
||||
export const fmtMs = (ms: number): string => {
|
||||
if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
|
||||
if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
|
||||
|
|
@ -267,14 +270,10 @@ export const parseJson = (value: string): unknown => {
|
|||
}
|
||||
};
|
||||
|
||||
const isMessage = (value: unknown): value is TraceMessage => {
|
||||
const isObject = typeof value === "object" && value !== null;
|
||||
return isObject && "role" in value && typeof (value as TraceMessage).role === "string";
|
||||
};
|
||||
|
||||
const blockText = (block: unknown): string | null => {
|
||||
if (typeof block !== "object" || block === null) return null;
|
||||
const text: unknown = Reflect.get(block, "text");
|
||||
const text: unknown =
|
||||
Reflect.get(block, "text") ?? (Reflect.get(block, "type") === "text" ? Reflect.get(block, "content") : undefined);
|
||||
return typeof text === "string" ? text : null;
|
||||
};
|
||||
|
||||
|
|
@ -296,13 +295,19 @@ export function messageText(content: string): string {
|
|||
.join("\n\n");
|
||||
}
|
||||
|
||||
const withText = (message: TraceMessage): TraceMessage => ({ ...message, content: messageText(message.content) });
|
||||
const parseMessage = (value: unknown): TraceMessage | null => {
|
||||
if (typeof value !== "object" || value === null) return null;
|
||||
const role: unknown = Reflect.get(value, "role");
|
||||
const content: unknown = Reflect.get(value, "content") ?? Reflect.get(value, "parts");
|
||||
if (typeof role !== "string" || (typeof content !== "string" && !Array.isArray(content))) return null;
|
||||
return { ...value, role, content: messageText(typeof content === "string" ? content : JSON.stringify(content)) };
|
||||
};
|
||||
|
||||
/** An llm span's input (array of messages) or output (one message); null when it isn't one. */
|
||||
export function parseMessages(value: string): TraceMessage[] | null {
|
||||
const parsed = parseJson(value);
|
||||
if (Array.isArray(parsed)) return parsed.every(isMessage) ? parsed.map(withText) : null;
|
||||
return isMessage(parsed) ? [withText(parsed)] : null;
|
||||
const messages = (Array.isArray(parsed) ? parsed : [parsed]).map(parseMessage);
|
||||
return messages.every((message) => message !== null) ? messages : null;
|
||||
}
|
||||
|
||||
/** Pretty JSON when the payload is JSON, else the raw string. */
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -46904,6 +46904,8 @@ export interface components {
|
|||
agent_count: number;
|
||||
/** Agent Invocations */
|
||||
agent_invocations: number;
|
||||
/** Agent Names */
|
||||
agent_names?: string[];
|
||||
/** Duration Ms */
|
||||
duration_ms: number;
|
||||
/** Error Count */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue