From a9b97007900f241994eeaf8582e7fde6bc557d1f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:50:35 -0700 Subject: [PATCH 1/6] perf(lens): bound single trace reads by the sampled start time (#45088) * perf(lens): bound single trace reads by the sampled start time Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): allow unused query fixture field in load tests Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): pass start_time in every lens content and evidence test Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../traces-clickhouse/query/lens_content.sql | 2 + .../traces-clickhouse/query/lens_evidence.sql | 2 + .../traces-clickhouse/src/query/lens.rs | 2 + .../traces-clickhouse/src/query/number.rs | 2 +- .../crates/traces-clickhouse/tests/load.rs | 83 ++++++++++++++++++- .../traces-clickhouse/tests/migrations.rs | 30 ++++++- litellm/proxy/lens/sources.py | 2 + litellm/rust_bridge/trace/generated/models.py | 2 + .../traces-clickhouse/LensContentParams.json | 4 + .../traces-clickhouse/LensEvidenceParams.json | 4 + tests/unit/proxy/lens/test_sources.py | 17 +++- tests/unit/rust_bridge/trace/test_queries.py | 2 + 12 files changed, 148 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_content.sql b/litellm-rust/crates/traces-clickhouse/query/lens_content.sql index 99fb56a5f48..52355a11061 100644 --- a/litellm-rust/crates/traces-clickhouse/query/lens_content.sql +++ b/litellm-rust/crates/traces-clickhouse/query/lens_content.sql @@ -17,6 +17,7 @@ SELECT * FROM ( FROM otel_traces WHERE {source:String}='traces' AND ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String}) AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId > {cursor:String} ORDER BY SpanId LIMIT 1 BY SpanId LIMIT 40 @@ -35,5 +36,6 @@ SELECT * FROM ( FROM spend_logs FINAL WHERE {source:String}='requests' AND ({all_teams:UInt8}=1 OR team_id={team:String}) AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY AND request_id={id:String} AND team_id={record_team:String} LIMIT 1 ) diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql b/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql index a0d600cdfde..b53617364cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql +++ b/litellm-rust/crates/traces-clickhouse/query/lens_evidence.sql @@ -2,6 +2,7 @@ SELECT sum(matches) AS count FROM ( SELECT count() AS matches FROM otel_traces WHERE {source:String}='traces' AND ({all_teams:UInt8}=1 OR TeamId={team:String}) AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND Timestamp >= parseDateTime64BestEffortOrZero({start_time:String}, 9) - INTERVAL 7 DAY AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String}) AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId={span:String} AND position(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),{quote:String})>0 @@ -9,6 +10,7 @@ SELECT sum(matches) AS count FROM ( SELECT count() AS matches FROM spend_logs FINAL WHERE {source:String}='requests' AND ({all_teams:UInt8}=1 OR team_id={team:String}) AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND spend_logs.start_time >= parseDateTime64BestEffortOrZero({start_time:String}, 3) - INTERVAL 7 DAY AND request_id={id:String} AND team_id={record_team:String} AND request_id={span:String} AND position(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),{quote:String})>0 ) diff --git a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs index 6439e696dad..77474c7143d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs @@ -209,6 +209,7 @@ pub struct LensContentParams { pub source: ContentSource, pub id: String, pub record_team: String, + pub start_time: String, pub trace_ref: String, pub cursor: String, #[serde(deserialize_with = "super::number::deserialize")] @@ -252,6 +253,7 @@ pub struct LensEvidenceParams { pub source: ContentSource, pub id: String, pub record_team: String, + pub start_time: String, pub trace_ref: String, pub span: String, pub quote: String, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/number.rs b/litellm-rust/crates/traces-clickhouse/src/query/number.rs index 9283903fee1..7a07845d0cd 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/number.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/number.rs @@ -99,7 +99,7 @@ mod tests { fn content_rejects_unsupported_sources(#[case] source: &str, #[case] valid: bool) { let parameters = serde_json::json!({ "all_teams": 0, "team": "team", "key_hash": "", "source": source, "id": "id", - "record_team": "team", "trace_ref": "", "cursor": "", "offset": 0 + "record_team": "team", "start_time": "", "trace_ref": "", "cursor": "", "offset": 0 }); assert_eq!( serde_json::from_value::(parameters).is_ok(), diff --git a/litellm-rust/crates/traces-clickhouse/tests/load.rs b/litellm-rust/crates/traces-clickhouse/tests/load.rs index 07c1095dfc3..aaec2c17e54 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/load.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/load.rs @@ -25,7 +25,8 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR "INSERT INTO {DATABASE}.otel_traces \ (Timestamp, TraceId, SpanId, ParentSpanId, SpanName, ServiceName, ObservationType, TeamId, ApiKeyHash, Duration, SpanAttributes) \ SELECT now64(9) - toIntervalHour(intDiv(number, {SPANS_PER_DAY}) * 24 + 12 + {first_day} * 24), \ - concat('load-', toString(number + {first_row})), concat('span-', toString(number + {first_row})), \ + if({first_day} = 0, concat('load-', toString(number + {first_row})), 'load-0'), \ + concat('span-', toString(number + {first_row})), \ '', 'span', 'service', 'agent', 'load-team', '', 0, \ if({first_day}=0 AND number < {SPANS_PER_DAY}, map('payload', repeat('x', 3000)), map()) \ FROM numbers({count})" @@ -41,6 +42,62 @@ async fn seed_days(fixture: &SeededDatabase, first_day: u64, days: u64) -> TestR Ok(()) } +async fn trace_start_time(fixture: &SeededDatabase) -> TestResult { + let query = format!( + "SELECT toString(Timestamp, 'UTC') AS start_time FROM {DATABASE}.otel_traces \ + WHERE TraceId = 'load-0' LIMIT 1 FORMAT JSON" + ); + let response = fixture + .database + .client + .post(&fixture.database.url) + .body(query) + .send() + .await? + .error_for_status()? + .text() + .await?; + let result: Value = serde_json::from_str(&response)?; + result["data"][0]["start_time"] + .as_str() + .map(str::to_owned) + .ok_or_else(|| "trace start time missing".into()) +} + +fn content_parameters(start_time: &str) -> BTreeMap { + BTreeMap::from([ + ("source".into(), Parameter::Text("traces".into())), + ("all_teams".into(), Parameter::Integer(0)), + ("team".into(), Parameter::Text("load-team".into())), + ("key_hash".into(), Parameter::Text(String::new())), + ("id".into(), Parameter::Text("load-0".into())), + ("record_team".into(), Parameter::Text("load-team".into())), + ("start_time".into(), Parameter::Text(start_time.into())), + ("trace_ref".into(), Parameter::Text(String::new())), + ("cursor".into(), Parameter::Text(String::new())), + ("offset".into(), Parameter::Integer(1)), + ]) +} + +async fn content(fixture: &SeededDatabase, start_time: &str, query_id: &str) -> TestResult { + let connection = Connection::configured( + &format!("{}?query_id={query_id}", fixture.database.url), + DATABASE, + "default", + "", + )?; + let response = execute_named_read( + &fixture.database.client, + &connection, + ReadQuery::Content, + &content_parameters(start_time), + ) + .await?; + let result: Value = serde_json::from_str(&response)?; + assert!(!result["data"].as_array().ok_or("content rows")?.is_empty()); + Ok(()) +} + fn sample_parameters(start: u64, end: u64) -> BTreeMap { BTreeMap::from([ ("source".into(), Parameter::Text("traces".into())), @@ -144,3 +201,27 @@ async fn lens_sample_reads_scale_with_window_not_retention( ); Ok(()) } + +#[rstest] +#[tokio::test] +async fn lens_content_reads_scale_with_trace_not_retention( + #[future(awt)] migrated_database: TestResult, +) -> TestResult { + let fixture = migrated_database?; + seed_days(&fixture, 0, 8).await?; + let start_time = trace_start_time(&fixture).await?; + let before_id = format!("lens_content_before_{}", std::process::id()); + content(&fixture, &start_time, &before_id).await?; + let before = query_read_rows(&fixture, &before_id).await?; + + seed_days(&fixture, 8, 24).await?; + let after_id = format!("lens_content_after_{}", std::process::id()); + content(&fixture, &start_time, &after_id).await?; + let after = query_read_rows(&fixture, &after_id).await?; + println!("lens_content read_rows: before={before}, after={after}"); + assert!( + after * 100 <= before * 105, + "read_rows grew from {before} to {after}" + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index e51d9083c59..7630b3033a7 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -1150,6 +1150,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( ("source".into(), Parameter::Text("traces".into())), ("id".into(), Parameter::Text("shared".into())), ("record_team".into(), Parameter::Text("team".into())), + ("start_time".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(first_ref.into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), @@ -1177,6 +1178,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( ("source".into(), Parameter::Text("traces".into())), ("id".into(), Parameter::Text("shared".into())), ("record_team".into(), Parameter::Text("team".into())), + ("start_time".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(first_ref.into())), ("span".into(), Parameter::Text("root".into())), ("quote".into(), Parameter::Text(opposite.into())), @@ -1339,7 +1341,7 @@ async fn lens_selection_pages_without_losing_or_repeating_runs( #[case::traces("traces", 9)] #[case::requests("requests", 3)] #[tokio::test] -async fn lens_content_keeps_original_span_and_request_timestamps( +async fn lens_content_keeps_original_timestamps_with_start_time_slack( #[future(awt)] database: TestResult, #[case] source: &str, #[case] precision: usize, @@ -1379,6 +1381,30 @@ async fn lens_content_keeps_original_span_and_request_timestamps( ) .await?; let connection = Connection::configured(&database.url, "trace_test", "default", "")?; + let start_time_body = execute_read( + &database.client, + &connection, + "SELECT toString(fromUnixTimestamp64Nano({timestamp:Int64})) AS start_time FORMAT JSON", + &BTreeMap::from([( + "timestamp".into(), + Parameter::Integer(root_start + 86_400_000_000_000), + )]), + ) + .await?; + let start_time: serde_json::Value = serde_json::from_str(&start_time_body)?; + let start_time = start_time["data"][0]["start_time"] + .as_str() + .ok_or("start time missing")? + .to_owned(); + let parsed_time_body = execute_read( + &database.client, + &connection, + "SELECT toString(parseDateTime64BestEffortOrZero({start_time:String}, 9)) AS start_time FORMAT JSON", + &BTreeMap::from([("start_time".into(), Parameter::Text(start_time.clone()))]), + ) + .await?; + let parsed_time: serde_json::Value = serde_json::from_str(&parsed_time_body)?; + assert_eq!(parsed_time["data"][0]["start_time"], start_time); let parameters = BTreeMap::from([ ("source".into(), Parameter::Text(source.into())), ("all_teams".into(), Parameter::Integer(0)), @@ -1386,6 +1412,7 @@ async fn lens_content_keeps_original_span_and_request_timestamps( ("record_team".into(), Parameter::Text("team".into())), ("key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), + ("start_time".into(), Parameter::Text(start_time)), ("id".into(), Parameter::Text("run".into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), @@ -1453,6 +1480,7 @@ async fn lens_content_keeps_output_visible_after_long_input( ("record_team".into(), Parameter::Text("team".into())), ("key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), + ("start_time".into(), Parameter::Text(String::new())), ("id".into(), Parameter::Text("request".into())), ("cursor".into(), Parameter::Text(String::new())), ("offset".into(), Parameter::Integer(1)), diff --git a/litellm/proxy/lens/sources.py b/litellm/proxy/lens/sources.py index 015be69ecca..b1d142a6748 100644 --- a/litellm/proxy/lens/sources.py +++ b/litellm/proxy/lens/sources.py @@ -142,6 +142,7 @@ class SourceReader: id=execution.trace_id, trace_ref=execution.trace_ref, record_team=execution.team_id, + start_time=execution.start_time, cursor=cursor, offset=offset + 1, ) @@ -175,6 +176,7 @@ class SourceReader: id=execution.trace_id, trace_ref=execution.trace_ref, record_team=execution.team_id, + start_time=execution.start_time, span=evidence.span_id, quote=evidence.quote, ) diff --git a/litellm/rust_bridge/trace/generated/models.py b/litellm/rust_bridge/trace/generated/models.py index ea2c8bda648..5d84003aba2 100644 --- a/litellm/rust_bridge/trace/generated/models.py +++ b/litellm/rust_bridge/trace/generated/models.py @@ -215,6 +215,7 @@ class LensContentParams(LiteLLMBaseModel): source: ContentSource id: str record_team: str + start_time: str trace_ref: str cursor: str offset: int = Field(..., ge=0, le=4294967295) @@ -232,6 +233,7 @@ class LensEvidenceParams(LiteLLMBaseModel): source: ContentSource id: str record_team: str + start_time: str trace_ref: str span: str quote: str diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json index 5ee5ab558ce..6026ccd26e1 100644 --- a/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensContentParams.json @@ -39,6 +39,9 @@ "source": { "$ref": "#/$defs/ContentSource" }, + "start_time": { + "type": "string" + }, "team": { "type": "string" }, @@ -53,6 +56,7 @@ "source", "id", "record_team", + "start_time", "trace_ref", "cursor", "offset" diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json index 07b9c216083..dbe9b32fdd6 100644 --- a/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensEvidenceParams.json @@ -36,6 +36,9 @@ "span": { "type": "string" }, + "start_time": { + "type": "string" + }, "team": { "type": "string" }, @@ -50,6 +53,7 @@ "source", "id", "record_team", + "start_time", "trace_ref", "span", "quote" diff --git a/tests/unit/proxy/lens/test_sources.py b/tests/unit/proxy/lens/test_sources.py index 8512db04eff..ad1f90f97ce 100644 --- a/tests/unit/proxy/lens/test_sources.py +++ b/tests/unit/proxy/lens/test_sources.py @@ -10,8 +10,10 @@ from litellm.proxy.lens.sources import SourceReader, execution_id, parse_executi from litellm.rust_bridge.trace.generated.models import ( ActivityAvailability, AgentRow, + CountRow, ExecutionRow, LensContentParams, + LensEvidenceParams, PartRow, ) from tests.unit.proxy.lens.test_agent_workspace import python_data @@ -183,9 +185,17 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python( class ContentStorage: async def lens_content(self, parameters: LensContentParams) -> tuple[PartRow, ...]: - assert parameters.source == source and parameters.record_team == "team" + assert ( + parameters.source == source + and parameters.record_team == "team" + and parameters.start_time == run.start_time + ) return rows + async def lens_evidence(self, parameters: LensEvidenceParams) -> tuple[CountRow, ...]: + assert parameters.start_time == run.start_time + return (CountRow(count=1),) + reader: Final = SourceReader(ContentStorage()) async def read(identity: str, cursor: str, offset: int) -> ExecutionContent: @@ -217,6 +227,11 @@ async def test_recorded_times_survive_source_catalog_reads_search_and_python( assert computed.sessions[0].parts == expected assert min(computed.sessions[0].parts, key=lambda part: part.start_time).span_id == rows[-1].span_id assert await workspace.valid(Evidence(execution_id=run.id, span_id=rows[0].span_id, quote=rows[0].content)) + assert await reader.verify_evidence( + Scope(team_id="team"), + run, + Evidence(execution_id=run.id, span_id=rows[0].span_id, quote=rows[0].content), + ) @pytest.mark.asyncio diff --git a/tests/unit/rust_bridge/trace/test_queries.py b/tests/unit/rust_bridge/trace/test_queries.py index 3c0556b1d97..9fd2964af20 100644 --- a/tests/unit/rust_bridge/trace/test_queries.py +++ b/tests/unit/rust_bridge/trace/test_queries.py @@ -18,6 +18,7 @@ def test_named_query_rejects_offsets_outside_the_native_integer_range(offset: in "source": "traces", "id": "trace", "record_team": "team", + "start_time": "", "trace_ref": "ref", "cursor": "", "offset": offset, @@ -34,6 +35,7 @@ def test_named_query_rejects_parameters_for_a_different_query() -> None: source="traces", id="trace", record_team="team", + start_time="", trace_ref="ref", cursor="", offset=0, From 7921716f3928ebf2d33b71012576efe0d26c31fa Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:57:04 -0700 Subject: [PATCH 2/6] feat(lens-ui): show findings ranked by priority with frequency and highlighted evidence (#45143) * feat(lens-ui): compute how often a finding hits sampled traces per day Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): copy a finding for an agent as markdown Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add affected, unaffected and quote highlight color tokens Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a frequency card with stacked affected traces per day Co-Authored-By: Claude Opus 5.5 * fix(lens-ui): keep the issue brief title out of the page heading outline Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): lay out a finding as summary, fix, frequency and highlighted examples Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): show findings as a dated list with percent affected beside the open finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover frequency and highlighted quotes on a finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): follow findings into the split list and example cards Co-Authored-By: Claude Opus 5.5 * style(lens-ui): take finding chart and quote colors from the dashboard theme Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a shared priority dot and pill for findings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): soften the frequency card and show its date range Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): rank findings under high, medium and low priority headings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): show finding priority, label quotes by content and collapse extra examples Co-Authored-By: Claude Opus 5.5 * test(lens-ui): prove findings are grouped and ordered by priority Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover finding priority, quote labels and example collapsing Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ui/litellm-dashboard/src/app/globals.css | 12 + .../lens/LensWorkspace.integration.test.tsx | 7 +- .../FindingDetails.integration.test.tsx | 76 +++- .../lens/investigations/FindingDetails.tsx | 423 +++++++++++++----- .../FindingsView.integration.test.tsx | 47 +- .../lens/investigations/FindingsView.tsx | 215 ++++----- .../lens/investigations/FrequencyCard.tsx | 88 ++++ .../InvestigationsView.integration.test.tsx | 12 +- .../lens/investigations/IssueBrief.tsx | 4 +- .../lens/investigations/PriorityMark.tsx | 40 ++ .../components/lens/model/findings.test.ts | 31 +- .../src/components/lens/model/findings.ts | 13 + .../components/lens/model/frequency.test.ts | 64 +++ .../src/components/lens/model/frequency.ts | 58 +++ .../src/components/lens/model/inbox.ts | 4 + 15 files changed, 845 insertions(+), 249 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FrequencyCard.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/PriorityMark.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.ts diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index 3bc500c4848..36c22097cdc 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -211,6 +211,10 @@ --trace-row-hover: oklch(0.975 0.008 215); --trace-row-selected: oklch(0.95 0.035 200); --trace-brand: oklch(0.6 0.13 195); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.551 0.027 264.364 / 0.45); + --finding-quote: color-mix(in oklab, var(--warning) 16%, transparent); + --finding-ring: 0 0 0 1px oklch(0 0 0 / 0.06), 0 1px 2px -1px oklch(0 0 0 / 0.06), 0 2px 4px 0 oklch(0 0 0 / 0.04); --trace-border: oklch(0.92 0.01 230); --trace-line: oklch(0.88 0.03 205); --trace-card-border: oklch(0.93 0.01 230); @@ -288,6 +292,10 @@ --trace-row-hover: oklch(0.23 0.018 230); --trace-row-selected: oklch(0.29 0.05 210); --trace-brand: oklch(0.78 0.13 190); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.707 0.022 261.325 / 0.35); + --finding-quote: color-mix(in oklab, var(--warning) 24%, transparent); + --finding-ring: 0 0 0 1px oklch(1 0 0 / 0.08); --trace-border: oklch(0.3 0.02 235); --trace-line: oklch(0.36 0.04 210); --trace-card-border: oklch(0.27 0.02 235); @@ -326,6 +334,10 @@ --color-trace-row-hover: var(--trace-row-hover); --color-trace-row-selected: var(--trace-row-selected); --color-trace-brand: var(--trace-brand); + --color-finding-affected: var(--finding-affected); + --color-finding-unaffected: var(--finding-unaffected); + --color-finding-quote: var(--finding-quote); + --shadow-finding-ring: var(--finding-ring); --color-trace-border: var(--trace-border); --color-trace-line: var(--trace-line); --color-trace-card-border: var(--trace-card-border); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index c251b36d190..262d7ff9a51 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -141,9 +141,7 @@ describe("Lens interactive demo", () => { await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ })); const finding = screen.getByRole("complementary", { name: "Finding details" }); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); - const summaries = within(finding).getAllByText("support_agent", { exact: true }); - await user.click(summaries[0]); - await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]); + await user.click(within(finding).getAllByRole("button", { name: "View span" })[0]); expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent( "I will check that for you.", ); @@ -157,7 +155,8 @@ describe("Lens interactive demo", () => { await user.click(within(finding).getByRole("button", { name: "Back to finding" })); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); await user.click(within(finding).getByRole("button", { name: "Close finding (Esc)" })); - expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible(); + expect(await screen.findByRole("grid", { name: "Findings" })).toBeVisible(); + expect(screen.queryByRole("complementary", { name: "Finding details" })).not.toBeInTheDocument(); expect(network).not.toHaveBeenCalled(); await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true")); await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false)); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx index e705ce17e81..a6c1e60cb3b 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx @@ -116,8 +116,7 @@ it("stacks a quote's original step over the finding and keeps the feedback draft const panel = screen.getByRole("complementary", { name: "Finding details" }); const reason = () => within(panel).getByRole("textbox", { name: "What should Lens remember?", hidden: true }); fireEvent.change(reason(), { target: { value: "Draft feedback" } }); - for (const summary of within(panel).getAllByText(/quote$/)) await user.click(summary); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[0]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[0]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-1 at step-a"); expect(reason()).not.toBeVisible(); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); @@ -127,13 +126,51 @@ it("stacks a quote's original step over the finding and keeps the feedback draft expect(reason()).toBeVisible(); expect(reason()).toHaveValue("Draft feedback"); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[1]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[1]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-2 at step-b"); const url = new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")); expect(url.get("evidence")).toBe(traceOf("trace-2")); expect(url.get("evidence_span")).toBe("step-b"); }); +it("reports how many sampled traces the finding affected and highlights each quoted line", () => { + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const sampled = ["a", "b", "c", "d"].map((id) => ({ + id: traceOf(id), + name: `run ${id}`, + start_time: "2026-10-01T10:00:00Z", + metadata: [], + root_seen: true, + service: "support_agent", + source: "traces" as const, + span_count: 1, + team_id: "", + trace_id: id, + trace_ref: "", + })); + const current: Finding = { + ...finding, + occurrences: [traceOf("a")], + evidence: [{ execution_id: traceOf("a"), span_id: "s", quote: "files:read is missing", role: "support" }], + }; + renderWithLens( + + + , + ); + expect(screen.getByRole("region", { name: "Frequency" })).toHaveTextContent(/25%\s*1 of 4 traces affected/); + const example = screen.getByRole("article", { name: "run a" }); + expect(within(example).getByText("files:read is missing").tagName).toBe("MARK"); + expect(screen.queryByRole("article", { name: "run b" })).not.toBeInTheDocument(); +}); + it("shows contributing investigation runs and every affected trace, including older traces without retained quotes", async () => { const traceId = btoa(JSON.stringify(["traces", "", "older-trace", ""])); const current: Finding = { @@ -142,8 +179,33 @@ it("shows contributing investigation runs and every affected trace, including ol investigation_runs: ["first-investigation-run", "second-investigation-run"], }; renderWithLens(); - expect(screen.getByText("Found across 2 investigation runs")).toBeInTheDocument(); - expect(screen.getByText(/1 affected trace/)).toBeInTheDocument(); - fireEvent.click(screen.getByText("older-trace")); - expect(screen.getByRole("button", { name: "Open original trace" })).toBeInTheDocument(); + expect(screen.getByText("1 affected trace")).toBeVisible(); + expect(screen.getByText("Found across 2 investigation runs")).toBeVisible(); + const example = screen.getByRole("article", { name: "Trace older-tr" }); + expect(within(example).getByText("No quote was retained for this trace.")).toBeVisible(); + expect(within(example).getByRole("button", { name: "View trace" })).toBeVisible(); +}); + +it("shows the finding's priority and keeps the first three examples, revealing the rest on request", async () => { + const user = userEvent.setup(); + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const ids = ["t1", "t2", "t3", "t4", "t5"]; + const current: Finding = { + ...finding, + occurrences: ids.map(traceOf), + evidence: ids.map((id) => ({ + execution_id: traceOf(id), + span_id: id, + quote: `Input: ${id}\nOutput: done`, + role: "support" as const, + })), + }; + renderWithLens(); + const panel = screen.getByRole("complementary", { name: "Finding details" }); + expect(within(panel).getByText("High priority")).toBeVisible(); + expect(within(panel).getAllByRole("article")).toHaveLength(3); + expect(within(panel).getAllByText("Call and result")).toHaveLength(3); + await user.click(within(panel).getByRole("button", { name: "Show 2 more examples" })); + expect(within(panel).getAllByRole("article")).toHaveLength(5); + expect(within(panel).queryByRole("button", { name: /Show \d+ more/ })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx index 95157a878d1..59ad282bbe0 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx @@ -1,23 +1,32 @@ "use client"; import { useState } from "react"; -import { ArrowUpRight } from "lucide-react"; +import { ChevronRight, ClipboardCopy, X } from "lucide-react"; import { Inspector } from "@/components/shared/Inspector"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; +import { useNow } from "@/hooks/useNow"; +import { copyToClipboard } from "@/utils/dataUtils"; import { AddToDatasetButton } from "../datasets/AddToDatasetDialog"; -import { evidenceTarget } from "../model/findings"; -import { runTime } from "../model/format"; +import { evidenceTarget, findingMarkdown } from "../model/findings"; +import { findingFrequency } from "../model/frequency"; +import { agoLabel, runTime } from "../model/format"; import { findingAgents, findingKey, type OwnedFinding, sampledExecutions } from "../model/inbox"; import type { Finding, Sample } from "../model/types"; import { EvidenceView } from "./Evidence"; +import { FrequencyCard } from "./FrequencyCard"; import { IssueBrief } from "./IssueBrief"; +import { PriorityPill } from "./PriorityMark"; import { type EvidenceRef, useEvidenceRoute } from "../route"; export const ownedFindingKey = (owned: OwnedFinding): string => findingKey(owned.lens, owned.finding); +type Quote = Finding["evidence"][number]; + +const SECTION_LABEL = "text-xs font-medium text-muted-foreground"; + export interface FindingDetailsProps { readonly finding: Finding; readonly lensId?: string; @@ -27,6 +36,240 @@ export interface FindingDetailsProps { readonly busy: boolean; readonly onOpenEvidence: (evidence: EvidenceRef) => void; readonly onReview: (status: Finding["status"], reason: string) => void; + readonly onClose?: () => void; +} + +function TopBar({ finding, onClose }: Pick) { + const now = useNow(30000); + return ( +
+

+ + {finding.id.slice(0, 8)} + + + + {agoLabel(Date.parse(finding.last_seen), now)} + +

+
+ + {onClose && ( + + )} +
+
+ ); +} + +function Disclosure({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+ + +
{children}
+
+ ); +} + +function ProseSection({ title, children }: { title: string; children: string }) { + return ( +
+

{title}

+

+ {children} +

+
+ ); +} + +const FIELD = /^(Input|Output|Status|Error)\s*:/gm; +const FIELD_LABEL: Readonly> = { + "Input,Output": "Call and result", + Input: "Call input", + Output: "Returned output", + Status: "Span status", + Error: "Error", +}; + +function quoteLabel(quote: Quote, isTrace: boolean): string { + if (quote.role === "counterexample") return "Counterexample"; + const fields = [...new Set(Array.from(quote.quote.matchAll(FIELD), (m) => m[1]))].join(","); + return FIELD_LABEL[fields] ?? (isTrace ? "Trace step" : "Logged request"); +} + +const MARK = { + support: "rounded-sm bg-finding-quote px-0.5 text-inherit", + counterexample: "rounded-sm bg-success/20 px-0.5 text-inherit", +} as const; + +function QuoteCard({ quote, onOpen }: { quote: Quote; onOpen: () => void }) { + const isTrace = evidenceTarget(quote.execution_id)?.source === "traces"; + return ( +
+
+ {quoteLabel(quote, isTrace)} + +
+ +
+ ); +} + +function EvidenceRail({ children }: { children: React.ReactNode }) { + return ( + <> +
+
+
+
+
+ +
Evidence
+
+
+ + + ); +} + +interface ExampleGroup { + readonly id: string; + readonly run: Sample["executions"][number] | undefined; + readonly quotes: readonly Quote[]; +} + +function Example({ group, onOpenEvidence }: { group: ExampleGroup; onOpenEvidence: (e: EvidenceRef) => void }) { + const traceId = evidenceTarget(group.id)?.id; + const name = group.run?.name ?? (traceId ? `Trace ${traceId.slice(0, 8)}` : "Recorded run"); + return ( +
+
+

+ {name} +

+ + {[group.run?.service, group.run && runTime(group.run.start_time)].filter(Boolean).join(" · ")} + +
+
+ {group.quotes.length === 0 ? ( +
+

No quote was retained for this trace.

+ +
+ ) : ( + + {group.quotes.map((quote, i) => ( + onOpenEvidence({ id: quote.execution_id, span: quote.span_id })} + /> + ))} + + )} +
+
+ ); +} + +const VISIBLE_EXAMPLES = 3; + +function Examples({ + groups, + onOpenEvidence, +}: { + groups: readonly ExampleGroup[]; + onOpenEvidence: (e: EvidenceRef) => void; +}) { + const [expanded, setExpanded] = useState(false); + if (groups.length === 0) return

No examples were recorded.

; + const shown = expanded ? groups : groups.slice(0, VISIBLE_EXAMPLES); + const hidden = groups.length - shown.length; + return ( + <> + {shown.map((group) => ( + + ))} + {hidden > 0 && ( + + )} + + ); +} + +function ReviewForm({ finding, busy, onReview }: Pick) { + const [reason, setReason] = useState(finding.reason ?? ""); + return ( +
+