feat(traces): render curated SQL examples from shared files (#44388)

* feat(traces): render curated SQL examples from shared files

* fix(traces): filter trace-summary example after grouping so boundary-spanning traces keep full totals

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(traces): order failed-spans example by timestamp so newest failures survive the limit

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Yujong Lee <yujong@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-03 12:40:01 -07:00 • committed by GitHub
parent 1b98748528
commit d306d6d70b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 232 additions and 111 deletions

View file

@ -0,0 +1,14 @@
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
FROM otel_traces AS t
INNER JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.LiteLLMRequestId = s.response_id
AND t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.LiteLLMRequestId != ''
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'project')
AND JSONExtractString(metadata, 'project') = 'example'
ORDER BY start_time DESC
LIMIT 100

View file

@ -0,0 +1,6 @@
SELECT
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 30 DAY
ORDER BY key
LIMIT 200

View file

@ -0,0 +1,7 @@
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
SpanId AS span_id, StatusMessage AS message
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND StatusCode = 'STATUS_CODE_ERROR'
ORDER BY Timestamp DESC, team, api_key, trace_id, span_id
LIMIT 100

View file

@ -1,5 +1,8 @@
SELECT team_id AS team, api_key, request_id, spend,
JSONExtractString(metadata, 'labels', 'priority') AS priority
FROM spend_logs FINAL
WHERE JSONExtractString(metadata, 'labels', 'priority') = 'high'
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
AND JSONExtractString(metadata, 'labels', 'priority') = 'high'
ORDER BY team, api_key, request_id
LIMIT 100

View file

@ -0,0 +1,17 @@
SELECT
team_id, model, requests, unknown_cost_requests,
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
input_tokens, output_tokens
FROM (
SELECT
team_id, model, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
sum(spend) AS recorded_spend,
sum(prompt_tokens) AS input_tokens,
sum(completion_tokens) AS output_tokens
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
GROUP BY team_id, model
)
ORDER BY team_id, model
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id,
JSONType(metadata, 'labels', 'priority') AS type,
JSONExtractRaw(metadata, 'labels', 'priority') AS value
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
TraceId, SpanId, Model, InputTokens, OutputTokens,
Duration / 1000000 AS duration_ms
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND ObservationType = 'llm'
ORDER BY Timestamp DESC
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id, response_id, trace_id, span_id, model, spend,
prompt_tokens, completion_tokens, status,
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
ORDER BY start_time DESC, request_id
LIMIT 100

View file

@ -0,0 +1,10 @@
SELECT
team_id, api_key, trace_id, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND trace_id != ''
GROUP BY team_id, api_key, trace_id
ORDER BY team_id, api_key, trace_id
LIMIT 100

View file

@ -7,4 +7,6 @@ SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
toUInt32(sum(OutputTokens)) AS output_tokens
FROM agent_traces_by_key
GROUP BY TeamId, ApiKeyHash, TraceId
HAVING min(StartTs) >= now() - INTERVAL 1 DAY
ORDER BY team, api_key, trace_id
LIMIT 100

View file

@ -0,0 +1,18 @@
SELECT
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
t.InputTokens, t.OutputTokens
FROM otel_traces AS t
LEFT ANTI JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
AND t.LiteLLMRequestId != ''
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.ObservationType = 'llm'
ORDER BY t.Timestamp DESC, t.SpanId
LIMIT 100

View file

@ -297,7 +297,7 @@ pub struct QueryHelp {
attributes: Vec<AttributeCatalog>,
relationships: &'static [Relationship],
#[cfg_attr(feature = "schema", schemars(with = "Vec<Example>"))]
examples: [Example; 9],
examples: [Example; 12],
#[cfg_attr(feature = "schema", schemars(with = "Vec<String>"))]
gotchas: [String; 13],
guide: String,

View file

@ -28,6 +28,12 @@ use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits};
"trace_spend_sql",
"unmatched_spans_name",
"unmatched_spans_sql",
"trace_summary_name",
"trace_summary_sql",
"failed_spans_name",
"failed_spans_sql",
"metadata_filter_name",
"metadata_filter_sql",
"missing_spend",
"partial_spend",
"time_window",
@ -60,7 +66,7 @@ impl QueryGuide<'_> {
])
}
pub fn examples(&self) -> Result<[Example; 9], Error> {
pub fn examples(&self) -> Result<[Example; 12], Error> {
Ok([
Example {
name: render(&self.as_recent_spans_name())?,
@ -98,6 +104,18 @@ impl QueryGuide<'_> {
name: render(&self.as_unmatched_spans_name())?,
sql: render(&self.as_unmatched_spans_sql())?,
},
Example {
name: render(&self.as_trace_summary_name())?,
sql: render(&self.as_trace_summary_sql())?,
},
Example {
name: render(&self.as_failed_spans_name())?,
sql: render(&self.as_failed_spans_sql())?,
},
Example {
name: render(&self.as_metadata_filter_name())?,
sql: render(&self.as_metadata_filter_sql())?,
},
])
}

View file

@ -60,14 +60,7 @@ Recent normalized LLM spans
{%- endblock %}
{% block recent_spans_sql -%}
SELECT
TraceId, SpanId, Model, InputTokens, OutputTokens,
Duration / 1000000 AS duration_ms
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND ObservationType = 'llm'
ORDER BY Timestamp DESC
LIMIT 100
{% include "../query/help/recent_spans.sql" %}
{%- endblock %}
{% block custom_metadata_name -%}
@ -75,14 +68,7 @@ Find calls by custom metadata
{%- endblock %}
{% block custom_metadata_sql -%}
SELECT
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'project')
AND JSONExtractString(metadata, 'project') = 'example'
ORDER BY start_time DESC
LIMIT 100
{% include "../query/help/custom_metadata.sql" %}
{%- endblock %}
{% block nested_metadata_name -%}
@ -90,14 +76,7 @@ Nested metadata with unknown types
{%- endblock %}
{% block nested_metadata_sql -%}
SELECT
request_id,
JSONType(metadata, 'labels', 'priority') AS type,
JSONExtractRaw(metadata, 'labels', 'priority') AS value
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
LIMIT 100
{% include "../query/help/nested_metadata.sql" %}
{%- endblock %}
{% block correlated_calls_name -%}
@ -105,20 +84,7 @@ Traces correlated with LLM call metadata
{%- endblock %}
{% block correlated_calls_sql -%}
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
FROM otel_traces AS t
INNER JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.LiteLLMRequestId = s.response_id
AND t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.LiteLLMRequestId != ''
LIMIT 100
{% include "../query/help/correlated_calls.sql" %}
{%- endblock %}
{% block discover_keys_name -%}
@ -126,12 +92,7 @@ Discover metadata keys over a different window
{%- endblock %}
{% block discover_keys_sql -%}
SELECT
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 30 DAY
ORDER BY key
LIMIT 200
{% include "../query/help/discover_keys.sql" %}
{%- endblock %}
{% block recent_spend_name -%}
@ -139,14 +100,7 @@ Recent spend records
{%- endblock %}
{% block recent_spend_sql -%}
SELECT
request_id, response_id, trace_id, span_id, model, spend,
prompt_tokens, completion_tokens, status,
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
ORDER BY start_time DESC, request_id
LIMIT 100
{% include "../query/help/recent_spend.sql" %}
{%- endblock %}
{% block model_spend_name -%}
@ -154,23 +108,7 @@ Spend and tokens by model
{%- endblock %}
{% block model_spend_sql -%}
SELECT
team_id, model, requests, unknown_cost_requests,
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
input_tokens, output_tokens
FROM (
SELECT
team_id, model, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
sum(spend) AS recorded_spend,
sum(prompt_tokens) AS input_tokens,
sum(completion_tokens) AS output_tokens
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
GROUP BY team_id, model
)
ORDER BY team_id, model
LIMIT 100
{% include "../query/help/model_spend.sql" %}
{%- endblock %}
{% block trace_spend_name -%}
@ -178,16 +116,7 @@ Recorded spend by trace
{%- endblock %}
{% block trace_spend_sql -%}
SELECT
team_id, api_key, trace_id, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND trace_id != ''
GROUP BY team_id, api_key, trace_id
ORDER BY team_id, api_key, trace_id
LIMIT 100
{% include "../query/help/trace_spend.sql" %}
{%- endblock %}
{% block unmatched_spans_name -%}
@ -195,24 +124,31 @@ LLM spans without a direct spend match
{%- endblock %}
{% block unmatched_spans_sql -%}
SELECT
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
t.InputTokens, t.OutputTokens
FROM otel_traces AS t
LEFT ANTI JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
AND t.LiteLLMRequestId != ''
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.ObservationType = 'llm'
ORDER BY t.Timestamp DESC, t.SpanId
LIMIT 100
{% include "../query/help/unmatched_spans.sql" %}
{%- endblock %}
{% block trace_summary_name -%}
Trace summaries with tokens and errors
{%- endblock %}
{% block trace_summary_sql -%}
{% include "../query/help/trace_summary.sql" %}
{%- endblock %}
{% block failed_spans_name -%}
Recent failed spans
{%- endblock %}
{% block failed_spans_sql -%}
{% include "../query/help/failed_spans.sql" %}
{%- endblock %}
{% block metadata_filter_name -%}
Filter calls by nested metadata
{%- endblock %}
{% block metadata_filter_sql -%}
{% include "../query/help/metadata_filter.sql" %}
{%- endblock %}
{% block time_window -%}

View file

@ -1683,7 +1683,15 @@ async fn query_help_discovers_live_schema_and_runs_its_examples(
let values: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(
values["data"].as_array().ok_or("missing data")?.is_empty(),
!populated || example["name"] == "LLM spans without a direct spend match",
!populated
|| matches!(
example["name"].as_str(),
Some(
"LLM spans without a direct spend match"
| "Recent failed spans"
| "Filter calls by nested metadata"
)
),
"{sql}"
);
if populated && example["name"] == "Traces correlated with LLM call metadata" {

View file

@ -5,7 +5,7 @@ use litellm_traces::query::named as contracts;
use litellm_traces_clickhouse::{
QueryScope,
query::named::{ListTraces, ListTracesParams, TraceSpans, TraceSpansParams},
query_sql,
query_help, query_sql,
};
use rstest::{fixture, rstest};
use serde::Deserialize;
@ -48,12 +48,9 @@ struct QueryResult {
}
#[rstest]
#[case::rollups(include_str!("queries/rollups.sql"), include_str!("queries/rollups.expected.json"))]
#[case::costs(include_str!("queries/trace_costs.sql"), include_str!("queries/trace_costs.expected.json"))]
#[case::errors(include_str!("queries/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
#[case::metadata(include_str!("queries/metadata_filters.sql"), include_str!("queries/metadata_filters.expected.json"))]
#[tokio::test]
async fn curated_queries_return_expected_rows(
async fn storage_queries_return_expected_rows(
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
#[case] sql: &str,
#[case] expected_json: &str,
@ -78,6 +75,64 @@ async fn curated_queries_return_expected_rows(
Ok(())
}
#[rstest]
#[case::trace_summary("Trace summaries with tokens and errors", include_str!("../query/help/trace_summary.sql"), include_str!("queries/rollups.expected.json"))]
#[case::failed_spans("Recent failed spans", include_str!("../query/help/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
#[case::metadata_filter("Filter calls by nested metadata", include_str!("../query/help/metadata_filter.sql"), include_str!("queries/metadata_filters.expected.json"))]
#[tokio::test]
async fn documented_queries_render_and_return_expected_rows(
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
fixture_clock: TestResult<u64>,
#[case] name: &str,
#[case] expected_sql: &str,
#[case] expected_json: &str,
#[values(ScopeCase::Admin, ScopeCase::Team, ScopeCase::OtherTeam)] scope: ScopeCase,
) -> TestResult {
let fixture = seeded_database?;
let reader = fixture
.readers
.connection(&fixture.database.client, &scope.scope(), "fixture-secret")
.await?;
let help = serde_json::to_value(query_help(&fixture.database.client, &reader).await?)?;
let example = help["examples"]
.as_array()
.ok_or("missing examples")?
.iter()
.find(|example| example["name"] == name)
.ok_or("missing documented query")?;
let sql = example["sql"].as_str().ok_or("missing example SQL")?;
assert_eq!(sql.trim(), expected_sql.trim());
assert!(
help["guide"]
.as_str()
.ok_or("missing guide")?
.contains(&format!("{name}\n{sql}"))
);
let sql_at_fixture_time = sql.replace("now()", &format!("toDateTime({})", fixture_clock?));
let result: QueryResult = serde_json::from_str(
&query_sql(&fixture.database.client, &reader, &sql_at_fixture_time).await?,
)?;
let expected: BTreeMap<String, Vec<Value>> = serde_json::from_str(expected_json)?;
assert_eq!(
&result.data,
expected
.get(scope.as_ref())
.ok_or("missing expected scope")?,
"{}: {sql}",
scope.as_ref()
);
Ok(())
}
#[fixture]
fn fixture_clock() -> TestResult<u64> {
let spans = litellm_traces::decode_otlp(
include_bytes!("../../traces/tests/fixtures/query_root.json"),
Some("application/json"),
)?;
Ok(spans.first().ok_or("missing fixture root")?.start_ns / 1_000_000_000)
}
#[fixture]
fn admin_access() -> TestResult<contracts::ReadAccessParams> {
Ok(serde_json::from_str(include_str!(

View file

@ -1,5 +0,0 @@
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
SpanId AS span_id, StatusMessage AS message
FROM otel_traces
WHERE StatusCode = 'STATUS_CODE_ERROR'
ORDER BY team, api_key, trace_id, span_id