mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
feat(lens): scope traces to one agent with a header picker (#45202)
* feat(lens): add trace_agents rollup query * feat(lens): register the trace_agents read query * feat(lens): add trace_agents params and row types * feat(lens): dispatch the trace_agents query * feat(lens): export trace_agents wire schemas * test(lens): pin the trace_agents query name * test(lens): cover trace_agents scope, window and failure counts * feat(lens): cap the agent list size * feat(lens): declare the trace_agents bridge query * feat(lens): read trace agents from clickhouse storage * feat(lens): add trace agent response models * feat(lens): list agents within the reader's trace scope * feat(lens): add GET /v1/traces/agents * chore(lens): regenerate trace models with trace_agents * chore(lens): regenerate trace types with trace_agents * chore(lens): regenerate read query name schema * chore(lens): add trace agent row schema * chore(lens): add trace agents params schema * test(lens): cover the trace agents route * test(lens): cover agent listing scope and timestamps * chore(ui): regenerate api types with trace agents route * feat(lens): derive trace agent types from the schema * feat(lens): fetch the agent list from the traces api * feat(lens): roll up demo runs into agents * feat(lens): serve the agent list in demo data * feat(lens): load agents seen in the last two weeks * feat(lens): remember the selected agent per browser * feat(lens): add the agent picker * feat(lens): wire agent selection into the lens header * feat(lens): show the agent picker next to the lens title * refactor(lens): drop the toolbar agent filter in favor of the header picker * test(lens): remove tests for the toolbar agent filter * test(lens): cover agent resolution and rollup * test(lens): cover scoping, switching and remembering the agent * test(lens): read only the create request in guided setup * test(lens): assert the toolbar agent filter is gone * test(lens): read stubbed requests without their abort signal
This commit is contained in:
ishaan-berri
GitHub
parent
6befb9ad7d
commit
bf9bd35469
37 changed files with 1199 additions and 103 deletions
33
litellm-rust/crates/traces-clickhouse/query/trace_agents.sql
Normal file
33
litellm-rust/crates/traces-clickhouse/query/trace_agents.sql
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
WITH runs AS (
|
||||
SELECT TeamId, ApiKeyHash, TraceId,
|
||||
toUnixTimestamp64Milli(min(StartTs)) AS start_ms,
|
||||
min(StartTs) AS trace_start, max(EndTs) AS trace_end,
|
||||
sum(ErrorCount) > 0 AS failed
|
||||
FROM agent_traces_by_key
|
||||
WHERE ({all_teams:UInt8} = 1
|
||||
OR ({user_id:String} != '' AND UserIds = [{user_id:String}])
|
||||
OR has({team_ids:Array(String)}, TeamId))
|
||||
GROUP BY TeamId, ApiKeyHash, TraceId
|
||||
HAVING min(StartTs) >= fromUnixTimestamp64Milli({start_ms:Int64})
|
||||
AND min(StartTs) < fromUnixTimestamp64Milli({end_ms:Int64})
|
||||
),
|
||||
named AS (
|
||||
SELECT DISTINCT o.TeamId AS TeamId, o.ApiKeyHash AS ApiKeyHash, o.TraceId AS TraceId,
|
||||
o.AgentName AS agent_name, toString(o.Framework) AS framework
|
||||
FROM otel_traces AS o
|
||||
WHERE o.AgentName != ''
|
||||
AND o.Timestamp >= (SELECT min(trace_start) FROM runs)
|
||||
AND o.Timestamp <= (SELECT max(trace_end) FROM runs)
|
||||
AND (o.TeamId, o.ApiKeyHash, o.TraceId) IN (SELECT TeamId, ApiKeyHash, TraceId FROM runs)
|
||||
)
|
||||
SELECT named.agent_name AS agent_name,
|
||||
uniqExact(named.TeamId, named.ApiKeyHash, named.TraceId) AS runs,
|
||||
uniqExactIf((named.TeamId, named.ApiKeyHash, named.TraceId), runs.failed) AS failed_runs,
|
||||
max(runs.start_ms) AS last_seen_ms,
|
||||
arraySort(groupUniqArrayIf(named.framework, named.framework != '')) AS frameworks
|
||||
FROM named
|
||||
INNER JOIN runs ON named.TeamId = runs.TeamId AND named.ApiKeyHash = runs.ApiKeyHash
|
||||
AND named.TraceId = runs.TraceId
|
||||
GROUP BY named.agent_name
|
||||
ORDER BY last_seen_ms DESC, agent_name
|
||||
LIMIT {limit:UInt32}
|
||||
|
|
@ -6,7 +6,8 @@ const SAMPLE_READ_LIMITS: ReadLimits = ReadLimits {
|
|||
..litellm_storage_clickhouse::READ_LIMITS
|
||||
};
|
||||
|
||||
pub const LENS_QUERIES: [litellm_traces::ReadQuery; 5] = [
|
||||
pub const LENS_QUERIES: [litellm_traces::ReadQuery; 6] = [
|
||||
litellm_traces::ReadQuery::TraceAgents,
|
||||
litellm_traces::ReadQuery::Availability,
|
||||
litellm_traces::ReadQuery::Agents,
|
||||
litellm_traces::ReadQuery::Sample,
|
||||
|
|
@ -107,6 +108,67 @@ impl Query for LensAgents {
|
|||
const SQL: &'static str = include_str!("../../query/lens_agents.sql");
|
||||
}
|
||||
|
||||
pub struct TraceAgents;
|
||||
|
||||
/// Same access shape as `list_traces`: every team, the caller's own traces, or their teams' traces.
|
||||
#[macro_rules_attribute::apply(wire_type)]
|
||||
#[derive(Debug)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
|
||||
pub struct TraceAgentsParams {
|
||||
#[serde(
|
||||
deserialize_with = "super::number::boolean",
|
||||
serialize_with = "litellm_traces::wire::serialize_flag"
|
||||
)]
|
||||
#[cfg_attr(
|
||||
feature = "schema",
|
||||
schemars(schema_with = "litellm_traces::schema::flag")
|
||||
)]
|
||||
pub all_teams: bool,
|
||||
pub user_id: String,
|
||||
pub team_ids: Vec<String>,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
pub start_ms: i64,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
pub end_ms: i64,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
pub limit: u32,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(wire_type)]
|
||||
#[derive(Debug)]
|
||||
#[cfg_attr(feature = "schema", schemars(rename = "TraceAgentRow"))]
|
||||
pub struct TraceAgentsRow {
|
||||
pub agent_name: String,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
#[cfg_attr(
|
||||
feature = "schema",
|
||||
schemars(schema_with = "crate::wire_schema::u64_number")
|
||||
)]
|
||||
pub runs: u64,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
#[cfg_attr(
|
||||
feature = "schema",
|
||||
schemars(schema_with = "crate::wire_schema::u64_number")
|
||||
)]
|
||||
pub failed_runs: u64,
|
||||
#[serde(deserialize_with = "super::number::deserialize")]
|
||||
#[cfg_attr(
|
||||
feature = "schema",
|
||||
schemars(schema_with = "crate::wire_schema::u64_number")
|
||||
)]
|
||||
pub last_seen_ms: u64,
|
||||
#[serde(default)]
|
||||
pub frameworks: Vec<String>,
|
||||
}
|
||||
|
||||
impl Query for TraceAgents {
|
||||
type Params = TraceAgentsParams;
|
||||
type Row = TraceAgentsRow;
|
||||
|
||||
const SQL: &'static str = include_str!("../../query/trace_agents.sql");
|
||||
}
|
||||
|
||||
pub struct LensSample;
|
||||
|
||||
#[macro_rules_attribute::apply(wire_type)]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ pub async fn execute_named_read(
|
|||
) -> Result<String, Error> {
|
||||
match query {
|
||||
ReadQuery::ListTraces => named_json::<ListTraces>(client, connection, parameters).await,
|
||||
ReadQuery::TraceAgents => named_json::<TraceAgents>(client, connection, parameters).await,
|
||||
ReadQuery::TraceIdentity => {
|
||||
named_json::<TraceIdentity>(client, connection, parameters).await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ pub fn schemas() -> BTreeMap<&'static str, Schema> {
|
|||
("PartRow", received::<lens::LensContentRow>()),
|
||||
("CountRow", received::<lens::LensEvidenceRow>()),
|
||||
("AgentRow", received::<lens::LensAgentsRow>()),
|
||||
("TraceAgentsParams", received::<lens::TraceAgentsParams>()),
|
||||
("TraceAgentRow", received::<lens::TraceAgentsRow>()),
|
||||
("TraceQueryHelp", crate::query::help_schema()),
|
||||
])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -754,6 +754,145 @@ async fn listed_agent_names_preserve_scope_and_cursor(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trace_agents_count_runs_and_failures_within_scope_and_window(
|
||||
#[future(awt)] database: TestResult<ClickHouseDatabase>,
|
||||
) -> TestResult {
|
||||
let database = database?;
|
||||
let writer = Connection::writer(&database.url)?;
|
||||
ensure_schema(&database.client, &writer, "trace_test", 7).await?;
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
|
||||
let old = now - 3 * 86_400_000_000_000_i64;
|
||||
for (team, trace, span, parent, agent, status, framework, timestamp) in [
|
||||
(
|
||||
"alpha",
|
||||
"run-1",
|
||||
"root",
|
||||
"",
|
||||
"moyai",
|
||||
"STATUS_CODE_OK",
|
||||
"pi",
|
||||
now,
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"run-1",
|
||||
"tool",
|
||||
"root",
|
||||
"moyai",
|
||||
"STATUS_CODE_ERROR",
|
||||
"pi",
|
||||
now,
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"run-2",
|
||||
"root",
|
||||
"",
|
||||
"moyai",
|
||||
"STATUS_CODE_OK",
|
||||
"",
|
||||
now - 1_000_000,
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"run-3",
|
||||
"root",
|
||||
"",
|
||||
"research",
|
||||
"STATUS_CODE_OK",
|
||||
"",
|
||||
now - 2_000_000,
|
||||
),
|
||||
(
|
||||
"alpha",
|
||||
"old-run",
|
||||
"root",
|
||||
"",
|
||||
"moyai",
|
||||
"STATUS_CODE_ERROR",
|
||||
"",
|
||||
old,
|
||||
),
|
||||
(
|
||||
"beta",
|
||||
"other-team",
|
||||
"root",
|
||||
"",
|
||||
"moyai",
|
||||
"STATUS_CODE_ERROR",
|
||||
"",
|
||||
now,
|
||||
),
|
||||
(
|
||||
"beta",
|
||||
"other-agent",
|
||||
"root",
|
||||
"",
|
||||
"hidden_agent",
|
||||
"STATUS_CODE_OK",
|
||||
"",
|
||||
now,
|
||||
),
|
||||
] {
|
||||
insert_rows(
|
||||
&database,
|
||||
"otel_traces",
|
||||
vec![serde_json::from_value(serde_json::json!({
|
||||
"Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent,
|
||||
"ServiceName": "app", "SpanName": span, "AgentName": agent, "UserId": "owner",
|
||||
"StatusCode": status, "Framework": framework, "ObservationType": "agent",
|
||||
"ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": "key"}
|
||||
}))?],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
|
||||
let parameters = BTreeMap::from([
|
||||
("all_teams".into(), Parameter::Integer(0)),
|
||||
("user_id".into(), Parameter::Text(String::new())),
|
||||
("team_ids".into(), Parameter::Strings(vec!["alpha".into()])),
|
||||
(
|
||||
"start_ms".into(),
|
||||
Parameter::Integer(now / 1_000_000 - 86_400_000),
|
||||
),
|
||||
("end_ms".into(), Parameter::Integer(now / 1_000_000 + 1000)),
|
||||
("limit".into(), Parameter::Integer(10)),
|
||||
]);
|
||||
let agents: serde_json::Value = serde_json::from_str(
|
||||
&execute_named_read(
|
||||
&database.client,
|
||||
&connection,
|
||||
ReadQuery::TraceAgents,
|
||||
¶meters,
|
||||
)
|
||||
.await?,
|
||||
)?;
|
||||
let rows = agents["data"].as_array().ok_or("missing agents")?;
|
||||
let summary = rows
|
||||
.iter()
|
||||
.map(|row| {
|
||||
(
|
||||
row["agent_name"].as_str().unwrap_or_default(),
|
||||
(
|
||||
row["runs"].to_string().trim_matches('"').to_owned(),
|
||||
row["failed_runs"].to_string().trim_matches('"').to_owned(),
|
||||
row["frameworks"].clone(),
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
summary,
|
||||
vec![
|
||||
("moyai", ("2".into(), "1".into(), serde_json::json!(["pi"]))),
|
||||
("research", ("1".into(), "0".into(), serde_json::json!([]))),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rollup_merges_spans_across_days_without_losing_root_fields(
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ pub mod named;
|
|||
#[strum(serialize_all = "snake_case")]
|
||||
pub enum ReadQuery {
|
||||
ListTraces,
|
||||
TraceAgents,
|
||||
TraceSpans,
|
||||
TracePageSpans,
|
||||
TraceIdentity,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use rstest::rstest;
|
|||
|
||||
#[rstest]
|
||||
#[case::list_traces("list_traces", ReadQuery::ListTraces)]
|
||||
#[case::trace_agents("trace_agents", ReadQuery::TraceAgents)]
|
||||
#[case::trace_spans("trace_spans", ReadQuery::TraceSpans)]
|
||||
#[case::span_detail("span_detail", ReadQuery::SpanDetail)]
|
||||
#[case::span_error("span_error", ReadQuery::SpanError)]
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ TRACE_READ_RETRY_AFTER_SECONDS: Final = get_env_int("TRACE_READ_RETRY_AFTER_SECO
|
|||
OTLP_MAX_CONCURRENT_INGESTS: Final = get_env_int("OTLP_MAX_CONCURRENT_INGESTS", 2)
|
||||
AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240)
|
||||
AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50)
|
||||
AGENT_TRACING_AGENT_LIST_LIMIT: Final = get_env_int("AGENT_TRACING_AGENT_LIST_LIMIT", 500)
|
||||
LENS_DATASET_MAX_CASES: Final = get_env_int("LENS_DATASET_MAX_CASES", 200)
|
||||
LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", 20_000)
|
||||
LENS_DATASET_TRACE_PAGE_SIZE: Final = 500
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Agent tracing endpoints. Thin wrappers over `TraceReceiver`: auth -> tenant/scop
|
|||
|
||||
POST /v1/traces OTLP/HTTP trace export (protobuf or JSON)
|
||||
GET /v1/traces TracePage
|
||||
GET /v1/traces/agents TraceAgentList
|
||||
GET /v1/traces/{trace_id} Trace
|
||||
GET /v1/traces/{trace_id}/spans/{span_id} SpanDetail
|
||||
"""
|
||||
|
|
@ -20,7 +21,11 @@ from pydantic import ConfigDict
|
|||
from typing_extensions import assert_never
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import OTLP_RETRY_AFTER_SECONDS, TRACE_READ_RETRY_AFTER_SECONDS
|
||||
from litellm.constants import (
|
||||
DEFAULT_AGENT_TRACING_RETENTION_DAYS,
|
||||
OTLP_RETRY_AFTER_SECONDS,
|
||||
TRACE_READ_RETRY_AFTER_SECONDS,
|
||||
)
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_trace_read_scope
|
||||
from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency
|
||||
|
|
@ -50,6 +55,7 @@ from litellm.rust_bridge.trace.generated.types import (
|
|||
from litellm.rust_bridge.trace.storage import ClickHouseStorage, Tenant
|
||||
from litellm.tracing import TraceReceiver, TracingPayloadTooLargeError
|
||||
from litellm.tracing.otlp_http import InvalidOTLPPayloadError, encode_otlp_response
|
||||
from litellm.tracing.types import TraceAgentList
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router = APIRouter(tags=["agent tracing"])
|
||||
|
|
@ -212,6 +218,34 @@ async def list_agent_traces(
|
|||
raise read_failure(error) from error
|
||||
|
||||
|
||||
class TraceAgentListRequest(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
start_ms: int | None = None
|
||||
end_ms: int | None = None
|
||||
|
||||
|
||||
@router.get("/v1/traces/agents", response_model=TraceAgentList)
|
||||
async def list_trace_agents(
|
||||
context: Annotated[TraceAccessContext, Depends(provide_trace_access)],
|
||||
now_ms: Annotated[int, Depends(current_time_ms)],
|
||||
request: Annotated[TraceAgentListRequest, Query()],
|
||||
) -> TraceAgentList:
|
||||
try:
|
||||
tracing, scope = context.reader()
|
||||
return await tracing.list_agents(
|
||||
scope=scope,
|
||||
start_ms=(
|
||||
request.start_ms
|
||||
if request.start_ms is not None
|
||||
else now_ms - DEFAULT_AGENT_TRACING_RETENTION_DAYS * MS_PER_DAY
|
||||
),
|
||||
end_ms=request.end_ms if request.end_ms is not None else now_ms,
|
||||
)
|
||||
except (ValueError, RuntimeError) as error:
|
||||
raise read_failure(error) from error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TraceQueryAccess:
|
||||
storage: ClickHouseStorage
|
||||
|
|
|
|||
|
|
@ -283,6 +283,131 @@ class PartRow(LiteLLMBaseModel):
|
|||
truncated: int = Field(..., ge=0, le=1)
|
||||
|
||||
|
||||
Runs: TypeAlias = Annotated[
|
||||
int,
|
||||
Field(
|
||||
...,
|
||||
ge=0,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
le=18446744073709551615,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
Runs1: TypeAlias = Annotated[
|
||||
str,
|
||||
Field(
|
||||
...,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
FailedRuns: TypeAlias = Annotated[
|
||||
int,
|
||||
Field(
|
||||
...,
|
||||
ge=0,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
le=18446744073709551615,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
FailedRuns1: TypeAlias = Annotated[
|
||||
str,
|
||||
Field(
|
||||
...,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
LastSeenMs: TypeAlias = Annotated[
|
||||
int,
|
||||
Field(
|
||||
...,
|
||||
ge=0,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
le=18446744073709551615,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
LastSeenMs1: TypeAlias = Annotated[
|
||||
str,
|
||||
Field(
|
||||
...,
|
||||
json_schema_extra={
|
||||
"x-python-normalized": {
|
||||
"type": "int",
|
||||
"minimum": 0,
|
||||
"maximum": 18446744073709551615,
|
||||
}
|
||||
},
|
||||
pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TraceAgentRow(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(
|
||||
frozen=True,
|
||||
)
|
||||
|
||||
agent_name: str
|
||||
runs: int = Field(..., ge=0, le=18446744073709551615)
|
||||
failed_runs: int = Field(..., ge=0, le=18446744073709551615)
|
||||
last_seen_ms: int = Field(..., ge=0, le=18446744073709551615)
|
||||
frameworks: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class TraceAgentsParams(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
frozen=True,
|
||||
)
|
||||
|
||||
all_teams: Literal[0, 1]
|
||||
user_id: str
|
||||
team_ids: tuple[str, ...]
|
||||
start_ms: int = Field(..., ge=-9223372036854775808, le=9223372036854775807)
|
||||
end_ms: int = Field(..., ge=-9223372036854775808, le=9223372036854775807)
|
||||
limit: int = Field(..., ge=0, le=4294967295)
|
||||
|
||||
|
||||
TraceTableName: TypeAlias = Literal["otel_traces", "agent_traces_by_key", "spend_logs"]
|
||||
|
||||
|
||||
|
|
@ -435,6 +560,8 @@ TraceWireModels: TypeAlias = Annotated[
|
|||
| LensEvidenceParams
|
||||
| LensSampleParams
|
||||
| PartRow
|
||||
| TraceAgentRow
|
||||
| TraceAgentsParams
|
||||
| TraceQueryHelp,
|
||||
Field(..., title="TraceWireModels"),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ class TraceScope(typing_extensions.TypedDict):
|
|||
team_ids: ReadOnly[tuple[str, ...]]
|
||||
|
||||
|
||||
ReadQueryName: TypeAlias = Literal["availability", "agents", "sample", "content", "evidence"]
|
||||
ReadQueryName: TypeAlias = Literal["trace_agents", "availability", "agents", "sample", "content", "evidence"]
|
||||
|
||||
|
||||
class UIFields(typing_extensions.TypedDict):
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ from .generated.models import (
|
|||
LensEvidenceParams,
|
||||
LensSampleParams,
|
||||
PartRow,
|
||||
TraceAgentRow,
|
||||
TraceAgentsParams,
|
||||
TraceQueryColumn,
|
||||
)
|
||||
from .generated.types import ReadQueryName
|
||||
|
|
@ -54,6 +56,9 @@ class ReadQuery(Generic[ParamsT, RowT]):
|
|||
response: TypeAdapter[QueryResponse[RowT]]
|
||||
|
||||
|
||||
TRACE_AGENTS: Final[ReadQuery[TraceAgentsParams, TraceAgentRow]] = ReadQuery(
|
||||
"trace_agents", TraceAgentsParams, TypeAdapter(QueryResponse[TraceAgentRow])
|
||||
)
|
||||
LENS_AVAILABILITY: Final[ReadQuery[LensAccessParams, ActivityAvailability]] = ReadQuery(
|
||||
"availability", LensAccessParams, TypeAdapter(QueryResponse[ActivityAvailability])
|
||||
)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ from litellm.rust_bridge.trace.generated.models import (
|
|||
LensEvidenceParams,
|
||||
LensSampleParams,
|
||||
PartRow,
|
||||
TraceAgentRow,
|
||||
TraceAgentsParams,
|
||||
)
|
||||
from litellm.rust_bridge.trace.generated.responses import TraceSQLResponse
|
||||
from litellm.rust_bridge.trace.generated.types import ReadQueryName
|
||||
|
|
@ -25,6 +27,7 @@ from litellm.rust_bridge.trace.queries import (
|
|||
LENS_CONTENT,
|
||||
LENS_EVIDENCE,
|
||||
LENS_SAMPLE,
|
||||
TRACE_AGENTS,
|
||||
ClickHouseSQLEnvelope,
|
||||
ParamsT,
|
||||
ReadQuery,
|
||||
|
|
@ -229,6 +232,9 @@ class ClickHouseStorage:
|
|||
result: Final = await self._native.query_help(scope, secret)
|
||||
return _validate_query_response(_HELP_RESPONSE, result)
|
||||
|
||||
async def trace_agents(self, parameters: TraceAgentsParams) -> tuple[TraceAgentRow, ...]:
|
||||
return await self.query(TRACE_AGENTS, parameters)
|
||||
|
||||
async def lens_sample(self, parameters: LensSampleParams) -> tuple[ExecutionRow, ...]:
|
||||
return await self.query(LENS_SAMPLE, parameters)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,15 +14,23 @@ The proxy endpoints are thin wrappers: auth -> build tenant/scope -> call one me
|
|||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterable, Callable, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
from threading import BoundedSemaphore
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE, OTLP_MAX_BODY_BYTES, OTLP_MAX_CONCURRENT_INGESTS
|
||||
from litellm.constants import (
|
||||
AGENT_TRACING_AGENT_LIST_LIMIT,
|
||||
AGENT_TRACING_LIST_PAGE_SIZE,
|
||||
OTLP_MAX_BODY_BYTES,
|
||||
OTLP_MAX_CONCURRENT_INGESTS,
|
||||
)
|
||||
from litellm.rust_bridge.trace.generated.models import TraceAgentsParams
|
||||
from litellm.rust_bridge.trace.generated.types import SpanDetail, SpanErrorPage, Trace, TracePage, TraceScope
|
||||
from litellm.rust_bridge.trace.storage import ClickHouseStorage, Tenant
|
||||
from litellm.tracing.config import trace_storage_config
|
||||
from litellm.tracing.otlp_http import InvalidOTLPPayloadError, TracingPayloadTooLargeError, decompress
|
||||
from litellm.tracing.types import TraceAgent, TraceAgentList
|
||||
|
||||
|
||||
class TracingOverloadedError(RuntimeError):
|
||||
|
|
@ -101,6 +109,30 @@ class TraceReceiver:
|
|||
async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage:
|
||||
return await self.storage.list_traces(scope, start_ms, end_ms, cursor, AGENT_TRACING_LIST_PAGE_SIZE)
|
||||
|
||||
async def list_agents(self, scope: TraceScope, start_ms: int, end_ms: int) -> TraceAgentList:
|
||||
rows: Final = await self.storage.trace_agents(
|
||||
TraceAgentsParams(
|
||||
all_teams=scope["all_teams"],
|
||||
user_id=scope["user_id"],
|
||||
team_ids=tuple(scope["team_ids"]),
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
limit=AGENT_TRACING_AGENT_LIST_LIMIT,
|
||||
)
|
||||
)
|
||||
return TraceAgentList(
|
||||
agents=tuple(
|
||||
TraceAgent(
|
||||
name=row.agent_name,
|
||||
runs=row.runs,
|
||||
failed_runs=row.failed_runs,
|
||||
last_seen=datetime.fromtimestamp(row.last_seen_ms / 1000, tz=timezone.utc),
|
||||
frameworks=row.frameworks,
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
)
|
||||
|
||||
async def get_trace(
|
||||
self,
|
||||
trace_id: str,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,29 @@
|
|||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
||||
class TraceAgent(LiteLLMBaseModel):
|
||||
"""One agent seen in the caller's traces, for picking which agent's runs to look at."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
name: str
|
||||
runs: int = Field(ge=0)
|
||||
failed_runs: int = Field(ge=0)
|
||||
last_seen: datetime
|
||||
frameworks: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class TraceAgentList(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
agents: tuple[TraceAgent, ...]
|
||||
|
||||
|
||||
class SpendLogRecord(TypedDict):
|
||||
"""One LiteLLM request, as written by the `clickhouse` logging callback."""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"enum": [
|
||||
"trace_agents",
|
||||
"availability",
|
||||
"agents",
|
||||
"sample",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"properties": {
|
||||
"agent_name": {
|
||||
"type": "string"
|
||||
},
|
||||
"failed_runs": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uint64",
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"x-python-normalized": {
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"frameworks": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"last_seen_ms": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uint64",
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"x-python-normalized": {
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "int"
|
||||
}
|
||||
},
|
||||
"runs": {
|
||||
"anyOf": [
|
||||
{
|
||||
"format": "uint64",
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$",
|
||||
"type": "string"
|
||||
}
|
||||
],
|
||||
"x-python-normalized": {
|
||||
"maximum": 18446744073709551615,
|
||||
"minimum": 0,
|
||||
"type": "int"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"agent_name",
|
||||
"runs",
|
||||
"failed_runs",
|
||||
"last_seen_ms"
|
||||
],
|
||||
"title": "TraceAgentRow",
|
||||
"type": "object"
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"additionalProperties": false,
|
||||
"description": "Same access shape as `list_traces`: every team, the caller's own traces, or their teams' traces.",
|
||||
"properties": {
|
||||
"all_teams": {
|
||||
"enum": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"type": "integer"
|
||||
},
|
||||
"end_ms": {
|
||||
"format": "int64",
|
||||
"maximum": 9223372036854775807,
|
||||
"minimum": -9223372036854775808,
|
||||
"type": "integer"
|
||||
},
|
||||
"limit": {
|
||||
"format": "uint32",
|
||||
"maximum": 4294967295,
|
||||
"minimum": 0,
|
||||
"type": "integer"
|
||||
},
|
||||
"start_ms": {
|
||||
"format": "int64",
|
||||
"maximum": 9223372036854775807,
|
||||
"minimum": -9223372036854775808,
|
||||
"type": "integer"
|
||||
},
|
||||
"team_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"user_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"all_teams",
|
||||
"user_id",
|
||||
"team_ids",
|
||||
"start_ms",
|
||||
"end_ms",
|
||||
"limit"
|
||||
],
|
||||
"title": "TraceAgentsParams",
|
||||
"type": "object"
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py).
|
|||
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from types import ModuleType
|
||||
from typing import Final, Literal, TypedDict
|
||||
from unittest.mock import AsyncMock, MagicMock, call
|
||||
|
|
@ -15,7 +16,7 @@ from httpx import Response
|
|||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
from litellm.constants import TRACE_READ_RETRY_AFTER_SECONDS
|
||||
from litellm.constants import DEFAULT_AGENT_TRACING_RETENTION_DAYS, TRACE_READ_RETRY_AFTER_SECONDS
|
||||
from litellm.proxy import tracing_endpoints
|
||||
from litellm.proxy._types import LitellmUserRoles, ProxyLifespanState, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.authorization import OwnedRows, ReadScope
|
||||
|
|
@ -29,6 +30,7 @@ from litellm.rust_bridge.trace.generated.responses import TraceSQLResponse
|
|||
from litellm.rust_bridge.trace.generated.types import AllQueryScope, TraceScope
|
||||
from litellm.rust_bridge.trace.storage import ClickHouseStorage, TraceStorageConfig
|
||||
from litellm.tracing import Tenant, TraceReceiver, TracingPayloadTooLargeError
|
||||
from litellm.tracing.types import TraceAgent, TraceAgentList
|
||||
|
||||
SQL_ROWS: Final[tuple[Mapping[str, JsonValue], ...]] = (
|
||||
{
|
||||
|
|
@ -198,6 +200,7 @@ def receiver(client) -> MagicMock:
|
|||
fake.list_traces = AsyncMock(return_value={"data": [], "next_cursor": None})
|
||||
fake.get_trace = AsyncMock(return_value=None)
|
||||
fake.get_span = AsyncMock(return_value=None)
|
||||
fake.list_agents = AsyncMock(return_value=TraceAgentList(agents=()))
|
||||
client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: fake
|
||||
return fake
|
||||
|
||||
|
|
@ -324,6 +327,69 @@ def test_list_traces_resolves_default_bounds_from_injected_clock(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected_start_ms", "expected_end_ms"),
|
||||
(
|
||||
({}, NOW_MS - DEFAULT_AGENT_TRACING_RETENTION_DAYS * tracing_endpoints.MS_PER_DAY, NOW_MS),
|
||||
({"start_ms": 123, "end_ms": 456}, 123, 456),
|
||||
),
|
||||
)
|
||||
def test_list_trace_agents_passes_reader_scope_and_window(
|
||||
client: TestClient,
|
||||
receiver: MagicMock,
|
||||
params: Mapping[str, int],
|
||||
expected_start_ms: int,
|
||||
expected_end_ms: int,
|
||||
) -> None:
|
||||
client.app.dependency_overrides[tracing_endpoints.current_time_ms] = lambda: NOW_MS
|
||||
receiver.list_agents.return_value = TraceAgentList(
|
||||
agents=(
|
||||
TraceAgent(
|
||||
name="moyai",
|
||||
runs=3,
|
||||
failed_runs=1,
|
||||
last_seen=datetime(2026, 10, 7, 20, 31, tzinfo=timezone.utc),
|
||||
frameworks=("openai-agents",),
|
||||
),
|
||||
)
|
||||
)
|
||||
response: Final = client.get("/v1/traces/agents", params=params)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {
|
||||
"agents": [
|
||||
{
|
||||
"name": "moyai",
|
||||
"runs": 3,
|
||||
"failed_runs": 1,
|
||||
"last_seen": "2026-10-07T20:31:00Z",
|
||||
"frameworks": ["openai-agents"],
|
||||
}
|
||||
]
|
||||
}
|
||||
receiver.list_agents.assert_awaited_once_with(
|
||||
scope={"all_teams": 0, "user_id": "user", "team_ids": ()},
|
||||
start_ms=expected_start_ms,
|
||||
end_ms=expected_end_ms,
|
||||
)
|
||||
receiver.get_trace.assert_not_awaited()
|
||||
|
||||
|
||||
def test_list_trace_agents_requires_read_access(client: TestClient, receiver: MagicMock) -> None:
|
||||
client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER
|
||||
)
|
||||
response: Final = client.get("/v1/traces/agents")
|
||||
assert response.status_code == 403, response.text
|
||||
receiver.list_agents.assert_not_awaited()
|
||||
|
||||
|
||||
def test_list_trace_agents_maps_storage_outage_to_503(client: TestClient, receiver: MagicMock) -> None:
|
||||
receiver.list_agents.side_effect = RuntimeError("private database details")
|
||||
response: Final = client.get("/v1/traces/agents")
|
||||
assert response.status_code == 503
|
||||
assert response.json()["detail"]["code"] == "unavailable"
|
||||
|
||||
|
||||
def test_list_traces_forwards_large_and_negative_bounds_unchanged(client: TestClient, receiver: MagicMock) -> None:
|
||||
response: Final = client.get("/v1/traces", params={"start_ms": 2**63, "end_ms": -1, "cursor": "next"})
|
||||
assert response.status_code == 200, response.text
|
||||
|
|
|
|||
71
tests/unit/tracing/test_receiver.py
Normal file
71
tests/unit/tracing/test_receiver.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from datetime import datetime, timezone
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import AGENT_TRACING_AGENT_LIST_LIMIT
|
||||
from litellm.rust_bridge.trace.generated.models import TraceAgentRow, TraceAgentsParams
|
||||
from litellm.rust_bridge.trace.generated.types import TraceScope
|
||||
from litellm.rust_bridge.trace.storage import ClickHouseStorage
|
||||
from litellm.tracing import TraceReceiver
|
||||
from litellm.tracing.types import TraceAgent
|
||||
|
||||
|
||||
class AgentRowsStorage:
|
||||
def __init__(self, rows: tuple[TraceAgentRow, ...]) -> None:
|
||||
self.rows: Final = rows
|
||||
self.requests: tuple[TraceAgentsParams, ...] = ()
|
||||
|
||||
async def trace_agents(self, parameters: TraceAgentsParams) -> tuple[TraceAgentRow, ...]:
|
||||
self.requests = (*self.requests, parameters)
|
||||
return self.rows
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("scope", "all_teams", "user_id", "team_ids"),
|
||||
(
|
||||
pytest.param(TraceScope(all_teams=1, user_id="", team_ids=()), 1, "", (), id="all-teams"),
|
||||
pytest.param(TraceScope(all_teams=0, user_id="u1", team_ids=("t1", "t2")), 0, "u1", ("t1", "t2"), id="owned"),
|
||||
),
|
||||
)
|
||||
async def test_list_agents_queries_the_reader_scope_and_shapes_rows(
|
||||
scope: TraceScope, all_teams: int, user_id: str, team_ids: tuple[str, ...]
|
||||
) -> None:
|
||||
storage: Final = AgentRowsStorage(
|
||||
(
|
||||
TraceAgentRow(
|
||||
agent_name="moyai", runs=5, failed_runs=2, last_seen_ms=1_791_405_060_123, frameworks=("pi",)
|
||||
),
|
||||
TraceAgentRow(agent_name="research", runs=1, failed_runs=0, last_seen_ms=0),
|
||||
)
|
||||
)
|
||||
receiver: Final = TraceReceiver(storage=cast(ClickHouseStorage, storage))
|
||||
|
||||
result: Final = await receiver.list_agents(scope, start_ms=10, end_ms=20)
|
||||
|
||||
assert storage.requests == (
|
||||
TraceAgentsParams(
|
||||
all_teams=all_teams,
|
||||
user_id=user_id,
|
||||
team_ids=team_ids,
|
||||
start_ms=10,
|
||||
end_ms=20,
|
||||
limit=AGENT_TRACING_AGENT_LIST_LIMIT,
|
||||
),
|
||||
)
|
||||
assert result.agents == (
|
||||
TraceAgent(
|
||||
name="moyai",
|
||||
runs=5,
|
||||
failed_runs=2,
|
||||
last_seen=datetime(2026, 10, 7, 20, 31, 0, 123000, tzinfo=timezone.utc),
|
||||
frameworks=("pi",),
|
||||
),
|
||||
TraceAgent(
|
||||
name="research",
|
||||
runs=1,
|
||||
failed_runs=0,
|
||||
last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||
),
|
||||
)
|
||||
|
|
@ -6,6 +6,7 @@ import { readRequest, requestPath } from "@/../tests/lens-test-utils";
|
|||
import { LensWorkspace } from "./LensWorkspace";
|
||||
import { createLensDemoData } from "./data/demo/fixtures";
|
||||
import type { LensList } from "./model/types";
|
||||
import { rollUpAgents } from "./agents/agentRollup";
|
||||
|
||||
const network = vi.fn<typeof fetch>();
|
||||
const list = vi.fn<() => Promise<LensList>>();
|
||||
|
|
@ -26,6 +27,8 @@ function serve({ enabled = false, traces = false, requests = false, connected =
|
|||
return enabled
|
||||
? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] })
|
||||
: Response.json({ detail: "Tracing is not enabled" }, { status: 501 });
|
||||
if (path === "/v1/traces/agents")
|
||||
return Response.json({ agents: traces ? rollUpAgents([data.runs[0].trace.summary]) : [] });
|
||||
if (path === "/lens/activity/available") return Response.json({ traces, requests });
|
||||
if (path === "/lens/traces/findings") return Response.json([]);
|
||||
if (path === "/lens" && method === "POST") {
|
||||
|
|
@ -305,8 +308,10 @@ describe("Lens setup journey", () => {
|
|||
within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Investigations" }),
|
||||
).toHaveAttribute("aria-selected", "true");
|
||||
await waitFor(() => expect(setupParam(onUrlUpdate)).toBeNull());
|
||||
const requests = await Promise.all(network.mock.calls.map(([input, init]) => readRequest(input, init)));
|
||||
const create = requests.find((request) => request.path === "/lens" && request.method === "POST");
|
||||
const creates = network.mock.calls.filter(
|
||||
([input, init]) => requestPath(input) === "/lens" && (init?.method ?? (input as Request).method) === "POST",
|
||||
);
|
||||
const [create] = await Promise.all(creates.map(([input, init]) => readRequest(input, init)));
|
||||
expect(create).toBeDefined();
|
||||
expect(create?.body).toEqual(expect.objectContaining({ name: "My first review", source }));
|
||||
},
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ describe("Lens interactive demo", () => {
|
|||
expect([...url.entries()]).toEqual([
|
||||
["tab", "traces"],
|
||||
["demo", "true"],
|
||||
["agent", "support_agent"],
|
||||
]),
|
||||
);
|
||||
expect(screen.queryByText(/Could not load trace/)).not.toBeInTheDocument();
|
||||
|
|
@ -500,3 +501,47 @@ it("keeps trace quick filters in links and clears them when leaving demo data",
|
|||
await user.click(screen.getByRole("switch", { name: "Demo data" }));
|
||||
await expectUrl(onUrlUpdate, (url) => expect([...url.keys()]).toEqual(["tab"]));
|
||||
});
|
||||
|
||||
describe("Lens agent selector", () => {
|
||||
it("scopes traces to one agent, switches from the header, and reopens the pick after a refresh", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn();
|
||||
const first = renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
|
||||
searchParams: "?demo=true",
|
||||
onUrlUpdate,
|
||||
});
|
||||
const picker = await screen.findByRole("button", { name: "Agent: support_agent" });
|
||||
const runs = await screen.findByRole("table", { name: "Agent runs" });
|
||||
expect(await within(runs).findByText("Where is order #1042?")).toBeVisible();
|
||||
expect(screen.queryByRole("combobox", { name: "Filter traces by agent" })).not.toBeInTheDocument();
|
||||
|
||||
await user.click(picker);
|
||||
await user.type(screen.getByRole("textbox", { name: "Find agent" }), "release");
|
||||
const options = screen.getByRole("list", { name: "Agents" });
|
||||
expect(
|
||||
within(options)
|
||||
.getAllByRole("button")
|
||||
.map((button) => button.textContent),
|
||||
).toEqual([expect.stringContaining("release_agent")]);
|
||||
await user.click(within(options).getByRole("button", { name: /release_agent/ }));
|
||||
expect(await screen.findByRole("button", { name: "Agent: release_agent" })).toBeVisible();
|
||||
await waitFor(() => expect(within(runs).queryByText("Where is order #1042?")).not.toBeInTheDocument());
|
||||
await expectUrl(onUrlUpdate, (url) => expect(url.get("agent")).toBe("release_agent"));
|
||||
expect(window.localStorage.getItem("litellm.lens.agent.demo")).toBe("release_agent");
|
||||
expect(window.localStorage.getItem("litellm.lens.agent")).toBeNull();
|
||||
|
||||
first.unmount();
|
||||
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
|
||||
searchParams: "?demo=true",
|
||||
});
|
||||
expect(await screen.findByRole("button", { name: "Agent: release_agent" })).toBeVisible();
|
||||
});
|
||||
|
||||
it("lets a shared link choose the agent over the remembered one", async () => {
|
||||
window.localStorage.setItem("litellm.lens.agent.demo", "release_agent");
|
||||
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
|
||||
searchParams: "?demo=true&agent=research_agent",
|
||||
});
|
||||
expect(await screen.findByRole("button", { name: "Agent: research_agent" })).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import { LensGettingStarted } from "./onboarding/LensGettingStarted";
|
|||
import { useLensReadiness, type LensReadiness } from "./hooks/useLensReadiness";
|
||||
import { OnboardingProvider, type Onboarding } from "./onboarding/OnboardingContext";
|
||||
import { traceRefOf, useOpenTraceRouting, type TraceRef } from "@/components/lens/traces/routing";
|
||||
import { AgentBreadcrumb, useLensAgents } from "./agents/AgentScoped";
|
||||
|
||||
type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean };
|
||||
|
||||
|
|
@ -85,6 +86,7 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
|
|||
const { dialog, openDialog } = useDialogRoute();
|
||||
const { issueKey } = useIssueRoute();
|
||||
const { trace, openTrace } = useOpenTraceRouting();
|
||||
const agents = useLensAgents(accessToken);
|
||||
const canViewInvestigations = isProxyAdminTierRole(userRole);
|
||||
const isAdmin = isProxyAdminRole(userRole);
|
||||
const canConfigure = canViewInvestigations && !readOnly;
|
||||
|
|
@ -155,10 +157,13 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
|
|||
className="@container/lens-frame min-h-0 flex-1 gap-0"
|
||||
>
|
||||
<header className="grid shrink-0 grid-cols-[1fr_auto] items-center gap-x-4 gap-y-2 border-b pb-2 @min-[42rem]/lens-frame:grid-cols-[auto_1fr_auto]">
|
||||
<h1 className="flex items-center gap-1.5 text-sm font-semibold">
|
||||
<Aperture aria-hidden="true" className="size-4" strokeWidth={2} />
|
||||
Lens
|
||||
</h1>
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<h1 className="flex items-center gap-1.5 text-sm font-semibold">
|
||||
<Aperture aria-hidden="true" className="size-4" strokeWidth={2} />
|
||||
Lens
|
||||
</h1>
|
||||
<AgentBreadcrumb agents={agents} />
|
||||
</div>
|
||||
<div className="col-span-2 row-start-2 min-w-0 @min-[42rem]/lens-frame:col-span-1 @min-[42rem]/lens-frame:col-start-2 @min-[42rem]/lens-frame:row-start-1">
|
||||
<LensModeSwitch activity={activity} workers={workers} />
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
"use client";
|
||||
|
||||
import { Bot, Check, ChevronsUpDown, Search } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import { FrameworkLogo, traceFramework } from "../traces/ui/TraceFramework";
|
||||
import type { AgentSummary } from "./agentRollup";
|
||||
|
||||
export function AgentMark({ agent }: { agent: Pick<AgentSummary, "frameworks"> | undefined }) {
|
||||
const framework = agent ? traceFramework({ frameworks: [...agent.frameworks] }) : null;
|
||||
return framework ? (
|
||||
<FrameworkLogo framework={framework} className="size-4" />
|
||||
) : (
|
||||
<Bot aria-hidden className="size-4 shrink-0 text-muted-foreground" />
|
||||
);
|
||||
}
|
||||
|
||||
export const matchesAgent = (agent: Pick<AgentSummary, "name">, query: string): boolean =>
|
||||
agent.name.toLowerCase().includes(query.trim().toLowerCase());
|
||||
|
||||
interface AgentPickerProps {
|
||||
agent: string;
|
||||
agents: readonly AgentSummary[];
|
||||
onSelect: (agent: string) => void;
|
||||
}
|
||||
|
||||
const ITEM = "flex w-full items-center gap-2 rounded-md px-2 py-1.5 text-left text-sm hover:bg-muted";
|
||||
|
||||
/** The agent every Lens view is scoped to, like the project switcher in Braintrust. */
|
||||
export function AgentPicker({ agent, agents, onSelect }: AgentPickerProps) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [query, setQuery] = useState("");
|
||||
const shown = agents.filter((item) => matchesAgent(item, query));
|
||||
const choose = (next: string) => {
|
||||
setOpen(false);
|
||||
setQuery("");
|
||||
onSelect(next);
|
||||
};
|
||||
return (
|
||||
<Popover open={open} onOpenChange={setOpen}>
|
||||
<PopoverTrigger
|
||||
aria-label={`Agent: ${agent}`}
|
||||
className="inline-flex h-7 max-w-56 items-center gap-1.5 rounded-md border px-2 text-sm font-medium hover:bg-muted"
|
||||
>
|
||||
<AgentMark agent={agents.find((item) => item.name === agent)} />
|
||||
<span className="truncate">{agent}</span>
|
||||
<ChevronsUpDown aria-hidden className="size-3.5 shrink-0 text-muted-foreground" />
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align="start" className="w-72 gap-1 p-1.5">
|
||||
<div className="relative">
|
||||
<Search aria-hidden className="absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted-foreground" />
|
||||
<Input
|
||||
aria-label="Find agent"
|
||||
placeholder="Find agent"
|
||||
autoFocus
|
||||
value={query}
|
||||
onChange={(event) => setQuery(event.target.value)}
|
||||
className="h-8 border-0 pl-8 text-sm shadow-none focus-visible:ring-0"
|
||||
/>
|
||||
</div>
|
||||
<div className="border-t" />
|
||||
<p className="px-2 pt-1.5 pb-0.5 text-xs text-muted-foreground">Agents</p>
|
||||
<ul aria-label="Agents" className="flex max-h-72 flex-col overflow-y-auto">
|
||||
{shown.map((item) => (
|
||||
<li key={item.name}>
|
||||
<button type="button" className={ITEM} onClick={() => choose(item.name)}>
|
||||
<Check aria-hidden className={cn("size-3.5 shrink-0", item.name !== agent && "invisible")} />
|
||||
<AgentMark agent={item} />
|
||||
<span className="truncate">{item.name}</span>
|
||||
<span className="ml-auto text-xs text-muted-foreground tabular-nums">{item.runs.toLocaleString()}</span>
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
{shown.length === 0 && <li className="px-2 py-1.5 text-sm text-muted-foreground">No agents match</li>}
|
||||
</ul>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
"use client";
|
||||
|
||||
import { useTracesLive } from "../traces/api";
|
||||
import { AgentPicker } from "./AgentPicker";
|
||||
import { useAgents } from "./useAgents";
|
||||
import { useAgentSelection } from "./useAgentSelection";
|
||||
|
||||
export interface LensAgents {
|
||||
readonly agent: string | null;
|
||||
select(agent: string): void;
|
||||
readonly list: ReturnType<typeof useAgents>;
|
||||
}
|
||||
|
||||
export function useLensAgents(accessToken: string): LensAgents {
|
||||
const list = useAgents(accessToken);
|
||||
const { agent, select } = useAgentSelection(
|
||||
!useTracesLive(),
|
||||
list.agents.map((item) => item.name),
|
||||
);
|
||||
return { agent, select, list };
|
||||
}
|
||||
|
||||
/** `Lens / agent ▾`, shown whenever there is an agent to scope to. */
|
||||
export function AgentBreadcrumb({ agents }: { agents: LensAgents }) {
|
||||
if (!agents.agent) return null;
|
||||
return (
|
||||
<>
|
||||
<span aria-hidden className="text-muted-foreground">
|
||||
/
|
||||
</span>
|
||||
<AgentPicker agent={agents.agent} agents={agents.list.agents} onSelect={agents.select} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import type { TraceAgent, TraceSummary } from "../traces/types";
|
||||
import { traceAgentNames } from "../traces/utils";
|
||||
|
||||
export type AgentSummary = TraceAgent;
|
||||
|
||||
const failed = (trace: TraceSummary): boolean => trace.status === "error" || trace.error_count > 0;
|
||||
|
||||
const latest = (times: readonly string[]): string => times.reduce((a, b) => (Date.parse(a) >= Date.parse(b) ? a : b));
|
||||
|
||||
/** One row per agent across the given runs, newest activity first; mirrors what `/v1/traces/agents` returns. */
|
||||
export function rollUpAgents(traces: readonly TraceSummary[]): AgentSummary[] {
|
||||
const names = [...new Set(traces.flatMap(traceAgentNames))];
|
||||
return names
|
||||
.map((name) => {
|
||||
const runs = traces.filter((trace) => traceAgentNames(trace).includes(name));
|
||||
return {
|
||||
name,
|
||||
runs: runs.length,
|
||||
failed_runs: runs.filter(failed).length,
|
||||
last_seen: latest(runs.map((trace) => trace.start_time)),
|
||||
frameworks: [...new Set(runs.flatMap((trace) => trace.frameworks ?? []))].sort(),
|
||||
};
|
||||
})
|
||||
.sort((a, b) => Date.parse(b.last_seen) - Date.parse(a.last_seen) || a.name.localeCompare(b.name));
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { TraceSummary } from "../traces/types";
|
||||
import { rollUpAgents } from "./agentRollup";
|
||||
import { matchesAgent } from "./AgentPicker";
|
||||
import { resolveAgent } from "./useAgentSelection";
|
||||
|
||||
const summary = (overrides: Partial<TraceSummary>): TraceSummary =>
|
||||
({
|
||||
trace_id: "t",
|
||||
name: "run",
|
||||
service: "svc",
|
||||
agent_names: [],
|
||||
frameworks: [],
|
||||
input_preview: "",
|
||||
start_time: "2026-10-07T12:00:00+00:00",
|
||||
duration_ms: 1,
|
||||
status: "ok",
|
||||
span_count: 1,
|
||||
agent_count: 1,
|
||||
agent_invocations: 1,
|
||||
llm_calls: 0,
|
||||
tool_calls: 0,
|
||||
error_count: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
models: [],
|
||||
spend: null,
|
||||
priced_calls: 0,
|
||||
...overrides,
|
||||
}) as TraceSummary;
|
||||
|
||||
describe("resolveAgent", () => {
|
||||
const available = ["moyai", "researcher", "writer"];
|
||||
|
||||
it("lets a shared link pick the agent", () => {
|
||||
expect(resolveAgent("writer", "moyai", available)).toBe("writer");
|
||||
});
|
||||
|
||||
it("reopens the agent this browser picked last", () => {
|
||||
expect(resolveAgent("", "researcher", available)).toBe("researcher");
|
||||
});
|
||||
|
||||
it("falls back to the most recently active agent when the remembered one is gone", () => {
|
||||
expect(resolveAgent("", "retired", available)).toBe("moyai");
|
||||
});
|
||||
|
||||
it("opens the most recently active agent on a first visit", () => {
|
||||
expect(resolveAgent("", "", available)).toBe("moyai");
|
||||
});
|
||||
|
||||
it("has no agent to scope to before any traces arrive", () => {
|
||||
expect(resolveAgent("", "moyai", [])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("rollUpAgents", () => {
|
||||
it("counts runs and failures per agent and orders by latest activity", () => {
|
||||
const agents = rollUpAgents([
|
||||
summary({ agent_names: ["moyai"], start_time: "2026-10-07T10:00:00+00:00" }),
|
||||
summary({ agent_names: ["moyai", "researcher"], status: "error", start_time: "2026-10-07T12:00:00+00:00" }),
|
||||
summary({ agent_names: ["researcher"], error_count: 2, start_time: "2026-10-07T13:00:00+00:00" }),
|
||||
summary({ agent_names: ["writer"], frameworks: ["langgraph"], start_time: "2026-10-07T09:00:00+00:00" }),
|
||||
]);
|
||||
expect(agents).toEqual([
|
||||
{ name: "researcher", runs: 2, failed_runs: 2, last_seen: "2026-10-07T13:00:00+00:00", frameworks: [] },
|
||||
{ name: "moyai", runs: 2, failed_runs: 1, last_seen: "2026-10-07T12:00:00+00:00", frameworks: [] },
|
||||
{ name: "writer", runs: 1, failed_runs: 0, last_seen: "2026-10-07T09:00:00+00:00", frameworks: ["langgraph"] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("matchesAgent", () => {
|
||||
it("finds agents by a case-insensitive part of the name", () => {
|
||||
expect(matchesAgent({ name: "Support-Bot" }, " support")).toBe(true);
|
||||
expect(matchesAgent({ name: "moyai" }, "research")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
"use client";
|
||||
|
||||
import { parseAsString, useQueryStates } from "nuqs";
|
||||
import { useCallback, useEffect } from "react";
|
||||
import { useLocalStorage } from "usehooks-ts";
|
||||
|
||||
const SELECTED_AGENT_KEY = "litellm.lens.agent";
|
||||
export const selectedAgentKey = (demo: boolean): string => (demo ? `${SELECTED_AGENT_KEY}.demo` : SELECTED_AGENT_KEY);
|
||||
|
||||
const AGENT_PARSERS = { agent: parseAsString.withDefault("") };
|
||||
|
||||
/**
|
||||
* Traces are always scoped to one agent: a shared link's agent first, then this browser's last pick if it still
|
||||
* exists, then the most recently active agent.
|
||||
*/
|
||||
export function resolveAgent(fromUrl: string, remembered: string, available: readonly string[]): string | null {
|
||||
if (fromUrl) return fromUrl;
|
||||
if (remembered && available.includes(remembered)) return remembered;
|
||||
return available[0] ?? null;
|
||||
}
|
||||
|
||||
export interface AgentSelection {
|
||||
readonly agent: string | null;
|
||||
select(agent: string): void;
|
||||
}
|
||||
|
||||
/** In the URL for sharing, and remembered per browser (separately for the sample session) across refresh and login. */
|
||||
export function useAgentSelection(demo: boolean, available: readonly string[]): AgentSelection {
|
||||
const [{ agent: fromUrl }, setParams] = useQueryStates(AGENT_PARSERS, { history: "push" });
|
||||
const [remembered, setRemembered] = useLocalStorage(selectedAgentKey(demo), "", {
|
||||
serializer: (value) => value,
|
||||
deserializer: (raw) => raw,
|
||||
});
|
||||
const agent = resolveAgent(fromUrl, remembered, available);
|
||||
const implied = !fromUrl ? agent : null;
|
||||
useEffect(() => {
|
||||
if (implied) void setParams({ agent: implied }, { history: "replace" });
|
||||
}, [implied, setParams]);
|
||||
const select = useCallback(
|
||||
(next: string) => {
|
||||
setRemembered(next);
|
||||
void setParams({ agent: next });
|
||||
},
|
||||
[setParams, setRemembered],
|
||||
);
|
||||
return { agent, select };
|
||||
}
|
||||
27
ui/litellm-dashboard/src/components/lens/agents/useAgents.ts
Normal file
27
ui/litellm-dashboard/src/components/lens/agents/useAgents.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
import { useTracesApi } from "../traces/api";
|
||||
import type { AgentSummary } from "./agentRollup";
|
||||
|
||||
export const AGENT_WINDOW_DAYS = 14;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** Agents seen in the last two weeks, matching the default Braintrust project window. */
|
||||
export function useAgents(accessToken: string): {
|
||||
agents: AgentSummary[];
|
||||
isLoading: boolean;
|
||||
error: Error | null;
|
||||
} {
|
||||
const traces = useTracesApi(accessToken);
|
||||
const { data, isLoading, error } = useQuery({
|
||||
queryKey: ["lensAgents", accessToken, traces.live],
|
||||
queryFn: () => {
|
||||
const endMs = Date.now();
|
||||
return traces.agents({ startMs: endMs - AGENT_WINDOW_DAYS * DAY_MS, endMs });
|
||||
},
|
||||
staleTime: 60_000,
|
||||
});
|
||||
return { agents: data ?? [], isLoading, error };
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { ApiError } from "@/lib/http/client";
|
||||
import type { TracesApi } from "@/components/lens/traces/api";
|
||||
import { rollUpAgents } from "@/components/lens/agents/agentRollup";
|
||||
import type { LensServices } from "../LensServices";
|
||||
import type { LensApi } from "../service";
|
||||
import { demoDatasetsApi } from "./demoDatasets";
|
||||
|
|
@ -49,6 +50,11 @@ function demoLensApi(data: LensDemoData): LensApi {
|
|||
};
|
||||
}
|
||||
|
||||
const summariesIn = (data: LensDemoData, startMs: number, endMs: number) =>
|
||||
data.runs
|
||||
.map((item) => item.trace.summary)
|
||||
.filter((trace) => Date.parse(trace.start_time) >= startMs && Date.parse(trace.start_time) <= endMs);
|
||||
|
||||
function demoTracesApi(data: LensDemoData): TracesApi {
|
||||
const run = (traceId: string) => data.runs.find(({ trace }) => trace.summary.trace_id === traceId);
|
||||
return {
|
||||
|
|
@ -58,12 +64,8 @@ function demoTracesApi(data: LensDemoData): TracesApi {
|
|||
const step = spanId ? found?.details.find((span) => span.span_id === spanId) : found;
|
||||
return { text: JSON.stringify(step, null, 2), copied: spanId ? "Step copied" : "Trace copied" };
|
||||
},
|
||||
list: async ({ startMs, endMs }) => ({
|
||||
data: data.runs
|
||||
.map((item) => item.trace.summary)
|
||||
.filter((trace) => Date.parse(trace.start_time) >= startMs && Date.parse(trace.start_time) <= endMs),
|
||||
next_cursor: null,
|
||||
}),
|
||||
list: async ({ startMs, endMs }) => ({ data: summariesIn(data, startMs, endMs), next_cursor: null }),
|
||||
agents: async ({ startMs, endMs }) => rollUpAgents(summariesIn(data, startMs, endMs)),
|
||||
findings: async (traces) =>
|
||||
traces.map((trace) => {
|
||||
const jobs = data.lenses.flatMap((lens) => lens.jobs).filter((job) => job.status === "completed");
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
apiClient,
|
||||
getProxyBaseUrl,
|
||||
} from "../../networking";
|
||||
import type { AgentSummary } from "../agents/agentRollup";
|
||||
import type {
|
||||
SpanDetail,
|
||||
SpanQuery,
|
||||
|
|
@ -20,6 +21,8 @@ import type {
|
|||
TraceFindingCount,
|
||||
TraceFindingsRequest,
|
||||
TraceSignals,
|
||||
TraceAgentList,
|
||||
TraceAgentsQuery,
|
||||
} from "./types";
|
||||
|
||||
export interface TraceWindow {
|
||||
|
|
@ -38,6 +41,7 @@ export interface TracesApi {
|
|||
readonly live: boolean;
|
||||
handoff(traceId: string, spanId?: string | null, traceRef?: string): TraceHandoff;
|
||||
list(window: TraceWindow): Promise<TracePage>;
|
||||
agents(window: TraceWindow): Promise<AgentSummary[]>;
|
||||
findings(traces: TraceFindingsRequest["traces"]): Promise<TraceFindingCount[]>;
|
||||
signals(traces: TraceFindingsRequest["traces"]): Promise<TraceSignals[]>;
|
||||
anyRecorded(): Promise<boolean>;
|
||||
|
|
@ -79,6 +83,13 @@ export function liveTracesApi(accessToken: string): TracesApi {
|
|||
copied: "Command copied",
|
||||
}),
|
||||
list: (window) => agentTraceListCall({ accessToken, ...window }),
|
||||
agents: async ({ startMs, endMs }) => {
|
||||
const page = await apiClient.get<TraceAgentList>("/v1/traces/agents", {
|
||||
accessToken,
|
||||
query: { start_ms: startMs, end_ms: endMs } satisfies TraceAgentsQuery,
|
||||
});
|
||||
return page.agents ?? [];
|
||||
},
|
||||
findings: (traces) =>
|
||||
apiClient.post<TraceFindingCount[]>("/lens/traces/findings", {
|
||||
accessToken,
|
||||
|
|
|
|||
|
|
@ -631,7 +631,7 @@ describe("AgentTracesPage", () => {
|
|||
const live = screen.getByRole("button", { name: "Live" });
|
||||
expect(live).toHaveAttribute("aria-pressed", "true");
|
||||
expect(trigger).toHaveTextContent("Last 24 hours");
|
||||
expect(screen.getByRole("combobox", { name: "Filter traces by agent" })).toBeVisible();
|
||||
expect(screen.queryByRole("combobox", { name: "Filter traces by agent" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("combobox", { name: "Filter traces by status" })).toBeVisible();
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Refresh traces" })).toBeEnabled());
|
||||
expect(screen.getByRole("button", { name: "Set up tracing" })).toBeEnabled();
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { withNuqsTestingAdapter, type UrlUpdateEvent } from "nuqs/adapters/testing";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { runs } from "./__fixtures__/runs";
|
||||
import { RunsToolbar } from "./RunsToolbar";
|
||||
|
||||
const agentBox = () => screen.getByRole("combobox", { name: "Filter traces by agent" });
|
||||
|
||||
describe("RunsToolbar", () => {
|
||||
it("narrows the agent filter as you type and selects the match", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
render(<RunsToolbar query="" onQueryChange={vi.fn()} runs={runs} />, {
|
||||
wrapper: withNuqsTestingAdapter({ onUrlUpdate }),
|
||||
});
|
||||
expect(agentBox()).toHaveAttribute("placeholder", "All agents");
|
||||
await user.click(agentBox());
|
||||
await user.keyboard("tri");
|
||||
expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["triage"]);
|
||||
await user.click(screen.getByRole("option", { name: "triage" }));
|
||||
expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("agent")).toBe("triage");
|
||||
});
|
||||
|
||||
it("picks the first match on Enter", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
render(<RunsToolbar query="" onQueryChange={vi.fn()} runs={runs} />, {
|
||||
wrapper: withNuqsTestingAdapter({ onUrlUpdate }),
|
||||
});
|
||||
await user.click(agentBox());
|
||||
await user.keyboard("res{Enter}");
|
||||
expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("agent")).toBe("researcher");
|
||||
});
|
||||
|
||||
it("says when no agent matches", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<RunsToolbar query="" onQueryChange={vi.fn()} runs={runs} />, { wrapper: withNuqsTestingAdapter() });
|
||||
await user.click(agentBox());
|
||||
await user.keyboard("zzz");
|
||||
expect(screen.getByText("No matching agents")).toBeVisible();
|
||||
});
|
||||
|
||||
it("clears back to all agents", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>();
|
||||
render(<RunsToolbar query="" onQueryChange={vi.fn()} runs={runs} />, {
|
||||
wrapper: withNuqsTestingAdapter({ searchParams: "?agent=triage", onUrlUpdate }),
|
||||
});
|
||||
expect(agentBox()).toHaveValue("triage");
|
||||
await user.click(screen.getByRole("button", { name: "Clear" }));
|
||||
expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("agent")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -3,17 +3,8 @@
|
|||
import type { TraceSummary } from "../../types";
|
||||
import type { TimeWindow } from "@/components/shared/timeRange/timeRange";
|
||||
|
||||
import {
|
||||
Combobox,
|
||||
ComboboxContent,
|
||||
ComboboxEmpty,
|
||||
ComboboxInput,
|
||||
ComboboxItem,
|
||||
ComboboxList,
|
||||
} from "@/components/ui/combobox";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useRunFilterRouting } from "../../routing";
|
||||
import { traceAgentNames } from "../../utils";
|
||||
import { RunSearch } from "./RunSearch";
|
||||
|
||||
interface RunsToolbarProps {
|
||||
|
|
@ -29,8 +20,7 @@ interface RunsToolbarProps {
|
|||
}
|
||||
|
||||
export function RunsToolbar({ query, onQueryChange, runs, range, busy, children }: RunsToolbarProps) {
|
||||
const { agent, status, setAgent, setStatus } = useRunFilterRouting();
|
||||
const agents = [...new Set([...runs.flatMap(traceAgentNames), ...(agent ? [agent] : [])])].sort();
|
||||
const { status, setStatus } = useRunFilterRouting();
|
||||
const statuses = [
|
||||
{ value: "all", label: "All status" },
|
||||
{ value: "ok", label: "No errors" },
|
||||
|
|
@ -43,24 +33,6 @@ export function RunsToolbar({ query, onQueryChange, runs, range, busy, children
|
|||
{children && <div className="ml-auto flex h-10 max-w-full items-stretch">{children}</div>}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2 border-b px-3 py-2">
|
||||
<Combobox items={agents} value={agent || null} onValueChange={(name) => setAgent(name ?? "")} autoHighlight>
|
||||
<ComboboxInput
|
||||
aria-label="Filter traces by agent"
|
||||
placeholder="All agents"
|
||||
showClear={!!agent}
|
||||
className="h-8 w-48 max-w-full text-xs"
|
||||
/>
|
||||
<ComboboxContent>
|
||||
<ComboboxEmpty>No matching agents</ComboboxEmpty>
|
||||
<ComboboxList>
|
||||
{(name: string) => (
|
||||
<ComboboxItem key={name} value={name} className="text-xs">
|
||||
{name}
|
||||
</ComboboxItem>
|
||||
)}
|
||||
</ComboboxList>
|
||||
</ComboboxContent>
|
||||
</Combobox>
|
||||
<Select
|
||||
items={statuses}
|
||||
value={status}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ export type SpanErrorQuery = NonNullable<
|
|||
paths["/v1/traces/{trace_id}/spans/{span_id}/error"]["get"]["parameters"]["query"]
|
||||
>;
|
||||
export type TraceQueryBody = components["schemas"]["TraceQueryRequest"];
|
||||
export type TraceAgent = components["schemas"]["TraceAgent"];
|
||||
export type TraceAgentList = components["schemas"]["TraceAgentList"];
|
||||
export type TraceAgentsQuery = NonNullable<paths["/v1/traces/agents"]["get"]["parameters"]["query"]>;
|
||||
export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"];
|
||||
export type TraceFindingCount = components["schemas"]["TraceFindingCount"];
|
||||
export type TraceSignals = components["schemas"]["TraceSignals"];
|
||||
|
|
|
|||
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
76
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -22591,6 +22591,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/traces/agents": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** List Trace Agents */
|
||||
get: operations["list_trace_agents_v1_traces_agents_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/traces/query": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -47700,6 +47717,33 @@ export interface components {
|
|||
spans: components["schemas"]["Span"][];
|
||||
summary: components["schemas"]["TraceSummary"];
|
||||
};
|
||||
/**
|
||||
* TraceAgent
|
||||
* @description One agent seen in the caller's traces, for picking which agent's runs to look at.
|
||||
*/
|
||||
TraceAgent: {
|
||||
/** Failed Runs */
|
||||
failed_runs: number;
|
||||
/**
|
||||
* Frameworks
|
||||
* @default []
|
||||
*/
|
||||
frameworks: string[];
|
||||
/**
|
||||
* Last Seen
|
||||
* Format: date-time
|
||||
*/
|
||||
last_seen: string;
|
||||
/** Name */
|
||||
name: string;
|
||||
/** Runs */
|
||||
runs: number;
|
||||
};
|
||||
/** TraceAgentList */
|
||||
TraceAgentList: {
|
||||
/** Agents */
|
||||
agents: components["schemas"]["TraceAgent"][];
|
||||
};
|
||||
/** TraceFindingCount */
|
||||
TraceFindingCount: {
|
||||
/** Finding Count */
|
||||
|
|
@ -81287,6 +81331,38 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
list_trace_agents_v1_traces_agents_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
start_ms?: number | null;
|
||||
end_ms?: number | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["TraceAgentList"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
query_agent_traces_v1_traces_query_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ export const requestPath = (input: RequestInfo | URL): string =>
|
|||
|
||||
/** Reads a stubbed fetch call the same way whether a client passed a URL and init or a Request. */
|
||||
export async function readRequest(input: RequestInfo | URL, init?: RequestInit): Promise<StubbedRequest> {
|
||||
const request = input instanceof Request ? input : new Request(new URL(String(input), "http://localhost"), init);
|
||||
const request =
|
||||
input instanceof Request
|
||||
? input
|
||||
: new Request(new URL(String(input), "http://localhost"), { ...init, signal: undefined });
|
||||
const url = new URL(request.url);
|
||||
const text = await request.clone().text();
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue