mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(lens): show investigation findings for agent traces (#44696)
* fix(lens): show investigation findings for agent traces Replace child tool-error counts in the trace table with distinct investigation findings. Keep unassessed traces separate from completed clean investigations and use the root status for failure filters and timeline counts * fix(lens): stabilize findings updates and repair UI checks * fix(lens): allow viewer findings reads and index trace lookups * test(lens): cover viewer findings reads with Postgres
This commit is contained in:
moe-berri
GitHub
parent
65b0557f80
commit
c561372a02
30 changed files with 592 additions and 115 deletions
|
|
@ -0,0 +1,3 @@
|
|||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_LensRun_completed_executions_idx"
|
||||
ON "LiteLLM_LensRun" USING GIN ((data->'sample'->'executions') jsonb_path_ops)
|
||||
WHERE data->>'status'='completed';
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_jobs_idx"
|
||||
ON "LiteLLM_Lens" USING GIN ((data->'jobs') jsonb_path_ops);
|
||||
|
|
@ -1053,6 +1053,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# updating this list — the default-allow behavior covers it automatically.
|
||||
admin_viewer_routes = (
|
||||
[
|
||||
"/lens/traces/findings",
|
||||
"/user/list",
|
||||
"/user/available_users",
|
||||
"/user/available_roles",
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ from litellm.proxy.lens.models import (
|
|||
RunRequest,
|
||||
Sample,
|
||||
Scope,
|
||||
TraceFindingCount,
|
||||
TraceFindingsRequest,
|
||||
WatchAllResult,
|
||||
WatchSkipped,
|
||||
Worker,
|
||||
|
|
@ -240,6 +242,12 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
|
|||
return await source_reader(storage).agents(scope) if storage is not None else ()
|
||||
|
||||
|
||||
@router.post("/traces/findings", response_model=tuple[TraceFindingCount, ...])
|
||||
async def trace_findings(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceFindingCount, ...]:
|
||||
user_scope(auth)
|
||||
return await repository().trace_findings(body.traces)
|
||||
|
||||
|
||||
def watching(lens: Lens) -> Lens:
|
||||
if lens.settings.enabled:
|
||||
return lens
|
||||
|
|
|
|||
|
|
@ -193,6 +193,19 @@ class RunAssessment(Record):
|
|||
cannot_assess: bool = False
|
||||
|
||||
|
||||
class TraceIdentity(Record):
|
||||
trace_id: str = Field(min_length=1, max_length=128)
|
||||
trace_ref: str = Field(default="", max_length=512)
|
||||
|
||||
|
||||
class TraceFindingsRequest(Record):
|
||||
traces: tuple[TraceIdentity, ...] = Field(min_length=1, max_length=500)
|
||||
|
||||
|
||||
class TraceFindingCount(TraceIdentity):
|
||||
finding_count: int | None = Field(ge=0)
|
||||
|
||||
|
||||
MAX_STEPS = 200
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from typing import Final, Protocol
|
|||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
from litellm.proxy.db.prisma_client import PrismaWrapper
|
||||
from litellm.proxy.lens.models import Job, Lens, Scope, Worker
|
||||
from litellm.proxy.lens.models import Job, Lens, Scope, TraceFindingCount, TraceIdentity, Worker
|
||||
|
||||
|
||||
class Database(Protocol):
|
||||
|
|
@ -125,6 +125,50 @@ class LensRepository:
|
|||
)
|
||||
return Job.model_validate(rows[0].data) if rows else None
|
||||
|
||||
async def trace_findings(self, traces: tuple[TraceIdentity, ...]) -> tuple[TraceFindingCount, ...]:
|
||||
rows: Final = _ROWS.validate_python(
|
||||
await self.db.query_raw(
|
||||
"""WITH targets AS (
|
||||
SELECT DISTINCT trace_id, trace_ref,
|
||||
jsonb_build_array(jsonb_build_object('source', 'traces', 'trace_id', trace_id)) AS executions
|
||||
FROM jsonb_to_recordset($1::jsonb) AS target(trace_id text, trace_ref text)
|
||||
), jobs AS (
|
||||
SELECT target.trace_id, target.trace_ref, run.data AS job
|
||||
FROM targets AS target JOIN "LiteLLM_LensRun" AS run
|
||||
ON run.data->'sample'->'executions' @> target.executions
|
||||
WHERE run.data->>'status'='completed'
|
||||
UNION ALL
|
||||
SELECT target.trace_id, target.trace_ref, job
|
||||
FROM targets AS target JOIN "LiteLLM_Lens" AS lens
|
||||
ON lens.data->'jobs' @> jsonb_build_array(jsonb_build_object(
|
||||
'status', 'completed', 'sample', jsonb_build_object('executions', target.executions)))
|
||||
CROSS JOIN LATERAL jsonb_array_elements(lens.data->'jobs') AS job
|
||||
WHERE job->>'status'='completed'
|
||||
), assessed AS (
|
||||
SELECT jobs.trace_id, jobs.trace_ref, execution->>'id' AS execution_id, job
|
||||
FROM jobs, jsonb_array_elements(job->'sample'->'executions') AS execution
|
||||
WHERE execution->>'trace_id'=jobs.trace_id
|
||||
AND COALESCE(execution->>'trace_ref', '')=jobs.trace_ref
|
||||
AND execution->>'source'='traces' AND EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements(job->'assessments') AS assessment
|
||||
WHERE assessment->>'execution_id'=execution->>'id'
|
||||
AND COALESCE((assessment->>'cannot_assess')::boolean, false)=false
|
||||
)
|
||||
)
|
||||
SELECT jsonb_build_object(
|
||||
'trace_id', target.trace_id, 'trace_ref', target.trace_ref,
|
||||
'finding_count', CASE WHEN count(assessed.execution_id)=0 THEN NULL
|
||||
ELSE count(DISTINCT finding->>'id') END
|
||||
) AS data FROM targets AS target
|
||||
LEFT JOIN assessed USING (trace_id, trace_ref)
|
||||
LEFT JOIN LATERAL jsonb_array_elements(NULLIF(assessed.job->'findings', 'null'::jsonb)) AS finding
|
||||
ON finding->'occurrences' ? assessed.execution_id
|
||||
GROUP BY target.trace_id, target.trace_ref""",
|
||||
json.dumps(tuple(trace.model_dump() for trace in traces)),
|
||||
)
|
||||
)
|
||||
return tuple(TraceFindingCount.model_validate(row.data) for row in rows)
|
||||
|
||||
async def workers(self) -> tuple[Worker, ...]:
|
||||
rows: Final = _ROWS.validate_python(await self.db.query_raw('SELECT data FROM "LiteLLM_LensWorker"'))
|
||||
return tuple(Worker.model_validate(row.data) for row in rows)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import os
|
|||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from uuid import uuid4
|
||||
|
|
@ -13,8 +14,24 @@ import pytest_asyncio
|
|||
from prisma import Prisma
|
||||
from psycopg import sql
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.db.prisma_client import PrismaWrapper
|
||||
from litellm.proxy.lens.models import Check, Lens, LensSettings, Scope, Worker
|
||||
from litellm.proxy.lens.models import (
|
||||
Check,
|
||||
Evidence,
|
||||
Execution,
|
||||
Finding,
|
||||
Job,
|
||||
Lens,
|
||||
LensSettings,
|
||||
RunAssessment,
|
||||
Sample,
|
||||
Scope,
|
||||
TraceFindingCount,
|
||||
TraceFindingsRequest,
|
||||
TraceIdentity,
|
||||
Worker,
|
||||
)
|
||||
from litellm.proxy.lens.repository import LensRepository, WriterDatabase
|
||||
from litellm.proxy.lens.state import claim_job, queue_job
|
||||
|
||||
|
|
@ -69,6 +86,109 @@ async def test_heartbeat_never_restores_revoked_access(lens_db: Prisma) -> None:
|
|||
await lens_db.execute_raw('DELETE FROM "LiteLLM_LensWorker" WHERE id=$1', worker.id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trace_findings_include_archived_assessments_without_counting_retries_or_counterexamples(
|
||||
lens_db: Prisma,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.lens.endpoints import trace_findings
|
||||
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=lens_db))
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
prefix: Final = uuid4().hex
|
||||
repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db)))
|
||||
settings: Final = LensSettings(name="Finding counts", model="test", context="Answer the question")
|
||||
identities: Final = tuple(TraceIdentity(trace_id=prefix, trace_ref=f"{prefix}-{i}") for i in range(7))
|
||||
executions: Final = tuple(
|
||||
Execution(
|
||||
id=f"{prefix}-{i}",
|
||||
source="traces",
|
||||
trace_id=identity.trace_id,
|
||||
trace_ref=identity.trace_ref,
|
||||
team_id="",
|
||||
name="Run",
|
||||
start_time=now.isoformat(),
|
||||
span_count=1,
|
||||
)
|
||||
for i, identity in enumerate(identities)
|
||||
)
|
||||
finding: Final = Finding(
|
||||
id=prefix,
|
||||
title="Repeated lookup",
|
||||
description="The agent never answered the question",
|
||||
check_id="expected_behavior",
|
||||
first_seen=now,
|
||||
last_seen=now,
|
||||
revision=1,
|
||||
occurrences=(executions[0].id,),
|
||||
evidence=(
|
||||
Evidence(execution_id=executions[0].id, span_id="step", quote="no answer"),
|
||||
Evidence(execution_id=executions[1].id, span_id="step", quote="answered", role="counterexample"),
|
||||
),
|
||||
)
|
||||
completed: Final = Job(
|
||||
id=f"{prefix}-old",
|
||||
status="completed",
|
||||
created_at=now,
|
||||
start=now,
|
||||
end=now,
|
||||
settings=settings,
|
||||
revision=1,
|
||||
sample=Sample(executions=executions[:4], eligible=4),
|
||||
assessments=(
|
||||
RunAssessment(execution_id=executions[0].id),
|
||||
RunAssessment(execution_id=executions[1].id),
|
||||
RunAssessment(execution_id=executions[2].id, cannot_assess=True),
|
||||
),
|
||||
findings=(finding,),
|
||||
)
|
||||
lens: Final = Lens(
|
||||
id=prefix,
|
||||
scope=Scope(all_teams=True),
|
||||
settings=settings,
|
||||
created_at=now,
|
||||
next_run_at=now,
|
||||
budget_month=now.strftime("%Y-%m"),
|
||||
jobs=(completed,),
|
||||
)
|
||||
await repo.create(lens)
|
||||
try:
|
||||
current: Final = completed.model_copy(update={"id": f"{prefix}-current"})
|
||||
unfinished: Final = tuple(
|
||||
completed.model_copy(
|
||||
update={
|
||||
"id": f"{prefix}-{status}",
|
||||
"status": status,
|
||||
"sample": Sample(executions=(executions[index],), eligible=1),
|
||||
"assessments": (RunAssessment(execution_id=executions[index].id),),
|
||||
"findings": (),
|
||||
}
|
||||
)
|
||||
for index, status in enumerate(("running", "failed", "cancelled"), start=4)
|
||||
)
|
||||
await repo.update(prefix, lambda item: item.model_copy(update={"jobs": (current, *unfinished)}))
|
||||
archived: Final = await repo.job(prefix, completed.id)
|
||||
assert archived is not None and archived.status == "completed"
|
||||
expected: Final = tuple(
|
||||
TraceFindingCount(**identity.model_dump(), finding_count=1 if i == 0 else 0 if i == 1 else None)
|
||||
for i, identity in enumerate(identities)
|
||||
)
|
||||
counts: Final = await trace_findings(
|
||||
TraceFindingsRequest(traces=identities), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
)
|
||||
assert sorted(counts, key=lambda item: item.trace_ref) == list(expected)
|
||||
await repo.update(prefix, lambda item: item.model_copy(update={"jobs": unfinished}))
|
||||
archived_counts: Final = await repo.trace_findings(identities)
|
||||
assert sorted(archived_counts, key=lambda item: item.trace_ref) == list(expected)
|
||||
assert await repo.trace_findings((TraceIdentity(trace_id=prefix),)) == (
|
||||
TraceFindingCount(trace_id=prefix, finding_count=None),
|
||||
)
|
||||
finally:
|
||||
await lens_db.execute_raw('DELETE FROM "LiteLLM_LensRun" WHERE lens_id=$1', prefix)
|
||||
await lens_db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=$1', prefix)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("populated", (False, True))
|
||||
@pytest.mark.parametrize("preceding_schema", (False, True))
|
||||
def test_lens_rename_preserves_saved_data_and_worker_credentials(populated: bool, preceding_schema: bool) -> None:
|
||||
|
|
|
|||
|
|
@ -2568,6 +2568,29 @@ def test_proxy_admin_viewer_post_blocked_outside_allowlists(route):
|
|||
assert exc_info.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route,allowed", (("/lens/traces/findings", True), ("/lens/example/run", False)))
|
||||
def test_admin_viewer_can_read_trace_findings_but_cannot_start_investigations(route: str, allowed: bool) -> None:
|
||||
request: Final = Request({"type": "http", "method": "POST", "path": route, "query_string": b""})
|
||||
auth: Final = UserAPIKeyAuth(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
|
||||
def check_access() -> None:
|
||||
RouteChecks.non_proxy_admin_allowed_routes_check(
|
||||
user_obj=LiteLLM_UserTable(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY),
|
||||
_user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
|
||||
route=route,
|
||||
request=request,
|
||||
valid_token=auth,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
if allowed:
|
||||
assert check_access() is None
|
||||
else:
|
||||
with pytest.raises(HTTPException) as error:
|
||||
check_access()
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
# ── Admin Viewer: management_routes write endpoints stay blocked ─────────────
|
||||
#
|
||||
# `management_routes` is a mix of reads (info/list, handled via the safe-method
|
||||
|
|
|
|||
|
|
@ -15,13 +15,24 @@ from litellm.proxy.lens.endpoints import (
|
|||
result,
|
||||
run_settings,
|
||||
run_window,
|
||||
trace_findings,
|
||||
user_scope,
|
||||
validate_model,
|
||||
watchable,
|
||||
watching,
|
||||
worker_supports_model,
|
||||
)
|
||||
from litellm.proxy.lens.models import ActivitySelection, Coverage, Lens, LensSettings, Result, RunRequest, Scope
|
||||
from litellm.proxy.lens.models import (
|
||||
ActivitySelection,
|
||||
Coverage,
|
||||
Lens,
|
||||
LensSettings,
|
||||
Result,
|
||||
RunRequest,
|
||||
Scope,
|
||||
TraceFindingsRequest,
|
||||
TraceIdentity,
|
||||
)
|
||||
from litellm.proxy.lens.repository import Row
|
||||
from litellm.proxy.lens.state import claim_job, queue_job, replace_job
|
||||
from tests.unit.proxy.lens.test_state import NOW, lens, worker
|
||||
|
|
@ -194,6 +205,15 @@ async def test_agent_discovery_without_trace_storage_still_requires_admin_access
|
|||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trace_finding_counts_require_investigation_read_access() -> None:
|
||||
auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)
|
||||
request: Final = TraceFindingsRequest(traces=(TraceIdentity(trace_id="trace"),))
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await trace_findings(request, auth)
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
(LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.TEAM),
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ function serve({ enabled = false, traces = false, requests = false, connected =
|
|||
? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] })
|
||||
: Response.json({ detail: "Tracing is not enabled" }, { status: 501 });
|
||||
if (path === "/lens/activity/available") return Response.json({ traces, requests });
|
||||
if (path === "/lens/traces/findings") return Response.json([]);
|
||||
if (path === "/lens" && method === "POST") {
|
||||
const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...(body as object) } };
|
||||
list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true });
|
||||
|
|
|
|||
|
|
@ -177,6 +177,7 @@ describe("Lens interactive demo", () => {
|
|||
if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed });
|
||||
if (path.endsWith("/runs")) return Response.json(saved.jobs);
|
||||
if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) });
|
||||
if (path === "/lens/traces/findings") return Response.json([]);
|
||||
return Response.json({ data: [], traces: true, requests: false });
|
||||
});
|
||||
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
|
||||
|
|
@ -355,6 +356,7 @@ describe("Lens interactive demo", () => {
|
|||
if (path === "/lens/agents") return Response.json([]);
|
||||
if (path.startsWith("/lens/preview")) return Response.json({ eligible: 0, selected: 0, executions: [] });
|
||||
if (path === "/v1/traces") return Response.json({ data: [createLensDemoData().runs[0].trace.summary] });
|
||||
if (path === "/lens/traces/findings") return Response.json([]);
|
||||
return Response.json({ data: [], traces: true, requests: false });
|
||||
});
|
||||
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
|
||||
|
|
|
|||
|
|
@ -194,6 +194,7 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
|
|||
isActive={activeTab === "traces"}
|
||||
readOnly={readOnly}
|
||||
canMintTracingKey={isAdmin}
|
||||
canViewFindings={canViewInvestigations}
|
||||
/>
|
||||
</TabsContent>
|
||||
<TabsContent value="findings" className={PANEL}>
|
||||
|
|
|
|||
|
|
@ -60,6 +60,34 @@ function demoTracesApi(data: LensDemoData): TracesApi {
|
|||
.filter((trace) => Date.parse(trace.start_time) >= startMs && Date.parse(trace.start_time) <= endMs),
|
||||
next_cursor: null,
|
||||
}),
|
||||
findings: async (traces) =>
|
||||
traces.map((trace) => {
|
||||
const jobs = data.lenses.flatMap((lens) => lens.jobs).filter((job) => job.status === "completed");
|
||||
const assessed = jobs.flatMap((job) =>
|
||||
(job.sample?.executions ?? [])
|
||||
.filter((execution) => {
|
||||
const matches =
|
||||
execution.source === "traces" &&
|
||||
execution.trace_id === trace.trace_id &&
|
||||
(execution.trace_ref ?? "") === (trace.trace_ref ?? "");
|
||||
return (
|
||||
matches &&
|
||||
job.assessments.some(
|
||||
(assessment) => assessment.execution_id === execution.id && !assessment.cannot_assess,
|
||||
)
|
||||
);
|
||||
})
|
||||
.map((execution) => ({ job, execution })),
|
||||
);
|
||||
const findings = new Set(
|
||||
assessed.flatMap(({ job, execution }) =>
|
||||
(job.findings ?? [])
|
||||
.filter((finding) => finding.occurrences.includes(execution.id))
|
||||
.map((finding) => finding.id),
|
||||
),
|
||||
);
|
||||
return { ...trace, finding_count: assessed.length ? findings.size : null };
|
||||
}),
|
||||
anyRecorded: async () => data.runs.length > 0,
|
||||
trace: (traceId) => found(run(traceId)?.trace),
|
||||
span: (traceId, spanId) => found(run(traceId)?.details.find((span) => span.span_id === spanId)),
|
||||
|
|
|
|||
|
|
@ -9,7 +9,15 @@ import {
|
|||
apiClient,
|
||||
getProxyBaseUrl,
|
||||
} from "../../networking";
|
||||
import type { SpanDetail, SpanErrorPage, Trace, TraceListQuery, TracePage } from "./types";
|
||||
import type {
|
||||
SpanDetail,
|
||||
SpanErrorPage,
|
||||
Trace,
|
||||
TraceListQuery,
|
||||
TracePage,
|
||||
TraceFindingCount,
|
||||
TraceFindingsRequest,
|
||||
} from "./types";
|
||||
|
||||
export interface TraceWindow {
|
||||
readonly startMs: number;
|
||||
|
|
@ -27,6 +35,7 @@ export interface TracesApi {
|
|||
readonly live: boolean;
|
||||
handoff(traceId: string, spanId?: string | null, traceRef?: string): TraceHandoff;
|
||||
list(window: TraceWindow): Promise<TracePage>;
|
||||
findings(traces: TraceFindingsRequest["traces"]): Promise<TraceFindingCount[]>;
|
||||
anyRecorded(): Promise<boolean>;
|
||||
trace(traceId: string, traceRef?: string, cursor?: string | null): Promise<Trace>;
|
||||
span(traceId: string, spanId: string, traceRef?: string): Promise<SpanDetail>;
|
||||
|
|
@ -41,7 +50,7 @@ export interface TracesApi {
|
|||
export const agentHandoffText = (traceId: string, spanId?: string | null, traceRef?: string): string => {
|
||||
const url = `${getProxyBaseUrl().replace(/\/$/, "")}/v1/traces/${traceId}?format=md${spanId ? `&span_id=${spanId}` : ""}${traceRef ? `&trace_ref=${traceRef}` : ""}`;
|
||||
const what = spanId ? "this step of a LiteLLM agent trace" : "this LiteLLM agent trace";
|
||||
return `Read ${what} and explain what happened and why it failed:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`;
|
||||
return `Read ${what}, explain what happened, and investigate any issues:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`;
|
||||
};
|
||||
|
||||
export function liveTracesApi(accessToken: string): TracesApi {
|
||||
|
|
@ -52,6 +61,11 @@ export function liveTracesApi(accessToken: string): TracesApi {
|
|||
copied: "Command copied",
|
||||
}),
|
||||
list: (window) => agentTraceListCall({ accessToken, ...window }),
|
||||
findings: (traces) =>
|
||||
apiClient.post<TraceFindingCount[]>("/lens/traces/findings", {
|
||||
accessToken,
|
||||
body: { traces } satisfies TraceFindingsRequest,
|
||||
}),
|
||||
anyRecorded: async () => {
|
||||
const page = await apiClient.get<TracePage>("/v1/traces", {
|
||||
accessToken,
|
||||
|
|
|
|||
|
|
@ -14,11 +14,13 @@ export default function AgentTracesPage({
|
|||
isActive = true,
|
||||
readOnly = false,
|
||||
canMintTracingKey = false,
|
||||
canViewFindings = true,
|
||||
}: {
|
||||
accessToken: string;
|
||||
isActive?: boolean;
|
||||
readOnly?: boolean;
|
||||
canMintTracingKey?: boolean;
|
||||
canViewFindings?: boolean;
|
||||
}) {
|
||||
const sourceLive = useTracesLive();
|
||||
const [rangeHours, setRangeHours] = useRangeHoursRouting();
|
||||
|
|
@ -55,6 +57,7 @@ export default function AgentTracesPage({
|
|||
isLiveTail={isLiveTail}
|
||||
readOnly={readOnly}
|
||||
canMintTracingKey={canMintTracingKey}
|
||||
canViewFindings={canViewFindings}
|
||||
timeControls={{ rangeHours, onRangeHoursChange: changeRange, onLiveChange: changeLive }}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import traceList from "../__fixtures__/trace_list.json";
|
|||
import AgentTracesPage from "./AgentTracesPage";
|
||||
import { filterRuns } from "./runSearch/runQuery";
|
||||
import { AgentTracesSection, type TimeControls } from "./AgentTracesSection";
|
||||
import type { TracePage, TraceSummary } from "../types";
|
||||
import type { TraceFindingCount, TracePage, TraceSummary } from "../types";
|
||||
|
||||
vi.mock("../../../networking", () => ({
|
||||
apiClient: { get: vi.fn(), post: vi.fn() },
|
||||
|
|
@ -37,7 +37,7 @@ const runs = (traceList as TracePage).data as TraceSummary[];
|
|||
const lastUrl = (onUrlUpdate: ReturnType<typeof vi.fn>) =>
|
||||
new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? ""));
|
||||
|
||||
const renderSection = () =>
|
||||
const renderSection = (canViewFindings = true) =>
|
||||
renderWithProviders(
|
||||
<AgentTracesSection
|
||||
accessToken="sk-test"
|
||||
|
|
@ -46,6 +46,7 @@ const renderSection = () =>
|
|||
endTime="2026-09-30T00:00"
|
||||
isCustomDate={false}
|
||||
isLiveTail={false}
|
||||
canViewFindings={canViewFindings}
|
||||
/>,
|
||||
);
|
||||
|
||||
|
|
@ -88,6 +89,10 @@ describe("AgentTracesSection", () => {
|
|||
testQueryClient.clear();
|
||||
vi.mocked(agentTraceListCall).mockReset();
|
||||
vi.mocked(apiClient.get).mockResolvedValue({ data: [] });
|
||||
vi.mocked(apiClient.post).mockImplementation(async (_path, options) => {
|
||||
const body = options?.body as { traces: { trace_id: string; trace_ref?: string }[] };
|
||||
return body.traces.map((trace) => ({ ...trace, finding_count: null }));
|
||||
});
|
||||
});
|
||||
|
||||
it("loads the next page only once the list scrolls near its end, then stops at the last page", async () => {
|
||||
|
|
@ -296,7 +301,7 @@ describe("AgentTracesSection", () => {
|
|||
expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL");
|
||||
});
|
||||
|
||||
it("lists every run with its input, counts and failed column", async () => {
|
||||
it("lists uninvestigated runs without presenting tool errors as failures", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
|
||||
renderSection();
|
||||
|
||||
|
|
@ -305,11 +310,73 @@ describe("AgentTracesSection", () => {
|
|||
const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans"));
|
||||
expect(lead).toBeDefined();
|
||||
const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement;
|
||||
expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument();
|
||||
expect(within(failed).queryByLabelText("2 errors")).not.toBeInTheDocument();
|
||||
expect(await within(failed).findByTitle("No conclusive investigation for this trace")).toHaveTextContent("-");
|
||||
expect(screen.getByRole("columnheader", { name: "Findings" })).toBeInTheDocument();
|
||||
expect(screen.queryByRole("columnheader", { name: "Failed" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument();
|
||||
expect(within(failed).getByText("—")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("distinguishes uninvestigated traces, completed clean investigations, and findings", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null });
|
||||
vi.mocked(apiClient.post).mockResolvedValue(
|
||||
runs.map((run, index) => ({
|
||||
trace_id: run.trace_id,
|
||||
trace_ref: run.trace_ref ?? "",
|
||||
finding_count: [null, 0, 3][index],
|
||||
})),
|
||||
);
|
||||
renderSection();
|
||||
expect(await screen.findByTitle("No conclusive investigation for this trace")).toHaveTextContent("-");
|
||||
expect(await screen.findByTitle("0 findings from completed investigations")).toHaveTextContent("0");
|
||||
expect(await screen.findByTitle("3 findings from completed investigations")).toHaveTextContent("3");
|
||||
});
|
||||
|
||||
it("keeps failure labels out of the run totals above the findings table", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({
|
||||
data: [{ ...runs[0], status: "error", error_count: 8 }],
|
||||
next_cursor: null,
|
||||
});
|
||||
renderSection();
|
||||
expect(await screen.findByTestId("agent-trace-row")).toBeVisible();
|
||||
expect(screen.getByText(/1 run from/)).toBeVisible();
|
||||
expect(screen.queryByText(/failed runs|with errors/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("does not present a failed findings lookup as an uninvestigated or clean trace", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null });
|
||||
vi.mocked(apiClient.post).mockRejectedValue(new ApiError("Unavailable", 503, {}));
|
||||
renderSection();
|
||||
expect(await screen.findByTitle("Could not load findings")).toHaveTextContent("Unavailable");
|
||||
expect(screen.queryByTitle("No conclusive investigation for this trace")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the column picker open when findings finish loading", async () => {
|
||||
const user = userEvent.setup();
|
||||
const pending = Promise.withResolvers<TraceFindingCount[]>();
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null });
|
||||
vi.mocked(apiClient.post).mockReturnValue(pending.promise);
|
||||
renderSection();
|
||||
expect(await screen.findByTestId("agent-trace-row")).toBeVisible();
|
||||
await user.click(screen.getByRole("button", { name: "Columns" }));
|
||||
expect(await screen.findByTestId("view-option-cost")).toBeVisible();
|
||||
await act(async () => {
|
||||
pending.resolve([{ trace_id: runs[0].trace_id, trace_ref: runs[0].trace_ref ?? "", finding_count: 1 }]);
|
||||
});
|
||||
expect(await screen.findByTitle("1 findings from completed investigations")).toBeVisible();
|
||||
expect(screen.getByTestId("view-option-cost")).toBeVisible();
|
||||
});
|
||||
|
||||
it("keeps traces available without requesting investigation data for users without access", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null });
|
||||
vi.mocked(apiClient.post).mockClear();
|
||||
renderSection(false);
|
||||
expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length);
|
||||
expect(screen.queryByRole("columnheader", { name: "Findings" })).not.toBeInTheDocument();
|
||||
expect(apiClient.post).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows the spend returned for a run", async () => {
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({
|
||||
...(traceList as TracePage),
|
||||
|
|
@ -517,7 +584,9 @@ describe("AgentTracesSection", () => {
|
|||
const user = userEvent.setup();
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
|
||||
const onUrlUpdate = vi.fn();
|
||||
const failed = filterRuns(runs, "status:error");
|
||||
const mixed = runs.map((run, index) => (index === 1 ? { ...run, status: "error" as const } : run));
|
||||
vi.mocked(agentTraceListCall).mockResolvedValue({ data: mixed, next_cursor: null });
|
||||
const failed = filterRuns(mixed, "status:error");
|
||||
renderWithProviders(
|
||||
<AgentTracesSection
|
||||
accessToken="sk-test"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { Inspector } from "@/components/shared/Inspector";
|
|||
import { Button } from "@/components/ui/button";
|
||||
|
||||
import { AgentTracesTable } from "./AgentTracesTable";
|
||||
import { useTraceFindings } from "./useTraceFindings";
|
||||
import {
|
||||
type TraceRef,
|
||||
traceKey,
|
||||
|
|
@ -53,6 +54,7 @@ interface AgentTracesSectionProps {
|
|||
timeControls?: TimeControls;
|
||||
readOnly?: boolean;
|
||||
canMintTracingKey?: boolean;
|
||||
canViewFindings?: boolean;
|
||||
}
|
||||
|
||||
function useTracingSetup(traces: AgentTracesResult, isActive: boolean, rangeChanged: boolean) {
|
||||
|
|
@ -93,6 +95,7 @@ export function AgentTracesSection({
|
|||
timeControls,
|
||||
readOnly = false,
|
||||
canMintTracingKey = false,
|
||||
canViewFindings,
|
||||
}: AgentTracesSectionProps) {
|
||||
const live = useTracesLive();
|
||||
const { trace: openTrace, openTrace: openRun, selection, fullScreen, setFullScreen } = useOpenTraceRouting();
|
||||
|
|
@ -123,6 +126,7 @@ export function AgentTracesSection({
|
|||
);
|
||||
const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
|
||||
const runRefs = useMemo(() => runs.map(traceRefOf), [runs]);
|
||||
const findings = useTraceFindings(accessToken, runs, isActive, canViewFindings);
|
||||
|
||||
const changeRange = (hours: number, apply: (hours: number) => void) => {
|
||||
setZoom(null);
|
||||
|
|
@ -219,6 +223,8 @@ export function AgentTracesSection({
|
|||
<TracesTimeline runs={filtered} range={range} selection={zoom} onSelect={setZoom} />
|
||||
<AgentTracesTable
|
||||
traces={runs}
|
||||
findings={findings}
|
||||
canViewFindings={canViewFindings}
|
||||
isLoading={traces.isLoading || (checkHistory && history.isLoading)}
|
||||
error={traces.error}
|
||||
hasMore={traces.hasMore}
|
||||
|
|
@ -244,13 +250,11 @@ function TracesReceived({ received }: { received: boolean }) {
|
|||
}
|
||||
|
||||
function TraceCounts({ runs }: { runs: readonly TraceSummary[] }) {
|
||||
const failed = runs.filter((run) => run.error_count > 0).length;
|
||||
return (
|
||||
<div className="flex shrink-0 flex-wrap items-center gap-x-3 gap-y-1 px-3 pt-2 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{runs.length} {runs.length === 1 ? "run" : "runs"} from {new Set(runs.flatMap(traceAgentNames)).size} agents
|
||||
</span>
|
||||
{failed > 0 && <span className="text-destructive">{failed} with errors</span>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ const renderEmpty = (rangeEmpty: boolean) => {
|
|||
inList(
|
||||
<AgentTracesTable
|
||||
traces={[]}
|
||||
findings={new Map()}
|
||||
isLoading={false}
|
||||
error={null}
|
||||
hasMore={false}
|
||||
|
|
@ -58,6 +59,7 @@ describe("AgentTracesTable loading state", () => {
|
|||
inList(
|
||||
<AgentTracesTable
|
||||
traces={[]}
|
||||
findings={new Map()}
|
||||
isLoading
|
||||
error={null}
|
||||
hasMore={false}
|
||||
|
|
@ -89,6 +91,7 @@ describe("AgentTracesTable virtualization", () => {
|
|||
inList(
|
||||
<AgentTracesTable
|
||||
traces={manyRuns}
|
||||
findings={new Map()}
|
||||
isLoading={false}
|
||||
error={null}
|
||||
hasMore={false}
|
||||
|
|
@ -117,6 +120,7 @@ describe("AgentTracesTable column picker", () => {
|
|||
inList(
|
||||
<AgentTracesTable
|
||||
traces={runs}
|
||||
findings={new Map()}
|
||||
isLoading={false}
|
||||
error={null}
|
||||
hasMore={false}
|
||||
|
|
@ -163,6 +167,7 @@ describe("AgentTracesTable column picker", () => {
|
|||
inList(
|
||||
<AgentTracesTable
|
||||
traces={[]}
|
||||
findings={new Map()}
|
||||
isLoading
|
||||
error={null}
|
||||
hasMore={false}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import { getCoreRowModel, useReactTable, type ColumnDef, type TableOptions } from "@tanstack/react-table";
|
||||
import { ArrowDown, ChevronRight } from "lucide-react";
|
||||
import { useEffect } from "react";
|
||||
import { createContext, useContext, useEffect } from "react";
|
||||
import { useInView } from "react-intersection-observer";
|
||||
|
||||
import { DataTableViewOptions } from "@/components/shared/DataTable/DataTableViewOptions";
|
||||
|
|
@ -13,7 +13,7 @@ import { Skeleton } from "@/components/ui/skeleton";
|
|||
import { formatActivityTimestamp, formatRunTimestamp, localTimeZoneAbbreviation } from "@/utils/activityTimestamp";
|
||||
|
||||
import { SpanIcon } from "../ui/SpanIcon";
|
||||
import { StatusMark } from "../ui/StatusMark";
|
||||
import type { TraceFindingState } from "./useTraceFindings";
|
||||
import { FrameworkLogo, traceFramework } from "../ui/TraceFramework";
|
||||
import type { TraceSummary } from "../types";
|
||||
import { traceRefOf } from "../routing";
|
||||
|
|
@ -21,6 +21,8 @@ import { fmtMs, previewText, traceDisplayName, traceAgentNames } from "../utils"
|
|||
|
||||
interface AgentTracesTableProps {
|
||||
traces: TraceSummary[];
|
||||
findings: ReadonlyMap<string, TraceFindingState>;
|
||||
canViewFindings?: boolean;
|
||||
isLoading: boolean;
|
||||
error: Error | null;
|
||||
hasMore: boolean;
|
||||
|
|
@ -46,6 +48,7 @@ const SKELETON_ROWS = Array.from({ length: 12 }, (_, i) => i);
|
|||
const ROW_HEIGHT = 36;
|
||||
const MUTED_NUM = "font-mono text-muted-foreground";
|
||||
const NUM = "font-mono text-foreground";
|
||||
const FindingsContext = createContext<ReadonlyMap<string, TraceFindingState>>(new Map());
|
||||
|
||||
function AgentCell({ run }: { run: TraceSummary }) {
|
||||
const framework = traceFramework(run);
|
||||
|
|
@ -62,7 +65,6 @@ function AgentCell({ run }: { run: TraceSummary }) {
|
|||
function InputCell({ run }: { run: TraceSummary }) {
|
||||
return (
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<StatusMark status={run.error_count > 0 ? "error" : "ok"} subtle />
|
||||
<span className="truncate text-foreground">
|
||||
{firstLine(previewText(run.input_preview)) || traceDisplayName(run)}
|
||||
</span>
|
||||
|
|
@ -79,6 +81,15 @@ function InputCell({ run }: { run: TraceSummary }) {
|
|||
);
|
||||
}
|
||||
|
||||
function FindingCount({ run }: { run: TraceSummary }) {
|
||||
const state = useContext(FindingsContext).get(runKey(run));
|
||||
if (!state || state.status === "pending")
|
||||
return <Skeleton aria-label="Loading findings" className="ml-auto h-3 w-5" />;
|
||||
if (state.status === "error") return <span title="Could not load findings">Unavailable</span>;
|
||||
if (state.count === null) return <span title="No conclusive investigation for this trace">-</span>;
|
||||
return <span title={`${state.count} findings from completed investigations`}>{state.count.toLocaleString()}</span>;
|
||||
}
|
||||
|
||||
const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
|
||||
{
|
||||
id: "time",
|
||||
|
|
@ -148,16 +159,11 @@ const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
|
|||
meta: { numeric: true, className: NUM },
|
||||
},
|
||||
{
|
||||
id: "failed",
|
||||
size: 72,
|
||||
header: "Failed",
|
||||
cell: ({ row }) =>
|
||||
row.original.error_count > 0 ? (
|
||||
<StatusMark status="error" count={row.original.error_count} />
|
||||
) : (
|
||||
<span className="font-mono text-muted-foreground/60">0</span>
|
||||
),
|
||||
meta: { numeric: true },
|
||||
id: "findings",
|
||||
size: 96,
|
||||
header: "Findings",
|
||||
cell: ({ row }) => <FindingCount run={row.original} />,
|
||||
meta: { numeric: true, className: NUM },
|
||||
},
|
||||
{
|
||||
id: "open",
|
||||
|
|
@ -208,6 +214,8 @@ function EmptyRuns({ rangeEmpty, onSetUpTracing }: { rangeEmpty: boolean; onSetU
|
|||
/** Devtool-dense runs list: one row per agent run, newest first. */
|
||||
export function AgentTracesTable({
|
||||
traces,
|
||||
findings,
|
||||
canViewFindings = true,
|
||||
isLoading,
|
||||
error,
|
||||
hasMore,
|
||||
|
|
@ -224,7 +232,7 @@ export function AgentTracesTable({
|
|||
const { columnVisibility, onColumnVisibilityChange } = usePersistedColumnVisibility("lens-traces");
|
||||
const tableOptions: TableOptions<TraceSummary> = {
|
||||
data: traces,
|
||||
columns: RUN_COLUMNS,
|
||||
columns: RUN_COLUMNS.filter((column) => canViewFindings || column.id !== "findings"),
|
||||
defaultColumn: { size: undefined },
|
||||
getRowId: runKey,
|
||||
autoResetAll: false,
|
||||
|
|
@ -234,54 +242,56 @@ export function AgentTracesTable({
|
|||
};
|
||||
const table = useReactTable(tableOptions);
|
||||
return (
|
||||
<InspectorTable.Root table={table} data-testid="runs-table">
|
||||
<InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
|
||||
<InspectorTable.Header />
|
||||
<InspectorTable.Body<TraceSummary>
|
||||
rowHeight={() => ROW_HEIGHT}
|
||||
after={
|
||||
<>
|
||||
{isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
|
||||
{autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
|
||||
</>
|
||||
}
|
||||
>
|
||||
{(row) => (
|
||||
<InspectorTable.Row
|
||||
row={row}
|
||||
item={traceRefOf(row.original)}
|
||||
data-testid="agent-trace-row"
|
||||
className="h-9"
|
||||
/>
|
||||
)}
|
||||
</InspectorTable.Body>
|
||||
</InspectorTable.Grid>
|
||||
{isLoading && (
|
||||
<p role="status" className="sr-only">
|
||||
Loading runs…
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
|
||||
</span>
|
||||
{onRetry && (
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
|
||||
Retry
|
||||
<FindingsContext.Provider value={findings}>
|
||||
<InspectorTable.Root table={table} data-testid="runs-table">
|
||||
<InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
|
||||
<InspectorTable.Header />
|
||||
<InspectorTable.Body<TraceSummary>
|
||||
rowHeight={() => ROW_HEIGHT}
|
||||
after={
|
||||
<>
|
||||
{isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
|
||||
{autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
|
||||
</>
|
||||
}
|
||||
>
|
||||
{(row) => (
|
||||
<InspectorTable.Row
|
||||
row={row}
|
||||
item={traceRefOf(row.original)}
|
||||
data-testid="agent-trace-row"
|
||||
className="h-9"
|
||||
/>
|
||||
)}
|
||||
</InspectorTable.Body>
|
||||
</InspectorTable.Grid>
|
||||
{isLoading && (
|
||||
<p role="status" className="sr-only">
|
||||
Loading runs…
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
|
||||
</span>
|
||||
{onRetry && (
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
|
||||
Retry
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{canContinue && traces.length === 0 && (
|
||||
<div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
|
||||
<span>No loaded runs match these filters.</span>
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
|
||||
Load older runs
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{canContinue && traces.length === 0 && (
|
||||
<div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
|
||||
<span>No loaded runs match these filters.</span>
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
|
||||
Load older runs
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
|
||||
</InspectorTable.Root>
|
||||
</div>
|
||||
)}
|
||||
{isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
|
||||
</InspectorTable.Root>
|
||||
</FindingsContext.Provider>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,11 +7,12 @@ const HOUR = 3600 * 1000;
|
|||
const START = Date.UTC(2026, 8, 30, 0, 0, 0);
|
||||
const range = { startMs: START, endMs: START + 10 * HOUR };
|
||||
|
||||
const run = (offsetMs: number, errorCount = 0): TraceSummary =>
|
||||
const run = (offsetMs: number, errorCount = 0, status: TraceSummary["status"] = "ok"): TraceSummary =>
|
||||
({
|
||||
trace_id: `t${offsetMs}`,
|
||||
start_time: new Date(START + offsetMs).toISOString(),
|
||||
error_count: errorCount,
|
||||
status,
|
||||
}) as TraceSummary;
|
||||
|
||||
describe("bucketRuns", () => {
|
||||
|
|
@ -35,9 +36,9 @@ describe("bucketRuns", () => {
|
|||
expect(buckets[1].total).toBe(1);
|
||||
});
|
||||
|
||||
it("counts runs with any errors as failed", () => {
|
||||
const buckets = bucketRuns([run(HOUR, 3), run(HOUR + 1), run(HOUR + 2, 1), run(5 * HOUR)], range, 10);
|
||||
expect(buckets[1]).toMatchObject({ total: 3, failed: 2 });
|
||||
it("counts failed runs without treating recovered tool errors as run failures", () => {
|
||||
const buckets = bucketRuns([run(HOUR, 8), run(HOUR + 1), run(HOUR + 2, 1, "error"), run(5 * HOUR)], range, 10);
|
||||
expect(buckets[1]).toMatchObject({ total: 3, failed: 1 });
|
||||
expect(buckets[5]).toMatchObject({ total: 1, failed: 0 });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ export function bucketRuns(runs: readonly TraceSummary[], range: TimeWindow, buc
|
|||
const width = (range.endMs - range.startMs) / buckets;
|
||||
const placed = runs.map((run) => ({
|
||||
index: Math.floor((moment(run.start_time).valueOf() - range.startMs) / width),
|
||||
failed: run.error_count > 0,
|
||||
failed: run.status === "error",
|
||||
agent: traceAgentNames(run)[0] ?? "",
|
||||
}));
|
||||
return Array.from({ length: buckets }, (_, i) => {
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ const researchOverrides = {
|
|||
agent_names: ["researcher"],
|
||||
models: ["claude-opus"],
|
||||
error_count: 2,
|
||||
status: "error" as const,
|
||||
};
|
||||
const research = run(researchOverrides);
|
||||
const plain = run({ trace_id: "ccc333", name: "health", service: "cron" });
|
||||
|
|
|
|||
|
|
@ -15,12 +15,22 @@ describe("filterRuns", () => {
|
|||
expect(ids("gpt-5")).toEqual([]);
|
||||
});
|
||||
|
||||
it("splits runs by status, judged by recorded errors", () => {
|
||||
it("splits runs by their overall status", () => {
|
||||
expect(ids("status:error")).toEqual(["bbb222"]);
|
||||
expect(ids("-status:error")).toEqual(["aaa111", "ccc333"]);
|
||||
expect(ids("status:OK")).toEqual(["aaa111", "ccc333"]);
|
||||
});
|
||||
|
||||
it.each(["ok", "unset"] as const)("keeps %s runs with recovered tool errors in non-failed filters", (status) => {
|
||||
const recovered = run({ status, error_count: 8 });
|
||||
expect(filterRuns([recovered], "status:error")).toEqual([]);
|
||||
expect(filterRuns([recovered], "", { agent: "", status: "error" })).toEqual([]);
|
||||
expect(filterRuns([recovered], "status:ok")).toEqual([recovered]);
|
||||
expect(filterRuns([recovered], "", { agent: "", status: "ok" })).toEqual([recovered]);
|
||||
expect(filterRuns([recovered], "-status:error")).toEqual([recovered]);
|
||||
expect(fieldValues(RUN_INDEX, [recovered], "status")).toEqual(["ok"]);
|
||||
});
|
||||
|
||||
it("reads agents from the trace, falling back to the service, and models from the run", () => {
|
||||
expect(ids("agent:triage")).toEqual(["aaa111"]);
|
||||
expect(ids("agent:cron")).toEqual(["ccc333"]);
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@ export type RunField = keyof typeof RUN_FIELDS;
|
|||
|
||||
export const RUN_QUERY: QueryLanguage<RunField> = { fields: RUN_FIELDS, ops: ALL_OPERATORS };
|
||||
|
||||
const runStatus = (run: TraceSummary): "ok" | "error" => (run.status === "error" ? "error" : "ok");
|
||||
|
||||
/** Reads the run fields off a loaded page; free text searches trace id, input and name. */
|
||||
export const RUN_INDEX: ClientIndex<TraceSummary, RunField> = {
|
||||
read: {
|
||||
name: (run) => [run.name],
|
||||
agent: traceAgentNames,
|
||||
status: (run) => [run.error_count > 0 ? "error" : "ok"],
|
||||
status: (run) => [runStatus(run)],
|
||||
model: (run) => run.models,
|
||||
input: (run) => [previewText(run.input_preview)],
|
||||
trace_id: (run) => [run.trace_id],
|
||||
|
|
@ -38,7 +40,6 @@ export function filterRuns(
|
|||
filters: { agent: string; status: "all" | "ok" | "error" } = { agent: "", status: "all" },
|
||||
): TraceSummary[] {
|
||||
const matchesAgent = (run: TraceSummary) => !filters.agent || traceAgentNames(run).includes(filters.agent);
|
||||
const matchesStatus = (run: TraceSummary) =>
|
||||
filters.status === "all" || (run.error_count > 0 ? "error" : "ok") === filters.status;
|
||||
const matchesStatus = (run: TraceSummary) => filters.status === "all" || runStatus(run) === filters.status;
|
||||
return filterItems(RUN_QUERY, RUN_INDEX, runs, query).filter((run) => matchesAgent(run) && matchesStatus(run));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ describe("runPredicates", () => {
|
|||
it("turns each run field into a predicate over the per-trace rollup columns", () => {
|
||||
expect(predicates("agent:researcher status:error model:gpt-5 name:support trace_id:aaa111")).toEqual([
|
||||
"arrayExists(x -> x ILIKE 'researcher', agents)",
|
||||
"errors > 0",
|
||||
"status = 'error'",
|
||||
"arrayExists(x -> x ILIKE 'gpt-5', models)",
|
||||
"name ILIKE 'support'",
|
||||
"trace_id ILIKE 'aaa111'",
|
||||
|
|
@ -33,7 +33,7 @@ describe("runPredicates", () => {
|
|||
});
|
||||
|
||||
it("resolves a status glob statically, since status has two values", () => {
|
||||
expect(predicates("status:ok")).toEqual(["errors = 0"]);
|
||||
expect(predicates("status:ok")).toEqual(["status = 'ok'"]);
|
||||
expect(predicates("status:*")).toEqual(["true"]);
|
||||
expect(predicates("-status:pending")).toEqual(["NOT (false)"]);
|
||||
});
|
||||
|
|
@ -52,12 +52,13 @@ describe("runQuerySql", () => {
|
|||
expect(sql).toBe(
|
||||
[
|
||||
"SELECT TraceId AS trace_id, any(RootName) AS name, any(RootInput) AS input, sum(ErrorCount) AS errors,",
|
||||
" if(ifNull(any(RootStatus), '') IN ('STATUS_CODE_ERROR', 'error'), 'error', 'ok') AS status,",
|
||||
" groupUniqArrayArray(AgentNames) AS agents, groupUniqArrayArray(Models) AS models",
|
||||
"FROM agent_traces_by_key",
|
||||
"GROUP BY TraceId",
|
||||
`HAVING min(StartTs) >= fromUnixTimestamp64Milli(${RANGE.startMs}) AND min(StartTs) < fromUnixTimestamp64Milli(${RANGE.endMs})`,
|
||||
" AND arrayExists(x -> x ILIKE 'researcher', agents)",
|
||||
" AND errors > 0",
|
||||
" AND status = 'error'",
|
||||
"ORDER BY min(StartTs) DESC",
|
||||
"LIMIT 100",
|
||||
].join("\n"),
|
||||
|
|
@ -87,6 +88,6 @@ describe("runQueryCommand", () => {
|
|||
const command = runQueryCommand(RANGE)(query("agent:researcher status:error"));
|
||||
expect(command).toBe(traceQueryCommand(runQuerySql(query("agent:researcher status:error"), RANGE)));
|
||||
expect(command).toContain("arrayExists(x -> x ILIKE 'researcher', agents)");
|
||||
expect(command).toContain("errors > 0");
|
||||
expect(command).toContain("status = 'error'");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type { SearchFilter, SearchQuery } from "@/components/shared/search/searc
|
|||
import type { RunField } from "./runQuery";
|
||||
|
||||
const RUN_ROWS = `SELECT TraceId AS trace_id, any(RootName) AS name, any(RootInput) AS input, sum(ErrorCount) AS errors,
|
||||
if(ifNull(any(RootStatus), '') IN ('STATUS_CODE_ERROR', 'error'), 'error', 'ok') AS status,
|
||||
groupUniqArrayArray(AgentNames) AS agents, groupUniqArrayArray(Models) AS models
|
||||
FROM agent_traces_by_key
|
||||
GROUP BY TraceId`;
|
||||
|
|
@ -21,7 +22,7 @@ const likePattern = (value: string): string => likeLiteral(value).replaceAll("*"
|
|||
const matches = (column: string, pattern: string): string => `${column} ILIKE ${sqlString(pattern)}`;
|
||||
const anyMatches = (column: string, pattern: string): string => `arrayExists(x -> ${matches("x", pattern)}, ${column})`;
|
||||
|
||||
const STATUS_PREDICATES = { error: "errors > 0", ok: "errors = 0" } as const;
|
||||
const STATUS_PREDICATES = { error: "status = 'error'", ok: "status = 'ok'" } as const;
|
||||
|
||||
/** Status has two values, so a glob over it resolves statically to one, both or neither predicate. */
|
||||
function statusPredicate(value: string): string {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
import { useQueries } from "@tanstack/react-query";
|
||||
import { chunk } from "es-toolkit";
|
||||
|
||||
import { useTracesApi } from "../api";
|
||||
import type { TraceSummary } from "../types";
|
||||
|
||||
export type TraceFindingState = { status: "ready"; count: number | null } | { status: "pending" } | { status: "error" };
|
||||
|
||||
export function useTraceFindings(accessToken: string, runs: TraceSummary[], isActive: boolean, canViewFindings = true) {
|
||||
const api = useTracesApi(accessToken);
|
||||
const enabled = isActive && canViewFindings;
|
||||
const batches = chunk(
|
||||
runs.map(({ trace_id, trace_ref }) => ({ trace_id, trace_ref: trace_ref ?? "" })),
|
||||
500,
|
||||
);
|
||||
const queries = useQueries({
|
||||
queries: batches.map((traces) => ({
|
||||
queryKey: ["traceFindings", accessToken, traces],
|
||||
queryFn: () => api.findings(traces),
|
||||
enabled,
|
||||
staleTime: 15000,
|
||||
refetchInterval: enabled && api.live ? 15000 : false,
|
||||
retry: false,
|
||||
})),
|
||||
});
|
||||
return new Map<string, TraceFindingState>(
|
||||
batches.flatMap((traces, index) => {
|
||||
const query = queries[index];
|
||||
const counts = new Map(query.data?.map((result) => [result.trace_ref || result.trace_id, result.finding_count]));
|
||||
return traces.map((trace): [string, TraceFindingState] => {
|
||||
const key = trace.trace_ref || trace.trace_id;
|
||||
if (query.isError) return [key, { status: "error" }];
|
||||
if (query.isPending) return [key, { status: "pending" }];
|
||||
if (!counts.has(key)) return [key, { status: "error" }];
|
||||
return [key, { status: "ready", count: counts.get(key) ?? null }];
|
||||
});
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
@ -11,6 +11,8 @@ export type SpanErrorQuery = NonNullable<
|
|||
paths["/v1/traces/{trace_id}/spans/{span_id}/error"]["get"]["parameters"]["query"]
|
||||
>;
|
||||
export type TraceQueryBody = components["schemas"]["TraceQueryRequest"];
|
||||
export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"];
|
||||
export type TraceFindingCount = components["schemas"]["TraceFindingCount"];
|
||||
type ApiSpanDetail =
|
||||
paths["/v1/traces/{trace_id}/spans/{span_id}"]["get"]["responses"][200]["content"]["application/json"];
|
||||
export type Span = Trace["spans"][number];
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
import { AlertCircle, Check, Circle } from "lucide-react";
|
||||
|
||||
interface StatusMarkProps {
|
||||
status: "ok" | "error" | "unset";
|
||||
count?: number;
|
||||
subtle?: boolean;
|
||||
}
|
||||
|
||||
/** Run / span status glyph: red alert (+ count) for errors, a quiet muted dot or check otherwise. */
|
||||
export function StatusMark({ status, count, subtle = false }: StatusMarkProps) {
|
||||
if (status === "error") {
|
||||
return (
|
||||
<span
|
||||
className="inline-flex items-center gap-1 text-xs text-destructive"
|
||||
aria-label={`${count ?? 1} error${count === 1 ? "" : "s"}`}
|
||||
>
|
||||
<AlertCircle className="size-3" />
|
||||
{count !== undefined && <span className="font-mono tabular-nums">{count}</span>}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
if (subtle)
|
||||
return (
|
||||
<Circle className="size-2 shrink-0 fill-muted-foreground/60 text-muted-foreground/60" aria-label="Success" />
|
||||
);
|
||||
return <Check className="size-3 text-muted-foreground" aria-label="Success" />;
|
||||
}
|
||||
77
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
77
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -9000,6 +9000,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/traces/findings": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Trace Findings */
|
||||
post: operations["trace_findings_lens_traces_findings_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/watch-all": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -46950,6 +46967,33 @@ export interface components {
|
|||
spans: components["schemas"]["Span"][];
|
||||
summary: components["schemas"]["TraceSummary"];
|
||||
};
|
||||
/** TraceFindingCount */
|
||||
TraceFindingCount: {
|
||||
/** Finding Count */
|
||||
finding_count: number | null;
|
||||
/** Trace Id */
|
||||
trace_id: string;
|
||||
/**
|
||||
* Trace Ref
|
||||
* @default
|
||||
*/
|
||||
trace_ref: string;
|
||||
};
|
||||
/** TraceFindingsRequest */
|
||||
TraceFindingsRequest: {
|
||||
/** Traces */
|
||||
traces: components["schemas"]["TraceIdentity"][];
|
||||
};
|
||||
/** TraceIdentity */
|
||||
TraceIdentity: {
|
||||
/** Trace Id */
|
||||
trace_id: string;
|
||||
/**
|
||||
* Trace Ref
|
||||
* @default
|
||||
*/
|
||||
trace_ref: string;
|
||||
};
|
||||
/** TracePage */
|
||||
TracePage: {
|
||||
/** Data */
|
||||
|
|
@ -62253,6 +62297,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
trace_findings_lens_traces_findings_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["TraceFindingsRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["TraceFindingCount"][];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
watch_all_lens_watch_all_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue