diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230000_lens_trace_findings_indexes/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230000_lens_trace_findings_indexes/migration.sql new file mode 100644 index 00000000000..a38477c7a3c --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230000_lens_trace_findings_indexes/migration.sql @@ -0,0 +1,3 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_LensRun_completed_executions_idx" +ON "LiteLLM_LensRun" USING GIN ((data->'sample'->'executions') jsonb_path_ops) +WHERE data->>'status'='completed'; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230100_lens_current_findings_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230100_lens_current_findings_index/migration.sql new file mode 100644 index 00000000000..acfbeaceb4c --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230100_lens_current_findings_index/migration.sql @@ -0,0 +1,2 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_jobs_idx" +ON "LiteLLM_Lens" USING GIN ((data->'jobs') jsonb_path_ops); diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 15b3c925a1d..0028838b673 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1053,6 +1053,7 @@ class LiteLLMRoutes(enum.Enum): # updating this list — the default-allow behavior covers it automatically. admin_viewer_routes = ( [ + "/lens/traces/findings", "/user/list", "/user/available_users", "/user/available_roles", diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 04f178a3107..2df1c9c1e16 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -40,6 +40,8 @@ from litellm.proxy.lens.models import ( RunRequest, Sample, Scope, + TraceFindingCount, + TraceFindingsRequest, WatchAllResult, WatchSkipped, Worker, @@ -240,6 +242,12 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]: return await source_reader(storage).agents(scope) if storage is not None else () +@router.post("/traces/findings", response_model=tuple[TraceFindingCount, ...]) +async def trace_findings(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceFindingCount, ...]: + user_scope(auth) + return await repository().trace_findings(body.traces) + + def watching(lens: Lens) -> Lens: if lens.settings.enabled: return lens diff --git a/litellm/proxy/lens/models.py b/litellm/proxy/lens/models.py index 704f7e74f79..7a0ca8b68a2 100644 --- a/litellm/proxy/lens/models.py +++ b/litellm/proxy/lens/models.py @@ -193,6 +193,19 @@ class RunAssessment(Record): cannot_assess: bool = False +class TraceIdentity(Record): + trace_id: str = Field(min_length=1, max_length=128) + trace_ref: str = Field(default="", max_length=512) + + +class TraceFindingsRequest(Record): + traces: tuple[TraceIdentity, ...] = Field(min_length=1, max_length=500) + + +class TraceFindingCount(TraceIdentity): + finding_count: int | None = Field(ge=0) + + MAX_STEPS = 200 diff --git a/litellm/proxy/lens/repository.py b/litellm/proxy/lens/repository.py index 15d2cfc34e6..ba626ea8dc3 100644 --- a/litellm/proxy/lens/repository.py +++ b/litellm/proxy/lens/repository.py @@ -8,7 +8,7 @@ from typing import Final, Protocol from pydantic import BaseModel, JsonValue, TypeAdapter from litellm.proxy.db.prisma_client import PrismaWrapper -from litellm.proxy.lens.models import Job, Lens, Scope, Worker +from litellm.proxy.lens.models import Job, Lens, Scope, TraceFindingCount, TraceIdentity, Worker class Database(Protocol): @@ -125,6 +125,50 @@ class LensRepository: ) return Job.model_validate(rows[0].data) if rows else None + async def trace_findings(self, traces: tuple[TraceIdentity, ...]) -> tuple[TraceFindingCount, ...]: + rows: Final = _ROWS.validate_python( + await self.db.query_raw( + """WITH targets AS ( + SELECT DISTINCT trace_id, trace_ref, + jsonb_build_array(jsonb_build_object('source', 'traces', 'trace_id', trace_id)) AS executions + FROM jsonb_to_recordset($1::jsonb) AS target(trace_id text, trace_ref text) + ), jobs AS ( + SELECT target.trace_id, target.trace_ref, run.data AS job + FROM targets AS target JOIN "LiteLLM_LensRun" AS run + ON run.data->'sample'->'executions' @> target.executions + WHERE run.data->>'status'='completed' + UNION ALL + SELECT target.trace_id, target.trace_ref, job + FROM targets AS target JOIN "LiteLLM_Lens" AS lens + ON lens.data->'jobs' @> jsonb_build_array(jsonb_build_object( + 'status', 'completed', 'sample', jsonb_build_object('executions', target.executions))) + CROSS JOIN LATERAL jsonb_array_elements(lens.data->'jobs') AS job + WHERE job->>'status'='completed' + ), assessed AS ( + SELECT jobs.trace_id, jobs.trace_ref, execution->>'id' AS execution_id, job + FROM jobs, jsonb_array_elements(job->'sample'->'executions') AS execution + WHERE execution->>'trace_id'=jobs.trace_id + AND COALESCE(execution->>'trace_ref', '')=jobs.trace_ref + AND execution->>'source'='traces' AND EXISTS ( + SELECT 1 FROM jsonb_array_elements(job->'assessments') AS assessment + WHERE assessment->>'execution_id'=execution->>'id' + AND COALESCE((assessment->>'cannot_assess')::boolean, false)=false + ) + ) + SELECT jsonb_build_object( + 'trace_id', target.trace_id, 'trace_ref', target.trace_ref, + 'finding_count', CASE WHEN count(assessed.execution_id)=0 THEN NULL + ELSE count(DISTINCT finding->>'id') END + ) AS data FROM targets AS target + LEFT JOIN assessed USING (trace_id, trace_ref) + LEFT JOIN LATERAL jsonb_array_elements(NULLIF(assessed.job->'findings', 'null'::jsonb)) AS finding + ON finding->'occurrences' ? assessed.execution_id + GROUP BY target.trace_id, target.trace_ref""", + json.dumps(tuple(trace.model_dump() for trace in traces)), + ) + ) + return tuple(TraceFindingCount.model_validate(row.data) for row in rows) + async def workers(self) -> tuple[Worker, ...]: rows: Final = _ROWS.validate_python(await self.db.query_raw('SELECT data FROM "LiteLLM_LensWorker"')) return tuple(Worker.model_validate(row.data) for row in rows) diff --git a/tests/integration/database/test_lens_repository.py b/tests/integration/database/test_lens_repository.py index 29c6ad13825..40cab58e2fd 100644 --- a/tests/integration/database/test_lens_repository.py +++ b/tests/integration/database/test_lens_repository.py @@ -3,6 +3,7 @@ import os from collections.abc import AsyncIterator from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace from typing import Final from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from uuid import uuid4 @@ -13,8 +14,24 @@ import pytest_asyncio from prisma import Prisma from psycopg import sql +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.db.prisma_client import PrismaWrapper -from litellm.proxy.lens.models import Check, Lens, LensSettings, Scope, Worker +from litellm.proxy.lens.models import ( + Check, + Evidence, + Execution, + Finding, + Job, + Lens, + LensSettings, + RunAssessment, + Sample, + Scope, + TraceFindingCount, + TraceFindingsRequest, + TraceIdentity, + Worker, +) from litellm.proxy.lens.repository import LensRepository, WriterDatabase from litellm.proxy.lens.state import claim_job, queue_job @@ -69,6 +86,109 @@ async def test_heartbeat_never_restores_revoked_access(lens_db: Prisma) -> None: await lens_db.execute_raw('DELETE FROM "LiteLLM_LensWorker" WHERE id=$1', worker.id) +@pytest.mark.asyncio +async def test_trace_findings_include_archived_assessments_without_counting_retries_or_counterexamples( + lens_db: Prisma, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.lens.endpoints import trace_findings + + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=lens_db)) + now: Final = datetime.now(timezone.utc) + prefix: Final = uuid4().hex + repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) + settings: Final = LensSettings(name="Finding counts", model="test", context="Answer the question") + identities: Final = tuple(TraceIdentity(trace_id=prefix, trace_ref=f"{prefix}-{i}") for i in range(7)) + executions: Final = tuple( + Execution( + id=f"{prefix}-{i}", + source="traces", + trace_id=identity.trace_id, + trace_ref=identity.trace_ref, + team_id="", + name="Run", + start_time=now.isoformat(), + span_count=1, + ) + for i, identity in enumerate(identities) + ) + finding: Final = Finding( + id=prefix, + title="Repeated lookup", + description="The agent never answered the question", + check_id="expected_behavior", + first_seen=now, + last_seen=now, + revision=1, + occurrences=(executions[0].id,), + evidence=( + Evidence(execution_id=executions[0].id, span_id="step", quote="no answer"), + Evidence(execution_id=executions[1].id, span_id="step", quote="answered", role="counterexample"), + ), + ) + completed: Final = Job( + id=f"{prefix}-old", + status="completed", + created_at=now, + start=now, + end=now, + settings=settings, + revision=1, + sample=Sample(executions=executions[:4], eligible=4), + assessments=( + RunAssessment(execution_id=executions[0].id), + RunAssessment(execution_id=executions[1].id), + RunAssessment(execution_id=executions[2].id, cannot_assess=True), + ), + findings=(finding,), + ) + lens: Final = Lens( + id=prefix, + scope=Scope(all_teams=True), + settings=settings, + created_at=now, + next_run_at=now, + budget_month=now.strftime("%Y-%m"), + jobs=(completed,), + ) + await repo.create(lens) + try: + current: Final = completed.model_copy(update={"id": f"{prefix}-current"}) + unfinished: Final = tuple( + completed.model_copy( + update={ + "id": f"{prefix}-{status}", + "status": status, + "sample": Sample(executions=(executions[index],), eligible=1), + "assessments": (RunAssessment(execution_id=executions[index].id),), + "findings": (), + } + ) + for index, status in enumerate(("running", "failed", "cancelled"), start=4) + ) + await repo.update(prefix, lambda item: item.model_copy(update={"jobs": (current, *unfinished)})) + archived: Final = await repo.job(prefix, completed.id) + assert archived is not None and archived.status == "completed" + expected: Final = tuple( + TraceFindingCount(**identity.model_dump(), finding_count=1 if i == 0 else 0 if i == 1 else None) + for i, identity in enumerate(identities) + ) + counts: Final = await trace_findings( + TraceFindingsRequest(traces=identities), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + ) + assert sorted(counts, key=lambda item: item.trace_ref) == list(expected) + await repo.update(prefix, lambda item: item.model_copy(update={"jobs": unfinished})) + archived_counts: Final = await repo.trace_findings(identities) + assert sorted(archived_counts, key=lambda item: item.trace_ref) == list(expected) + assert await repo.trace_findings((TraceIdentity(trace_id=prefix),)) == ( + TraceFindingCount(trace_id=prefix, finding_count=None), + ) + finally: + await lens_db.execute_raw('DELETE FROM "LiteLLM_LensRun" WHERE lens_id=$1', prefix) + await lens_db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=$1', prefix) + + @pytest.mark.parametrize("populated", (False, True)) @pytest.mark.parametrize("preceding_schema", (False, True)) def test_lens_rename_preserves_saved_data_and_worker_credentials(populated: bool, preceding_schema: bool) -> None: diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index d0d52dd6566..e60025eb0eb 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -2568,6 +2568,29 @@ def test_proxy_admin_viewer_post_blocked_outside_allowlists(route): assert exc_info.value.status_code == 403 +@pytest.mark.parametrize("route,allowed", (("/lens/traces/findings", True), ("/lens/example/run", False))) +def test_admin_viewer_can_read_trace_findings_but_cannot_start_investigations(route: str, allowed: bool) -> None: + request: Final = Request({"type": "http", "method": "POST", "path": route, "query_string": b""}) + auth: Final = UserAPIKeyAuth(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + + def check_access() -> None: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + route=route, + request=request, + valid_token=auth, + request_data={}, + ) + + if allowed: + assert check_access() is None + else: + with pytest.raises(HTTPException) as error: + check_access() + assert error.value.status_code == 403 + + # ── Admin Viewer: management_routes write endpoints stay blocked ───────────── # # `management_routes` is a mix of reads (info/list, handled via the safe-method diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 922bcb08cca..b49daaf12d6 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -15,13 +15,24 @@ from litellm.proxy.lens.endpoints import ( result, run_settings, run_window, + trace_findings, user_scope, validate_model, watchable, watching, worker_supports_model, ) -from litellm.proxy.lens.models import ActivitySelection, Coverage, Lens, LensSettings, Result, RunRequest, Scope +from litellm.proxy.lens.models import ( + ActivitySelection, + Coverage, + Lens, + LensSettings, + Result, + RunRequest, + Scope, + TraceFindingsRequest, + TraceIdentity, +) from litellm.proxy.lens.repository import Row from litellm.proxy.lens.state import claim_job, queue_job, replace_job from tests.unit.proxy.lens.test_state import NOW, lens, worker @@ -194,6 +205,15 @@ async def test_agent_discovery_without_trace_storage_still_requires_admin_access assert error.value.status_code == 403 +@pytest.mark.asyncio +async def test_trace_finding_counts_require_investigation_read_access() -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER) + request: Final = TraceFindingsRequest(traces=(TraceIdentity(trace_id="trace"),)) + with pytest.raises(HTTPException) as error: + await trace_findings(request, auth) + assert error.value.status_code == 403 + + @pytest.mark.parametrize( "role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.TEAM), diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx index 9ffcc585d05..7d41939112b 100644 --- a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -27,6 +27,7 @@ function serve({ enabled = false, traces = false, requests = false, connected = ? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] }) : Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); if (path === "/lens/activity/available") return Response.json({ traces, requests }); + if (path === "/lens/traces/findings") return Response.json([]); if (path === "/lens" && method === "POST") { const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...(body as object) } }; list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true }); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index 5a4966b190d..f42f2bd7e8c 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -177,6 +177,7 @@ describe("Lens interactive demo", () => { if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed }); if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) }); + if (path === "/lens/traces/findings") return Response.json([]); return Response.json({ data: [], traces: true, requests: false }); }); renderWithProviders(, { @@ -355,6 +356,7 @@ describe("Lens interactive demo", () => { if (path === "/lens/agents") return Response.json([]); if (path.startsWith("/lens/preview")) return Response.json({ eligible: 0, selected: 0, executions: [] }); if (path === "/v1/traces") return Response.json({ data: [createLensDemoData().runs[0].trace.summary] }); + if (path === "/lens/traces/findings") return Response.json([]); return Response.json({ data: [], traces: true, requests: false }); }); renderWithProviders(, { diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx index 8562c144056..2c796ebd6fe 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx @@ -194,6 +194,7 @@ function LensContent({ userRole, readOnly }: Omit isActive={activeTab === "traces"} readOnly={readOnly} canMintTracingKey={isAdmin} + canViewFindings={canViewInvestigations} /> diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts index ab856babc2a..36c1d9238b2 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts @@ -60,6 +60,34 @@ function demoTracesApi(data: LensDemoData): TracesApi { .filter((trace) => Date.parse(trace.start_time) >= startMs && Date.parse(trace.start_time) <= endMs), next_cursor: null, }), + findings: async (traces) => + traces.map((trace) => { + const jobs = data.lenses.flatMap((lens) => lens.jobs).filter((job) => job.status === "completed"); + const assessed = jobs.flatMap((job) => + (job.sample?.executions ?? []) + .filter((execution) => { + const matches = + execution.source === "traces" && + execution.trace_id === trace.trace_id && + (execution.trace_ref ?? "") === (trace.trace_ref ?? ""); + return ( + matches && + job.assessments.some( + (assessment) => assessment.execution_id === execution.id && !assessment.cannot_assess, + ) + ); + }) + .map((execution) => ({ job, execution })), + ); + const findings = new Set( + assessed.flatMap(({ job, execution }) => + (job.findings ?? []) + .filter((finding) => finding.occurrences.includes(execution.id)) + .map((finding) => finding.id), + ), + ); + return { ...trace, finding_count: assessed.length ? findings.size : null }; + }), anyRecorded: async () => data.runs.length > 0, trace: (traceId) => found(run(traceId)?.trace), span: (traceId, spanId) => found(run(traceId)?.details.find((span) => span.span_id === spanId)), diff --git a/ui/litellm-dashboard/src/components/lens/traces/api.ts b/ui/litellm-dashboard/src/components/lens/traces/api.ts index 6b9039eae7c..16503773dec 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/api.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/api.ts @@ -9,7 +9,15 @@ import { apiClient, getProxyBaseUrl, } from "../../networking"; -import type { SpanDetail, SpanErrorPage, Trace, TraceListQuery, TracePage } from "./types"; +import type { + SpanDetail, + SpanErrorPage, + Trace, + TraceListQuery, + TracePage, + TraceFindingCount, + TraceFindingsRequest, +} from "./types"; export interface TraceWindow { readonly startMs: number; @@ -27,6 +35,7 @@ export interface TracesApi { readonly live: boolean; handoff(traceId: string, spanId?: string | null, traceRef?: string): TraceHandoff; list(window: TraceWindow): Promise; + findings(traces: TraceFindingsRequest["traces"]): Promise; anyRecorded(): Promise; trace(traceId: string, traceRef?: string, cursor?: string | null): Promise; span(traceId: string, spanId: string, traceRef?: string): Promise; @@ -41,7 +50,7 @@ export interface TracesApi { export const agentHandoffText = (traceId: string, spanId?: string | null, traceRef?: string): string => { const url = `${getProxyBaseUrl().replace(/\/$/, "")}/v1/traces/${traceId}?format=md${spanId ? `&span_id=${spanId}` : ""}${traceRef ? `&trace_ref=${traceRef}` : ""}`; const what = spanId ? "this step of a LiteLLM agent trace" : "this LiteLLM agent trace"; - return `Read ${what} and explain what happened and why it failed:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`; + return `Read ${what}, explain what happened, and investigate any issues:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`; }; export function liveTracesApi(accessToken: string): TracesApi { @@ -52,6 +61,11 @@ export function liveTracesApi(accessToken: string): TracesApi { copied: "Command copied", }), list: (window) => agentTraceListCall({ accessToken, ...window }), + findings: (traces) => + apiClient.post("/lens/traces/findings", { + accessToken, + body: { traces } satisfies TraceFindingsRequest, + }), anyRecorded: async () => { const page = await apiClient.get("/v1/traces", { accessToken, diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesPage.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesPage.tsx index dfc4b770994..900c6574da4 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesPage.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesPage.tsx @@ -14,11 +14,13 @@ export default function AgentTracesPage({ isActive = true, readOnly = false, canMintTracingKey = false, + canViewFindings = true, }: { accessToken: string; isActive?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; + canViewFindings?: boolean; }) { const sourceLive = useTracesLive(); const [rangeHours, setRangeHours] = useRangeHoursRouting(); @@ -55,6 +57,7 @@ export default function AgentTracesPage({ isLiveTail={isLiveTail} readOnly={readOnly} canMintTracingKey={canMintTracingKey} + canViewFindings={canViewFindings} timeControls={{ rangeHours, onRangeHoursChange: changeRange, onLiveChange: changeLive }} /> diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx index 01861454654..d884bba3307 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx @@ -10,7 +10,7 @@ import traceList from "../__fixtures__/trace_list.json"; import AgentTracesPage from "./AgentTracesPage"; import { filterRuns } from "./runSearch/runQuery"; import { AgentTracesSection, type TimeControls } from "./AgentTracesSection"; -import type { TracePage, TraceSummary } from "../types"; +import type { TraceFindingCount, TracePage, TraceSummary } from "../types"; vi.mock("../../../networking", () => ({ apiClient: { get: vi.fn(), post: vi.fn() }, @@ -37,7 +37,7 @@ const runs = (traceList as TracePage).data as TraceSummary[]; const lastUrl = (onUrlUpdate: ReturnType) => new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")); -const renderSection = () => +const renderSection = (canViewFindings = true) => renderWithProviders( endTime="2026-09-30T00:00" isCustomDate={false} isLiveTail={false} + canViewFindings={canViewFindings} />, ); @@ -88,6 +89,10 @@ describe("AgentTracesSection", () => { testQueryClient.clear(); vi.mocked(agentTraceListCall).mockReset(); vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); + vi.mocked(apiClient.post).mockImplementation(async (_path, options) => { + const body = options?.body as { traces: { trace_id: string; trace_ref?: string }[] }; + return body.traces.map((trace) => ({ ...trace, finding_count: null })); + }); }); it("loads the next page only once the list scrolls near its end, then stops at the last page", async () => { @@ -296,7 +301,7 @@ describe("AgentTracesSection", () => { expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); }); - it("lists every run with its input, counts and failed column", async () => { + it("lists uninvestigated runs without presenting tool errors as failures", async () => { vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); renderSection(); @@ -305,11 +310,73 @@ describe("AgentTracesSection", () => { const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans")); expect(lead).toBeDefined(); const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement; - expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument(); + expect(within(failed).queryByLabelText("2 errors")).not.toBeInTheDocument(); + expect(await within(failed).findByTitle("No conclusive investigation for this trace")).toHaveTextContent("-"); + expect(screen.getByRole("columnheader", { name: "Findings" })).toBeInTheDocument(); + expect(screen.queryByRole("columnheader", { name: "Failed" })).not.toBeInTheDocument(); expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); expect(within(failed).getByText("—")).toBeInTheDocument(); }); + it("distinguishes uninvestigated traces, completed clean investigations, and findings", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null }); + vi.mocked(apiClient.post).mockResolvedValue( + runs.map((run, index) => ({ + trace_id: run.trace_id, + trace_ref: run.trace_ref ?? "", + finding_count: [null, 0, 3][index], + })), + ); + renderSection(); + expect(await screen.findByTitle("No conclusive investigation for this trace")).toHaveTextContent("-"); + expect(await screen.findByTitle("0 findings from completed investigations")).toHaveTextContent("0"); + expect(await screen.findByTitle("3 findings from completed investigations")).toHaveTextContent("3"); + }); + + it("keeps failure labels out of the run totals above the findings table", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + data: [{ ...runs[0], status: "error", error_count: 8 }], + next_cursor: null, + }); + renderSection(); + expect(await screen.findByTestId("agent-trace-row")).toBeVisible(); + expect(screen.getByText(/1 run from/)).toBeVisible(); + expect(screen.queryByText(/failed runs|with errors/)).not.toBeInTheDocument(); + }); + + it("does not present a failed findings lookup as an uninvestigated or clean trace", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null }); + vi.mocked(apiClient.post).mockRejectedValue(new ApiError("Unavailable", 503, {})); + renderSection(); + expect(await screen.findByTitle("Could not load findings")).toHaveTextContent("Unavailable"); + expect(screen.queryByTitle("No conclusive investigation for this trace")).not.toBeInTheDocument(); + }); + + it("keeps the column picker open when findings finish loading", async () => { + const user = userEvent.setup(); + const pending = Promise.withResolvers(); + vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null }); + vi.mocked(apiClient.post).mockReturnValue(pending.promise); + renderSection(); + expect(await screen.findByTestId("agent-trace-row")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Columns" })); + expect(await screen.findByTestId("view-option-cost")).toBeVisible(); + await act(async () => { + pending.resolve([{ trace_id: runs[0].trace_id, trace_ref: runs[0].trace_ref ?? "", finding_count: 1 }]); + }); + expect(await screen.findByTitle("1 findings from completed investigations")).toBeVisible(); + expect(screen.getByTestId("view-option-cost")).toBeVisible(); + }); + + it("keeps traces available without requesting investigation data for users without access", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null }); + vi.mocked(apiClient.post).mockClear(); + renderSection(false); + expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); + expect(screen.queryByRole("columnheader", { name: "Findings" })).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + }); + it("shows the spend returned for a run", async () => { vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), @@ -517,7 +584,9 @@ describe("AgentTracesSection", () => { const user = userEvent.setup(); vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); const onUrlUpdate = vi.fn(); - const failed = filterRuns(runs, "status:error"); + const mixed = runs.map((run, index) => (index === 1 ? { ...run, status: "error" as const } : run)); + vi.mocked(agentTraceListCall).mockResolvedValue({ data: mixed, next_cursor: null }); + const failed = filterRuns(mixed, "status:error"); renderWithProviders( (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]); const runRefs = useMemo(() => runs.map(traceRefOf), [runs]); + const findings = useTraceFindings(accessToken, runs, isActive, canViewFindings); const changeRange = (hours: number, apply: (hours: number) => void) => { setZoom(null); @@ -219,6 +223,8 @@ export function AgentTracesSection({ run.error_count > 0).length; return (
{runs.length} {runs.length === 1 ? "run" : "runs"} from {new Set(runs.flatMap(traceAgentNames)).size} agents - {failed > 0 && {failed} with errors}
); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx index a99c44a98a6..3b0db88d455 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx @@ -28,6 +28,7 @@ const renderEmpty = (rangeEmpty: boolean) => { inList( { inList( { inList( { inList( { inList( ; + canViewFindings?: boolean; isLoading: boolean; error: Error | null; hasMore: boolean; @@ -46,6 +48,7 @@ const SKELETON_ROWS = Array.from({ length: 12 }, (_, i) => i); const ROW_HEIGHT = 36; const MUTED_NUM = "font-mono text-muted-foreground"; const NUM = "font-mono text-foreground"; +const FindingsContext = createContext>(new Map()); function AgentCell({ run }: { run: TraceSummary }) { const framework = traceFramework(run); @@ -62,7 +65,6 @@ function AgentCell({ run }: { run: TraceSummary }) { function InputCell({ run }: { run: TraceSummary }) { return (
- 0 ? "error" : "ok"} subtle /> {firstLine(previewText(run.input_preview)) || traceDisplayName(run)} @@ -79,6 +81,15 @@ function InputCell({ run }: { run: TraceSummary }) { ); } +function FindingCount({ run }: { run: TraceSummary }) { + const state = useContext(FindingsContext).get(runKey(run)); + if (!state || state.status === "pending") + return ; + if (state.status === "error") return Unavailable; + if (state.count === null) return -; + return {state.count.toLocaleString()}; +} + const RUN_COLUMNS: ColumnDef[] = [ { id: "time", @@ -148,16 +159,11 @@ const RUN_COLUMNS: ColumnDef[] = [ meta: { numeric: true, className: NUM }, }, { - id: "failed", - size: 72, - header: "Failed", - cell: ({ row }) => - row.original.error_count > 0 ? ( - - ) : ( - 0 - ), - meta: { numeric: true }, + id: "findings", + size: 96, + header: "Findings", + cell: ({ row }) => , + meta: { numeric: true, className: NUM }, }, { id: "open", @@ -208,6 +214,8 @@ function EmptyRuns({ rangeEmpty, onSetUpTracing }: { rangeEmpty: boolean; onSetU /** Devtool-dense runs list: one row per agent run, newest first. */ export function AgentTracesTable({ traces, + findings, + canViewFindings = true, isLoading, error, hasMore, @@ -224,7 +232,7 @@ export function AgentTracesTable({ const { columnVisibility, onColumnVisibilityChange } = usePersistedColumnVisibility("lens-traces"); const tableOptions: TableOptions = { data: traces, - columns: RUN_COLUMNS, + columns: RUN_COLUMNS.filter((column) => canViewFindings || column.id !== "findings"), defaultColumn: { size: undefined }, getRowId: runKey, autoResetAll: false, @@ -234,54 +242,56 @@ export function AgentTracesTable({ }; const table = useReactTable(tableOptions); return ( - - - - - rowHeight={() => ROW_HEIGHT} - after={ - <> - {isLoading && SKELETON_ROWS.map((row) => )} - {autoContinue && } - - } - > - {(row) => ( - - )} - - - {isLoading && ( -

- Loading runs… -

- )} - {error && ( -
- - {traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message} - - {onRetry && ( - + )} +
+ )} + {canContinue && traces.length === 0 && ( +
+ No loaded runs match these filters. + - )} -
- )} - {canContinue && traces.length === 0 && ( -
- No loaded runs match these filters. - -
- )} - {isEmpty && } - +
+ )} + {isEmpty && } + + ); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.test.ts b/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.test.ts index 583f5886f5a..bb42ae3be1f 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.test.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.test.ts @@ -7,11 +7,12 @@ const HOUR = 3600 * 1000; const START = Date.UTC(2026, 8, 30, 0, 0, 0); const range = { startMs: START, endMs: START + 10 * HOUR }; -const run = (offsetMs: number, errorCount = 0): TraceSummary => +const run = (offsetMs: number, errorCount = 0, status: TraceSummary["status"] = "ok"): TraceSummary => ({ trace_id: `t${offsetMs}`, start_time: new Date(START + offsetMs).toISOString(), error_count: errorCount, + status, }) as TraceSummary; describe("bucketRuns", () => { @@ -35,9 +36,9 @@ describe("bucketRuns", () => { expect(buckets[1].total).toBe(1); }); - it("counts runs with any errors as failed", () => { - const buckets = bucketRuns([run(HOUR, 3), run(HOUR + 1), run(HOUR + 2, 1), run(5 * HOUR)], range, 10); - expect(buckets[1]).toMatchObject({ total: 3, failed: 2 }); + it("counts failed runs without treating recovered tool errors as run failures", () => { + const buckets = bucketRuns([run(HOUR, 8), run(HOUR + 1), run(HOUR + 2, 1, "error"), run(5 * HOUR)], range, 10); + expect(buckets[1]).toMatchObject({ total: 3, failed: 1 }); expect(buckets[5]).toMatchObject({ total: 1, failed: 0 }); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.tsx index 9cf947719d3..71c334efe4f 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/TracesTimeline.tsx @@ -13,7 +13,7 @@ export function bucketRuns(runs: readonly TraceSummary[], range: TimeWindow, buc const width = (range.endMs - range.startMs) / buckets; const placed = runs.map((run) => ({ index: Math.floor((moment(run.start_time).valueOf() - range.startMs) / width), - failed: run.error_count > 0, + failed: run.status === "error", agent: traceAgentNames(run)[0] ?? "", })); return Array.from({ length: buckets }, (_, i) => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/__fixtures__/runs.ts b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/__fixtures__/runs.ts index cd886dee349..035bc2ecb64 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/__fixtures__/runs.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/__fixtures__/runs.ts @@ -36,6 +36,7 @@ const researchOverrides = { agent_names: ["researcher"], models: ["claude-opus"], error_count: 2, + status: "error" as const, }; const research = run(researchOverrides); const plain = run({ trace_id: "ccc333", name: "health", service: "cron" }); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.test.ts b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.test.ts index 40fd4afd4db..bacc2dfcd0c 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.test.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.test.ts @@ -15,12 +15,22 @@ describe("filterRuns", () => { expect(ids("gpt-5")).toEqual([]); }); - it("splits runs by status, judged by recorded errors", () => { + it("splits runs by their overall status", () => { expect(ids("status:error")).toEqual(["bbb222"]); expect(ids("-status:error")).toEqual(["aaa111", "ccc333"]); expect(ids("status:OK")).toEqual(["aaa111", "ccc333"]); }); + it.each(["ok", "unset"] as const)("keeps %s runs with recovered tool errors in non-failed filters", (status) => { + const recovered = run({ status, error_count: 8 }); + expect(filterRuns([recovered], "status:error")).toEqual([]); + expect(filterRuns([recovered], "", { agent: "", status: "error" })).toEqual([]); + expect(filterRuns([recovered], "status:ok")).toEqual([recovered]); + expect(filterRuns([recovered], "", { agent: "", status: "ok" })).toEqual([recovered]); + expect(filterRuns([recovered], "-status:error")).toEqual([recovered]); + expect(fieldValues(RUN_INDEX, [recovered], "status")).toEqual(["ok"]); + }); + it("reads agents from the trace, falling back to the service, and models from the run", () => { expect(ids("agent:triage")).toEqual(["aaa111"]); expect(ids("agent:cron")).toEqual(["ccc333"]); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.ts b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.ts index 8cd66cde22a..203f4bd03d1 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.ts @@ -19,12 +19,14 @@ export type RunField = keyof typeof RUN_FIELDS; export const RUN_QUERY: QueryLanguage = { fields: RUN_FIELDS, ops: ALL_OPERATORS }; +const runStatus = (run: TraceSummary): "ok" | "error" => (run.status === "error" ? "error" : "ok"); + /** Reads the run fields off a loaded page; free text searches trace id, input and name. */ export const RUN_INDEX: ClientIndex = { read: { name: (run) => [run.name], agent: traceAgentNames, - status: (run) => [run.error_count > 0 ? "error" : "ok"], + status: (run) => [runStatus(run)], model: (run) => run.models, input: (run) => [previewText(run.input_preview)], trace_id: (run) => [run.trace_id], @@ -38,7 +40,6 @@ export function filterRuns( filters: { agent: string; status: "all" | "ok" | "error" } = { agent: "", status: "all" }, ): TraceSummary[] { const matchesAgent = (run: TraceSummary) => !filters.agent || traceAgentNames(run).includes(filters.agent); - const matchesStatus = (run: TraceSummary) => - filters.status === "all" || (run.error_count > 0 ? "error" : "ok") === filters.status; + const matchesStatus = (run: TraceSummary) => filters.status === "all" || runStatus(run) === filters.status; return filterItems(RUN_QUERY, RUN_INDEX, runs, query).filter((run) => matchesAgent(run) && matchesStatus(run)); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.test.ts b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.test.ts index faae0491fbe..215b0d06906 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.test.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.test.ts @@ -13,7 +13,7 @@ describe("runPredicates", () => { it("turns each run field into a predicate over the per-trace rollup columns", () => { expect(predicates("agent:researcher status:error model:gpt-5 name:support trace_id:aaa111")).toEqual([ "arrayExists(x -> x ILIKE 'researcher', agents)", - "errors > 0", + "status = 'error'", "arrayExists(x -> x ILIKE 'gpt-5', models)", "name ILIKE 'support'", "trace_id ILIKE 'aaa111'", @@ -33,7 +33,7 @@ describe("runPredicates", () => { }); it("resolves a status glob statically, since status has two values", () => { - expect(predicates("status:ok")).toEqual(["errors = 0"]); + expect(predicates("status:ok")).toEqual(["status = 'ok'"]); expect(predicates("status:*")).toEqual(["true"]); expect(predicates("-status:pending")).toEqual(["NOT (false)"]); }); @@ -52,12 +52,13 @@ describe("runQuerySql", () => { expect(sql).toBe( [ "SELECT TraceId AS trace_id, any(RootName) AS name, any(RootInput) AS input, sum(ErrorCount) AS errors,", + " if(ifNull(any(RootStatus), '') IN ('STATUS_CODE_ERROR', 'error'), 'error', 'ok') AS status,", " groupUniqArrayArray(AgentNames) AS agents, groupUniqArrayArray(Models) AS models", "FROM agent_traces_by_key", "GROUP BY TraceId", `HAVING min(StartTs) >= fromUnixTimestamp64Milli(${RANGE.startMs}) AND min(StartTs) < fromUnixTimestamp64Milli(${RANGE.endMs})`, " AND arrayExists(x -> x ILIKE 'researcher', agents)", - " AND errors > 0", + " AND status = 'error'", "ORDER BY min(StartTs) DESC", "LIMIT 100", ].join("\n"), @@ -87,6 +88,6 @@ describe("runQueryCommand", () => { const command = runQueryCommand(RANGE)(query("agent:researcher status:error")); expect(command).toBe(traceQueryCommand(runQuerySql(query("agent:researcher status:error"), RANGE))); expect(command).toContain("arrayExists(x -> x ILIKE 'researcher', agents)"); - expect(command).toContain("errors > 0"); + expect(command).toContain("status = 'error'"); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.ts b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.ts index 3ca0068f1b2..19a191de043 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runSql.ts @@ -7,6 +7,7 @@ import type { SearchFilter, SearchQuery } from "@/components/shared/search/searc import type { RunField } from "./runQuery"; const RUN_ROWS = `SELECT TraceId AS trace_id, any(RootName) AS name, any(RootInput) AS input, sum(ErrorCount) AS errors, + if(ifNull(any(RootStatus), '') IN ('STATUS_CODE_ERROR', 'error'), 'error', 'ok') AS status, groupUniqArrayArray(AgentNames) AS agents, groupUniqArrayArray(Models) AS models FROM agent_traces_by_key GROUP BY TraceId`; @@ -21,7 +22,7 @@ const likePattern = (value: string): string => likeLiteral(value).replaceAll("*" const matches = (column: string, pattern: string): string => `${column} ILIKE ${sqlString(pattern)}`; const anyMatches = (column: string, pattern: string): string => `arrayExists(x -> ${matches("x", pattern)}, ${column})`; -const STATUS_PREDICATES = { error: "errors > 0", ok: "errors = 0" } as const; +const STATUS_PREDICATES = { error: "status = 'error'", ok: "status = 'ok'" } as const; /** Status has two values, so a glob over it resolves statically to one, both or neither predicate. */ function statusPredicate(value: string): string { diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFindings.ts b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFindings.ts new file mode 100644 index 00000000000..c953ffa576e --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFindings.ts @@ -0,0 +1,39 @@ +import { useQueries } from "@tanstack/react-query"; +import { chunk } from "es-toolkit"; + +import { useTracesApi } from "../api"; +import type { TraceSummary } from "../types"; + +export type TraceFindingState = { status: "ready"; count: number | null } | { status: "pending" } | { status: "error" }; + +export function useTraceFindings(accessToken: string, runs: TraceSummary[], isActive: boolean, canViewFindings = true) { + const api = useTracesApi(accessToken); + const enabled = isActive && canViewFindings; + const batches = chunk( + runs.map(({ trace_id, trace_ref }) => ({ trace_id, trace_ref: trace_ref ?? "" })), + 500, + ); + const queries = useQueries({ + queries: batches.map((traces) => ({ + queryKey: ["traceFindings", accessToken, traces], + queryFn: () => api.findings(traces), + enabled, + staleTime: 15000, + refetchInterval: enabled && api.live ? 15000 : false, + retry: false, + })), + }); + return new Map( + batches.flatMap((traces, index) => { + const query = queries[index]; + const counts = new Map(query.data?.map((result) => [result.trace_ref || result.trace_id, result.finding_count])); + return traces.map((trace): [string, TraceFindingState] => { + const key = trace.trace_ref || trace.trace_id; + if (query.isError) return [key, { status: "error" }]; + if (query.isPending) return [key, { status: "pending" }]; + if (!counts.has(key)) return [key, { status: "error" }]; + return [key, { status: "ready", count: counts.get(key) ?? null }]; + }); + }), + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/traces/types.ts b/ui/litellm-dashboard/src/components/lens/traces/types.ts index 06a2b93b4ae..e2d169e124f 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/types.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/types.ts @@ -11,6 +11,8 @@ export type SpanErrorQuery = NonNullable< paths["/v1/traces/{trace_id}/spans/{span_id}/error"]["get"]["parameters"]["query"] >; export type TraceQueryBody = components["schemas"]["TraceQueryRequest"]; +export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"]; +export type TraceFindingCount = components["schemas"]["TraceFindingCount"]; type ApiSpanDetail = paths["/v1/traces/{trace_id}/spans/{span_id}"]["get"]["responses"][200]["content"]["application/json"]; export type Span = Trace["spans"][number]; diff --git a/ui/litellm-dashboard/src/components/lens/traces/ui/StatusMark.tsx b/ui/litellm-dashboard/src/components/lens/traces/ui/StatusMark.tsx deleted file mode 100644 index 8410e1caea9..00000000000 --- a/ui/litellm-dashboard/src/components/lens/traces/ui/StatusMark.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import { AlertCircle, Check, Circle } from "lucide-react"; - -interface StatusMarkProps { - status: "ok" | "error" | "unset"; - count?: number; - subtle?: boolean; -} - -/** Run / span status glyph: red alert (+ count) for errors, a quiet muted dot or check otherwise. */ -export function StatusMark({ status, count, subtle = false }: StatusMarkProps) { - if (status === "error") { - return ( - - - {count !== undefined && {count}} - - ); - } - if (subtle) - return ( - - ); - return ; -} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6b9551f1940..8afe6e81c25 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9000,6 +9000,23 @@ export interface paths { patch?: never; trace?: never; }; + "/lens/traces/findings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Trace Findings */ + post: operations["trace_findings_lens_traces_findings_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/lens/watch-all": { parameters: { query?: never; @@ -46950,6 +46967,33 @@ export interface components { spans: components["schemas"]["Span"][]; summary: components["schemas"]["TraceSummary"]; }; + /** TraceFindingCount */ + TraceFindingCount: { + /** Finding Count */ + finding_count: number | null; + /** Trace Id */ + trace_id: string; + /** + * Trace Ref + * @default + */ + trace_ref: string; + }; + /** TraceFindingsRequest */ + TraceFindingsRequest: { + /** Traces */ + traces: components["schemas"]["TraceIdentity"][]; + }; + /** TraceIdentity */ + TraceIdentity: { + /** Trace Id */ + trace_id: string; + /** + * Trace Ref + * @default + */ + trace_ref: string; + }; /** TracePage */ TracePage: { /** Data */ @@ -62253,6 +62297,39 @@ export interface operations { }; }; }; + trace_findings_lens_traces_findings_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TraceFindingsRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TraceFindingCount"][]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; watch_all_lens_watch_all_post: { parameters: { query?: never;