scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(lens): restore compact navigation with live investigation review (#44472)

Browse source 
* feat(lens): add per-trace review models to jobs and progress

* feat(lens): append worker reviews to the job, capped, and count every review

* feat(lens): report a review with reasoning for each screened trace

* chore(ui): regenerate api types for lens job reviews

* feat(lens): type job reviews and fill them in lens fixtures

* feat(lens): add live review playback model

* feat(lens): pick the analysis model and slow single-review pacing

* feat(lens): add sample reviews for previewing the live run

* feat(lens): add live run layout with queue, reading trace and conclusions

* feat(lens): show the live run on investigations and open it from run now

* feat(lens): stream large review backlogs at 150ms or less and list newest first

* fix(lens): show the live run only for real reviews and keep fixtures test-only

* refactor(lens): restyle the live run as the native progress panel

* fix(lens): retry contended investigation updates with jittered backoff

* feat(lens): add a reading ticker line and replay for finished runs

* feat(lens): collapse the live run to an ambient line with show work

* fix(ui): crop the cerebras logo viewBox to its mark so it reads at icon size

* feat(lens): format review span previews as readable messages

* feat(lens): derive strip status, honest issue counts and drawer focus from a job

* feat(lens): track active jobs before their first review

* feat(lens): add a live trace results drawer with readable spans

* feat(lens): put the live strip under the progress bar and drop the inline panel

* feat(lens): add an ambient live strip that opens the drawer

* fix(lens): wait out provider rate limits and retry model calls four times

* style(lens): format repository contention tests

* feat(lens): read review spans as a conversation timeline

Turns spans into the user's ask, tool calls with args and results, and the agent's reply, dropping system prompts. Also handles a preview cut that lands inside the Output header.

* fix(lens): list recorded agents in the run now dialog

The run now agent field used a native datalist, whose suggestions do not show inside the modal dialog, so the agent list looked empty even though /lens/agents returned names. Use the same Combobox as investigation setup.

* feat(lens): pace live playback so each trace stays readable

Every trace now stays up for at least 1.5s. A backlog is cleared by skipping to the newest few instead of flickering through them. Conclusions count traces per check and kind, and new helpers cover share bars, group filters and flashes.

* feat(lens): keep the live run ambient until View run is clicked

The drawer no longer opens on Run or when entering a running investigation. LiveRun takes reviews as a prop so it can move to a dedicated reviews endpoint.

* feat(lens): show the live run as a two-pane trace and conclusions view

Left pane: the trace being reviewed as a readable timeline, followed by Lens's reasoning and the verdict. Right pane: ranked conclusion groups with share bars, plus a trace list you can filter.

* fix(lens): run several investigations per worker and poll every two seconds

* feat(lens): add worker slot and poll interval settings

* feat(lens): add list summaries and an incremental review filter

* perf(lens): strip reviews and run attributes from the lens list and serve reviews separately

* test(lens): cover list summaries, review polling and review access

* feat(lens): explain why a queued investigation is waiting

Works out whether no worker is connected, the worker is busy (with its running investigations and an estimated start time), or it is just being picked up.

* feat(lens): show the queue reason and what the worker is doing in the live strip

The progress header and the strip replace "Queued for your worker" with the concrete reason. While waiting, the strip lists the busy worker's investigations; click one to open it.

* feat(lens): add a review page model carrying the total reviewed count

* fix(lens): page live reviews by index so out-of-order reviews are never skipped

* feat(lens): take an index cursor on the reviews endpoint

* test(lens): cover index cursors across out-of-order and rolled-over reviews

* chore(ui): regenerate api types for the lens reviews endpoint

* feat(lens): page job reviews by index cursor

Adds api.reviews for GET /lens/{id}/runs/{job}/reviews?after=N, with a demo implementation. appendPage adds pages in arrival order and keeps the latest 200. liveJob now keys off reviewed, since the list no longer carries reviews.

* feat(ui): add a lens reviews query that polls the index cursor while live

* fix(lens): feed the live run from the reviews endpoint and keep View run open

LiveRun now gets its reviews from useJobReviews instead of the list, which no longer carries them. View run stays clickable while a run is queued or running, and before the first trace the opened view says what the worker is doing.

* fix(lens): split live conclusions into issues and patterns

A check could show up twice with the same label, once as an issue and once as a pattern.

* fix(lens): group live conclusions by check with short labels

There is now one group per check_id: issue traces are the main count and pattern traces a secondary note, so there are no duplicate red and grey cards. A long instruction falls back to the humanized check id. Adds briefReasoning and traceRows for the simplified trace list, and drops helpers nothing uses.

* feat(lens): simplify View run to traces and conclusions

The left pane is the trace list. A soft highlighter carrying the provider and model slides to the trace being reviewed, and clicking a row shows just Lens's reasoning and verdicts. The right pane keeps one conclusion card per check.

* refactor(lens): drop client-side replay in favour of real in-flight rows

Removes the playback reducer and its pacing. liveRows lists the traces the worker is reading, from job.reading, followed by completed reviews newest first, keyed by execution_id so a trace keeps its row when it finishes.

* feat(lens): show what the worker is reading and make View run obvious

Each trace in flight gets a highlighted row with the model and a live timer, and becomes its completed row in place. Completed rows show the real review time. View run is an outline button next to the progress line, and clicking anywhere on the strip opens it too.

* feat(lens): sum up a finished live run with time taken

doneLine reads like "Reviewed 30 traces in 31s with", measured from when reading started.

* feat(lens): slide one model rectangle over the traces being read

A single rounded rectangle carrying the provider logo and model wraps the real in-flight rows from job.reading. It translates and resizes over 250ms as traces finish in place. Before job.reading arrives it sits on a top slot showing the honest progress line, and when the run completes it fades out over 400ms. Rows have a fixed height and stable execution_id keys, so polls don't cause jumps or flicker.

* feat(lens): add in-flight runs to jobs and worker progress

* feat(lens): store in-flight runs from progress and clear them when a job ends

* refactor(lens): route progress, cancel and results through shared job transitions

* feat(lens): report each run as in flight when its review starts

* feat(lens): send in-flight runs with worker progress

* test(lens): cover in-flight runs across progress, old workers and terminal states

* test(lens): cover in-flight reporting under original run ids

* chore(ui): regenerate api types for lens in-flight runs

* feat(lens): model live reading lanes from in-flight runs and reviews

* feat(lens): show a now reading stage that types each trace's reasoning

* feat(lens): put the now reading stage above the trace list in View run

* fix(lens): resolve the analysis provider logo from the model catalog

* fix(lens): give demo jobs an empty in-flight list

* style(lens): format endpoint tests

* refactor(lens): name the run now handler in investigations view

* refactor(lens): name now reading conditions

* refactor(lens): name inline objects in the live run

* style(lens): format live run files

* fix(lens): keep worker settings inside the standalone worker package

* refactor(lens): keep update retry settings next to the repository

* fix(lens): start review history over when a run is reclaimed

* chore(lens): drop the unused review fixture

* refactor(lens): remove dead live helpers and use generated in-flight types

* fix(lens): keep polling a finished run until its last reviews arrive

* perf(lens): tick fast only while reasoning is typing

* fix(lens): isolate retried reviews and finding identities

* fix(lens): space the model name in run summary

* Update review.md

* test(lens): await trace status filter option

---------

Co-authored-by: moe-berri <moe@berri.ai>
This commit is contained in:
ishaan-berri 2026-10-05 14:50:43 -07:00 • committed by GitHub
parent 76ffec9238
commit a3a52466a5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
65 changed files with 4066 additions and 666 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
litellm/proxy/lens/analysis.py
View file

@ -1,11 +1,13 @@
import asyncio
import json
import time
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
from contextlib import aclosing
from datetime import datetime, timezone
from functools import reduce
from itertools import chain, islice
from types import MappingProxyType
from typing import Final, Literal, TypeAlias, TypeVar
from typing import Final, Literal, Protocol, TypeAlias, TypeVar
from pydantic import Field, TypeAdapter, ValidationError
@ -16,10 +18,14 @@ from .models import (
Execution,
ExecutionContent,
FindingDraft,
InFlight,
ModelRequest,
ModelResult,
Record,
Result,
Review,
ReviewSpan,
ReviewVerdict,
RunAssessment,
Sample,
TracePart,
@ -38,6 +44,7 @@ class Observation(Record):
class Extraction(Record):
observations: tuple[Observation, ...] = ()
cannot_assess: bool = False
reasoning: str = Field(default="", max_length=800)
class SpanRead(Record):
@ -84,6 +91,8 @@ class Examined(Record):
partial: bool
cannot_assess: bool
error: str = ""
reasoning: str = ""
shown: tuple[TracePart, ...] = ()
class Investigation(Record):
@ -94,7 +103,17 @@ class Investigation(Record):
ModelCall: TypeAlias = Callable[[ModelRequest], Awaitable[ModelResult]]
ReadContent: TypeAlias = Callable[[str, str, int], Awaitable[ExecutionContent]]
ReportProgress: TypeAlias = Callable[[str, Coverage], Awaitable[None]]
class ReportProgress(Protocol):
def __call__(
self,
stage: str,
coverage: Coverage,
review: Review | None = None,
reading: tuple[InFlight, ...] | None = None,
/,
) -> Awaitable[None]: ...
ResponseT = TypeVar("ResponseT", bound=Record)
@ -326,7 +345,9 @@ async def extract_stored(
request: Final = ModelRequest(purpose="extract", prompt=prompt)
if must_decide:
final: Final = await structured_response(request, Extraction, model)
return TraceReview(observations=final.observations, cannot_assess=final.cannot_assess)
return TraceReview(
observations=final.observations, cannot_assess=final.cannot_assess, reasoning=final.reasoning
)
return await structured_response(request, TraceReview, model)
response: TraceReview
@ -375,6 +396,7 @@ async def extract_stored(
parts=evidence,
partial=page.partial or page.next_cursor is not None or bool(response.reads) or invalid_observations,
cannot_assess=not span_count or response.cannot_assess or bool(response.reads) or invalid_observations,
reasoning=response.reasoning,
)
reviews: Final = tuple([await examine(catalog) for catalog in store.catalogs(root_count)])
@ -383,12 +405,52 @@ async def extract_stored(
retained: Final = tuple(
p for p in chain.from_iterable(r.parts for r in reviews) if p.span_id in cited or not p.parent_span_id
)
leading: Final = MappingProxyType(
{
p.span_id: p
for p in (*((first_root,) if first_root else ()), *(p for p in store.parts() if p.span_id in cited))
}
)
shown: Final = islice(chain(leading.values(), (p for p in store.parts() if p.span_id not in leading)), 8)
return Examined(
execution=execution,
observations=observations,
parts=tuple(dict.fromkeys((*retained, *((first_root,) if first_root else ())))),
partial=any(r.partial for r in reviews),
cannot_assess=not reviews or all(r.cannot_assess for r in reviews),
reasoning=" ".join(r.reasoning for r in reviews if r.reasoning),
shown=tuple(p.model_copy(update=MappingProxyType({"content": overview_content(p, root_count)})) for p in shown),
)
def review_of(examined: Examined, model: str, duration_ms: int, at: datetime) -> Review:
execution: Final = examined.execution
cited: Final = frozenset(
(e.execution_id, e.span_id) for e in chain.from_iterable(o.evidence for o in examined.observations)
)
return Review(
execution_id=execution.id,
trace_id=execution.trace_id,
agent=execution.service or execution.name,
name=execution.name,
spans=tuple(
ReviewSpan(
span_id=p.span_id,
name=p.name[:120],
kind=p.kind[:40],
preview=p.content[:240],
cited=(p.execution_id, p.span_id) in cited,
)
for p in examined.shown[:8]
),
reasoning=examined.reasoning[:800],
verdicts=tuple(
ReviewVerdict(check_id=o.check_id, kind=o.kind, summary=o.summary[:300]) for o in examined.observations
),
cannot_assess=examined.cannot_assess,
model=model,
duration_ms=max(duration_ms, 0),
at=at,
)
@ -630,8 +692,29 @@ async def analyze_sample(
)
)
def original(identity: str) -> str:
return originals[identity].id
async def progress_original(
stage: str, coverage: Coverage, review: Review | None = None, reading: tuple[InFlight, ...] | None = None, /
) -> None:
await progress(
stage,
coverage,
review and review.model_copy(update=MappingProxyType({"execution_id": original(review.execution_id)})),
None
if reading is None
else tuple(
r.model_copy(update=MappingProxyType({"execution_id": original(r.execution_id)})) for r in reading
),
)
result: Final = await _analyze_sample(
claim, sample.model_copy(update=MappingProxyType({"executions": executions})), read_alias, model, progress
claim,
sample.model_copy(update=MappingProxyType({"executions": executions})),
read_alias,
model,
progress_original,
)
return result.model_copy(
update=MappingProxyType(
@ -851,16 +934,36 @@ async def merge_candidates(
async def examine_executions(
claim: Claim, sample: Sample, read: ReadContent, model: ModelCall, progress: ReportProgress
) -> AsyncIterator[Examined]:
async def examine(execution: Execution) -> Examined:
return await extract(claim, execution, read, model)
reading: tuple[InFlight, ...] = () # rebind-ok: the in-flight set changes as each read starts and finishes
screened = 0 # rebind-ok: counts finished reads for progress
reporting: Final = asyncio.Lock()
async def report(change: Callable[[tuple[InFlight, ...]], tuple[InFlight, ...]], review: Review | None) -> None:
nonlocal reading
async with reporting:
reading = change(reading)
coverage: Final = Coverage(eligible=sample.eligible, selected=len(sample.executions), screened=screened)
await progress("Reading executions", coverage, review, reading)
async def examine(execution: Execution) -> tuple[Examined, Review]:
entry: Final = InFlight(
execution_id=execution.id,
trace_id=execution.trace_id,
agent=execution.service or execution.name,
started_at=datetime.now(timezone.utc),
)
await report(lambda current: (*current, entry), None)
started: Final = time.perf_counter()
examined: Final = await extract(claim, execution, read, model)
elapsed: Final = round((time.perf_counter() - started) * 1000)
return examined, review_of(examined, claim.job.settings.model, elapsed, datetime.now(timezone.utc))
await progress("Reading executions", Coverage(eligible=sample.eligible, selected=len(sample.executions)))
completed: Final = iter(range(1, len(sample.executions) + 1))
async with aclosing(concurrent_results(sample.executions, examine, claim.job.settings.concurrency)) as results:
async for item in results:
await progress(
"Reading executions",
Coverage(eligible=sample.eligible, selected=len(sample.executions), screened=next(completed)),
async for item, review in results:
screened += 1
await report(
lambda current, done=item.execution.id: tuple(r for r in current if r.execution_id != done), review
)
yield item

45
litellm/proxy/lens/endpoints.py
View file

@ -35,10 +35,10 @@ from litellm.proxy.lens.models import (
ModelResult,
Progress,
Result,
ReviewPage,
RunRequest,
Sample,
Scope,
Step,
WatchAllResult,
WatchSkipped,
Worker,
@ -48,16 +48,20 @@ from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_ima
from litellm.proxy.lens.repository import LensRepository, WriterDatabase
from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
from litellm.proxy.lens.state import (
add_step,
apply_progress,
can_access,
cancel_job,
claim_job,
current_job,
end_job,
merge_finding,
next_scan_start,
queue_job,
replace_job,
reviews_after,
scheduled_window,
snapshot_finding,
summarized,
)
from litellm.proxy.tracing_runtime import provide_storage
@ -200,7 +204,7 @@ async def validate_workers(settings: LensSettings, scope: Scope) -> None:
async def list_lenses(auth: Auth, storage: StorageDep) -> LensList:
scope: Final = user_scope(auth)
return LensList(
lenses=tuple(e for e in await repository().lenses() if can_access(scope, e.scope)),
lenses=tuple(summarized(e) for e in await repository().lenses() if can_access(scope, e.scope)),
workers=tuple(w for w in await repository().workers() if can_access(scope, w.scope)),
tracing_enabled=storage is not None,
)
@ -330,7 +334,7 @@ async def run_lens(lens_id: str, body: RunRequest, auth: Auth) -> Lens:
@router.get("/{lens_id}", response_model=Lens)
async def read_lens(lens_id: str, auth: Auth) -> Lens:
return await get_lens(lens_id, user_scope(auth))
return summarized(await get_lens(lens_id, user_scope(auth)))
@router.get("/{lens_id}/runs", response_model=tuple[Job, ...])
@ -351,23 +355,17 @@ async def read_run(lens_id: str, job_id: str, auth: Auth) -> Job:
return job
@router.get("/{lens_id}/runs/{job_id}/reviews", response_model=ReviewPage)
async def read_reviews(lens_id: str, job_id: str, auth: Auth, after: int = Query(default=0, ge=0)) -> ReviewPage:
return reviews_after(await read_run(lens_id, job_id, auth), after)
@router.post("/{lens_id}/cancel", response_model=Lens)
async def cancel_lens(lens_id: str, auth: Auth) -> Lens:
await get_lens(lens_id, user_scope(auth, write=True))
now: Final = datetime.now(timezone.utc)
def cancel(e: Lens) -> Lens:
job: Final = current_job(e)
if job is None:
return e
cancelled: Final = job.model_copy(
update=MappingProxyType({"status": "cancelled", "stage": "Cancelled", "finished_at": now})
)
return replace_job(e, cancelled).model_copy(
update=MappingProxyType({"next_run_at": now + timedelta(minutes=e.settings.interval_minutes)})
)
return required(await repository().update(lens_id, cancel))
return required(await repository().update(lens_id, lambda e: cancel_job(e, now)))
@router.patch("/{lens_id}/findings/{finding_id}", response_model=Lens)
@ -505,15 +503,7 @@ async def progress(lens_id: str, job_id: str, body: Progress, worker: WorkerAuth
job: Final = current_job(e)
if job is None or job.id != job_id or job.worker_id != worker.id:
return e
renewed: Final = job.model_copy(
update=MappingProxyType(
{"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
)
)
return replace_job(
e,
renewed if body.stage == job.stage else add_step(renewed, Step(at=now, kind="stage", label=body.stage)),
)
return replace_job(e, apply_progress(job, body, now))
required(await repository().update(lens_id, renew))
await repository().heartbeat(worker.id, now.isoformat())
@ -639,12 +629,9 @@ async def result(lens_id: str, job_id: str, body: Result, worker: WorkerAuth, st
merged_ids: Final = frozenset(f.id for f in merged)
return replace_job(
e,
active.model_copy(
end_job(active, "failed" if body.error else "completed", now).model_copy(
update=MappingProxyType(
{
"status": "failed" if body.error else "completed",
"stage": "Failed" if body.error else "Complete",
"finished_at": now,
"coverage": active.coverage if body.error else body.coverage,
"error": body.error,
"assessments": body.assessments,

48
litellm/proxy/lens/models.py
View file

@ -207,6 +207,49 @@ class Step(Record):
cost: float = 0
MAX_REVIEWS = 60
class ReviewSpan(Record):
span_id: str
name: str = Field(max_length=120)
kind: str = Field(max_length=40)
preview: str = Field(max_length=240)
cited: bool = False
class ReviewVerdict(Record):
check_id: str
kind: Literal["issue", "pattern"]
summary: str = Field(max_length=300)
class Review(Record):
execution_id: str
trace_id: str
agent: str
name: str
spans: tuple[ReviewSpan, ...] = Field(default=(), max_length=8)
reasoning: str = Field(default="", max_length=800)
verdicts: tuple[ReviewVerdict, ...] = ()
cannot_assess: bool = False
model: str
duration_ms: int = Field(ge=0)
at: datetime
class ReviewPage(Record):
reviews: tuple[Review, ...]
reviewed: int
class InFlight(Record):
execution_id: str
trace_id: str
agent: str
started_at: datetime
class Job(Record):
id: str
status: Literal["queued", "running", "completed", "failed", "cancelled"] = "queued"
@ -227,6 +270,9 @@ class Job(Record):
findings: tuple[Finding, ...] | None = None
assessments: tuple[RunAssessment, ...] = ()
steps: tuple[Step, ...] = ()
reviews: tuple[Review, ...] = ()
reviewed: int = 0
reading: tuple[InFlight, ...] = ()
trigger: Literal["schedule", "manual"] = "schedule"
@ -307,6 +353,8 @@ class Claim(Record):
class Progress(Record):
stage: str = Field()
coverage: Coverage = Coverage()
review: Review | None = None
reading: tuple[InFlight, ...] | None = None
class Result(Record):

2
litellm/proxy/lens/prompts/review.md
View file

@ -26,3 +26,5 @@ If you need more evidence, return reads; otherwise return reads=[] and your fina
Carry forward still-valid earlier observations and remove disproved ones.
cannot_assess means insufficient evidence to assess this run, not absence of an issue.
Never manufacture an issue just to produce a result.
Set reasoning to 1-3 plain sentences: what the agent was asked, what happened, and why your observations follow, or why the run is fine.
Keep reasoning under 800 characters and do not quote any secrets or long trace text in it.

17
litellm/proxy/lens/repository.py
View file

@ -1,4 +1,6 @@
import asyncio
import json
import random
from collections.abc import AsyncIterator, Awaitable, Callable
from types import MappingProxyType
from typing import Final, Protocol
@ -19,11 +21,14 @@ class Row(BaseModel):
_ROWS: Final = TypeAdapter(tuple[Row, ...])
UPDATE_ATTEMPTS: Final = 40
UPDATE_BACKOFF_SECONDS: Final = 0.02
class LensRepository:
def __init__(self, db: Database) -> None:
def __init__(self, db: Database, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep) -> None:
self.db: Final = db
self.sleep: Final = sleep
async def lenses(self) -> tuple[Lens, ...]:
rows: Final = _ROWS.validate_python(await self.db.query_raw('SELECT data FROM "LiteLLM_Lens" ORDER BY id'))
@ -47,12 +52,18 @@ class LensRepository:
return lens
async def update(
self, lens_id: str, transform: Callable[[Lens], Lens], attempts: int = 8, *, changed_only: bool = False
self,
lens_id: str,
transform: Callable[[Lens], Lens],
attempts: int = UPDATE_ATTEMPTS,
*,
changed_only: bool = False,
) -> Lens | None:
for _ in range(attempts):
for attempt in range(attempts):
completed, updated = await self._try_update(lens_id, transform, changed_only)
if completed:
return updated
await self.sleep(random.uniform(0, UPDATE_BACKOFF_SECONDS * min(attempt + 1, 8)))
return None
async def _try_update(

77
litellm/proxy/lens/state.py
View file

@ -4,12 +4,17 @@ from types import MappingProxyType
from typing import Final, Literal
from litellm.proxy.lens.models import (
MAX_REVIEWS,
MAX_STEPS,
Finding,
FindingDraft,
Job,
Lens,
LensSettings,
Progress,
Review,
ReviewPage,
Sample,
Scope,
Step,
Worker,
@ -82,6 +87,45 @@ def add_step(job: Job, step: Step) -> Job:
return job.model_copy(update=MappingProxyType({"steps": (*job.steps, step)[-MAX_STEPS:]}))
def end_job(job: Job, status: Literal["completed", "failed", "cancelled"], now: datetime) -> Job:
stage: Final = {"completed": "Complete", "failed": "Failed", "cancelled": "Cancelled"}[status]
return job.model_copy(
update=MappingProxyType({"status": status, "stage": stage, "finished_at": now, "reading": ()})
)
def cancel_job(lens: Lens, now: datetime) -> Lens:
job: Final = current_job(lens)
if job is None:
return lens
return replace_job(lens, end_job(job, "cancelled", now)).model_copy(
update=MappingProxyType({"next_run_at": now + timedelta(minutes=lens.settings.interval_minutes)})
)
def apply_progress(job: Job, progress: Progress, now: datetime) -> Job:
updates: Final = MappingProxyType(
{
"stage": progress.stage,
"coverage": progress.coverage,
"lease_until": now + timedelta(minutes=5),
"reading": job.reading if progress.reading is None else progress.reading,
}
)
renewed: Final = add_review(job.model_copy(update=updates), progress.review)
if progress.stage == job.stage:
return renewed
return add_step(renewed, Step(at=now, kind="stage", label=progress.stage))
def add_review(job: Job, review: Review | None) -> Job:
if review is None:
return job
return job.model_copy(
update=MappingProxyType({"reviews": (*job.reviews, review)[-MAX_REVIEWS:], "reviewed": job.reviewed + 1})
)
def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
job: Final = current_job(lens)
if job is None or not can_access(worker.scope, lens.scope):
@ -91,15 +135,8 @@ def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
if job.attempts >= 3:
return replace_job(
lens,
job.model_copy(
update=MappingProxyType(
{
"status": "failed",
"stage": "Failed",
"error": "Worker disconnected repeatedly",
"finished_at": now,
}
)
end_job(job, "failed", now).model_copy(
update=MappingProxyType({"error": "Worker disconnected repeatedly"})
),
).model_copy(update=MappingProxyType({"next_run_at": now + timedelta(minutes=lens.settings.interval_minutes)}))
return replace_job(
@ -112,6 +149,9 @@ def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
"worker_id": worker.id,
"lease_until": now + timedelta(minutes=5),
"attempts": job.attempts + 1,
"reviews": (),
"reviewed": 0,
"reading": (),
}
)
),
@ -188,3 +228,22 @@ def snapshot_finding(lens: Lens, draft: FindingDraft, revision: int, now: dateti
}
)
)
def without_attributes(sample: Sample) -> Sample:
executions: Final = tuple(e.model_copy(update=MappingProxyType({"metadata": ()})) for e in sample.executions)
return sample.model_copy(update=MappingProxyType({"executions": executions}))
def summarized_job(job: Job) -> Job:
sample: Final = without_attributes(job.sample) if job.sample else None
return job.model_copy(update=MappingProxyType({"reviews": (), "sample": sample}))
def summarized(lens: Lens) -> Lens:
return lens.model_copy(update=MappingProxyType({"jobs": tuple(summarized_job(job) for job in lens.jobs)}))
def reviews_after(job: Job, after: int) -> ReviewPage:
first_kept: Final = job.reviewed - len(job.reviews)
return ReviewPage(reviews=job.reviews[max(0, after - first_kept) :], reviewed=job.reviewed)

63
litellm/proxy/lens/worker.py
View file

@ -10,10 +10,25 @@ import httpx
from pydantic import BaseModel, ConfigDict, ValidationError
from .analysis import AnalysisResponseError, analyze_sample, validation_details
from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult, Progress, Result, Sample
from .models import (
Claim,
Coverage,
ExecutionContent,
InFlight,
ModelRequest,
ModelResult,
Progress,
Result,
Review,
Sample,
)
from .release import PROTOCOL_VERSION, release_tag
logger: Final = logging.getLogger("litellm.lens.worker")
MODEL_RETRIES: Final = 4
MODEL_RETRY_MAX_SECONDS: Final = 60.0
SLOTS: Final = 3
POLL_SECONDS: Final = 2.0
class ClaimedJobIdentity(BaseModel):
@ -37,6 +52,17 @@ class ModelErrorEnvelope(BaseModel):
detail: PublicModelError
def retry_delay(error: httpx.TransportError | httpx.HTTPStatusError, attempt: int) -> float:
backoff: Final = float(min(2**attempt, MODEL_RETRY_MAX_SECONDS))
if not isinstance(error, httpx.HTTPStatusError):
return backoff
requested: Final = error.response.headers.get("retry-after", "")
try:
return min(max(float(requested), backoff), MODEL_RETRY_MAX_SECONDS)
except ValueError:
return backoff
def failure_message(error: Exception) -> str:
if isinstance(error, AnalysisResponseError):
return str(error)
@ -116,11 +142,23 @@ class LensWorker:
503,
504,
)
if not retryable or attempt >= 2:
if not retryable or attempt >= MODEL_RETRIES:
raise
await self.sleep(2**attempt)
await self.sleep(retry_delay(exc, attempt))
return await self.model_request(path, body, attempt + 1)
async def serve(self, slots: int, poll_seconds: float) -> None:
await asyncio.gather(*(self.slot(poll_seconds) for _ in range(slots)))
async def slot(self, poll_seconds: float) -> None:
while True:
try:
if await self.run_once():
continue
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Worker could not reach Lens (%s)", type(exc).__name__)
await self.sleep(poll_seconds)
async def report_unreadable_claim(self, identity: ClaimIdentity) -> None:
failure: Final = await self.client.post(
f"/lens/worker/{identity.lens_id}/{identity.job.id}/result",
@ -169,9 +207,16 @@ class LensWorker:
result.raise_for_status()
return ExecutionContent.model_validate(result.json())
async def progress(stage: str, coverage: Coverage) -> None:
async def progress(
stage: str,
coverage: Coverage,
review: Review | None = None,
reading: tuple[InFlight, ...] | None = None,
/,
) -> None:
result: Final = await self.client.post(
prefix + "/progress", json=Progress(stage=stage, coverage=coverage).model_dump()
prefix + "/progress",
json=Progress(stage=stage, coverage=coverage, review=review, reading=reading).model_dump(mode="json"),
)
result.raise_for_status()
@ -222,13 +267,7 @@ async def main() -> None:
async with httpx.AsyncClient(
base_url=url, headers=MappingProxyType({"Authorization": f"Bearer {token}"}), timeout=180
) as client:
worker: Final = LensWorker(client)
while True:
try:
await worker.run_once()
except (httpx.HTTPError, ValueError) as exc:
logger.warning("Worker could not reach Lens (%s)", type(exc).__name__)
await asyncio.sleep(10)
await LensWorker(client).serve(SLOTS, POLL_SECONDS)
if __name__ == "__main__":

181
tests/unit/proxy/lens/test_analysis.py
View file

@ -13,8 +13,10 @@ from litellm.proxy.lens.models import (
Evidence,
Execution,
ExecutionContent,
InFlight,
ModelRequest,
ModelResult,
Review,
Sample,
TracePart,
)
@ -66,8 +68,10 @@ async def test_parallel_review_shares_one_model_limit_and_cleans_up(outcome: str
finally:
exited.put(request.prompt)
async def progress(stage: str, coverage: Coverage) -> None:
if stage == "Reading executions":
async def progress(
stage: str, coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
if stage == "Reading executions" and (_reading is None or _review is not None):
counts.put(coverage.screened)
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
@ -117,7 +121,9 @@ async def test_independent_investigations_overlap_and_report_completions() -> No
async def read(_execution_id: str, _cursor: str, _offset: int) -> ExecutionContent:
pytest.fail("Inconclusive decisions must not fetch evidence")
async def progress(stage: str, coverage: Coverage) -> None:
async def progress(
stage: str, coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
assert stage == "Checking original evidence"
progress_counts.put(coverage.investigated)
@ -464,7 +470,9 @@ async def test_grouping_consolidates_prior_batches_and_reports_real_progress() -
)
stages: Final = iter((0, 1))
async def progress(stage: str, coverage: Coverage) -> None:
async def progress(
stage: str, coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
assert stage == "Grouping observations"
assert coverage.grouping_batches == 2
assert coverage.grouped_batches == next(stages)
@ -558,7 +566,9 @@ async def test_thousands_of_matching_runs_keep_all_members_without_a_growing_mod
cost=0,
)
async def progress(_stage: str, coverage: Coverage) -> None:
async def progress(
_stage: str, coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
counts.put(coverage.grouped_batches)
batches: Final = observation_batches(observations)
@ -632,7 +642,9 @@ async def test_review_keeps_original_ids_in_per_run_assessments() -> None:
async def model(_request: ModelRequest) -> ModelResult:
return ModelResult(content='{"observations":[],"cannot_assess":false}', cost=0)
async def progress(_stage: str, _coverage: Coverage) -> None:
async def progress(
_stage: str, _coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
pass
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
@ -891,7 +903,9 @@ async def test_final_registry_reconciles_patterns_split_across_pages() -> None:
)
return ModelResult(content=Clusters(candidates=grouped).model_dump_json(), cost=0)
async def progress(_stage: str, _coverage: Coverage) -> None:
async def progress(
_stage: str, _coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
return None
result: Final = await cluster_batches((observations,), model, progress, Coverage())
@ -918,7 +932,9 @@ async def test_distinct_patterns_are_consolidated_in_batches_without_losing_runs
payload: Final = json.loads(request.prompt)
return ModelResult(content=json.dumps({"candidates": payload["candidates"]}), cost=0)
async def progress(_stage: str, _coverage: Coverage) -> None:
async def progress(
_stage: str, _coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
pass
result: Final = await cluster_batches(observation_batches(observations), model, progress, Coverage())
@ -950,7 +966,9 @@ async def test_invalid_candidate_response_preserves_other_findings_and_reports_i
return ModelResult(content="not JSON", cost=0)
return ModelResult(content=json.dumps({"action": "submit", "finding": finding("run").model_dump()}), cost=0)
async def progress(_stage: str, coverage: Coverage) -> None:
async def progress(
_stage: str, coverage: Coverage, _review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
counts.put(coverage.inconclusive)
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
@ -1196,3 +1214,148 @@ async def test_investigator_can_read_all_evidence_pages_across_successive_span_b
assert result.error == ""
assert tuple(seen.get_nowait() for _ in range(seen.qsize())) == tuple(p.span_id for p in parts)
assert tuple(read_cursors.get_nowait() for _ in range(read_cursors.qsize())) == ("", "span039")
def test_review_flags_only_spans_cited_by_this_runs_observations() -> None:
from litellm.proxy.lens.analysis import Observation, review_of
execution: Final = Execution(
id="run1",
source="traces",
trace_id="trace-1",
team_id="",
name="task",
start_time="",
span_count=3,
service="bot",
)
shown: Final = tuple(
TracePart(execution_id="run1", span_id=span, name=span, kind="tool", content=f"{span} output")
for span in ("root", "search", "answer")
)
observation: Final = Observation(
check_id="retries",
summary="Search failed twice",
evidence=(
Evidence(execution_id="run1", span_id="search", quote="search output"),
Evidence(execution_id="other", span_id="answer", quote="answer output"),
),
)
examined: Final = Examined(
execution=execution,
observations=(observation,),
parts=shown,
partial=False,
cannot_assess=False,
reasoning="Asked to search; it retried without recovering.",
shown=shown,
)
review: Final = review_of(examined, "cerebras/model", 42, NOW)
assert tuple((s.span_id, s.cited) for s in review.spans) == (("root", False), ("search", True), ("answer", False))
assert (review.agent, review.trace_id, review.duration_ms) == ("bot", "trace-1", 42)
assert review.reasoning == examined.reasoning
assert tuple((v.check_id, v.summary) for v in review.verdicts) == (("retries", "Search failed twice"),)
@pytest.mark.asyncio
async def test_each_screened_run_reports_a_review_with_the_models_reasoning() -> None:
from litellm.proxy.lens.analysis import analyze_sample
execution: Final = Execution(
id="opaque-original", source="traces", trace_id="trace", team_id="", name="task", start_time="", span_count=2
)
reasoning: Final = "The user asked for a refund; the tool timed out and the agent gave up."
reviews: Final = SimpleQueue[Review]()
async def read(identity: str, _cursor: str, _offset: int) -> ExecutionContent:
return ExecutionContent(
execution=execution,
parts=(
TracePart(execution_id=identity, span_id="a-root", name="agent", kind="agent", content="Refund please"),
TracePart(
execution_id=identity,
span_id="b-tool",
parent_span_id="a-root",
name="refund",
kind="tool",
content="Tool timeout",
),
),
)
async def model(request: ModelRequest) -> ModelResult:
if request.purpose == "cluster":
return ModelResult(content='{"candidates":[]}', cost=0)
if request.purpose == "investigate":
return ModelResult(content='{"action":"inconclusive"}', cost=0)
return ModelResult(
content=json.dumps(
{
"reasoning": reasoning,
"observations": [
{
"check_id": "retries",
"summary": "Gave up after a timeout",
"evidence": [{"execution_id": "r0", "span_id": "b-tool", "quote": "Tool timeout"}],
}
],
}
),
cost=0,
)
async def progress(
_stage: str, _coverage: Coverage, review: Review | None = None, _reading: tuple[InFlight, ...] | None = None, /
) -> None:
if review is not None:
reviews.put(review)
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
await analyze_sample(claim, Sample(executions=(execution,), eligible=1), read, model, progress)
review: Final = reviews.get_nowait()
assert reviews.empty()
assert review.execution_id == execution.id
assert review.reasoning == reasoning
assert review.model == claim.job.settings.model
assert tuple((s.span_id, s.cited) for s in review.spans) == (("a-root", False), ("b-tool", True))
assert tuple(v.summary for v in review.verdicts) == ("Gave up after a timeout",)
@pytest.mark.asyncio
async def test_a_run_is_reported_in_flight_under_its_original_id_until_its_review_arrives() -> None:
from litellm.proxy.lens.analysis import analyze_sample
execution: Final = Execution(
id="opaque-original", source="traces", trace_id="trace", team_id="", name="task", start_time="", span_count=1
)
reports: Final = SimpleQueue[tuple[str | None, tuple[str, ...] | None]]()
async def read(identity: str, _cursor: str, _offset: int) -> ExecutionContent:
return ExecutionContent(
execution=execution,
parts=(TracePart(execution_id=identity, span_id="s", name="agent", kind="agent", content="Hi"),),
)
async def model(request: ModelRequest) -> ModelResult:
if request.purpose == "cluster":
return ModelResult(content='{"candidates":[]}', cost=0)
return ModelResult(content='{"observations":[]}', cost=0)
async def progress(
stage: str, _coverage: Coverage, review: Review | None = None, reading: tuple[InFlight, ...] | None = None, /
) -> None:
if stage == "Reading executions":
reports.put(
(
review and review.execution_id,
None if reading is None else tuple(f"{r.execution_id}:{r.trace_id}" for r in reading),
)
)
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
await analyze_sample(claim, Sample(executions=(execution,), eligible=1), read, model, progress)
assert tuple(reports.get_nowait() for _ in range(reports.qsize())) == (
(None, None),
(None, ("opaque-original:trace",)),
("opaque-original", ()),
)

10
tests/unit/proxy/lens/test_endpoints.py
View file

@ -10,6 +10,7 @@ from litellm import Router
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.lens.endpoints import (
list_agents,
read_reviews,
run_settings,
run_window,
user_scope,
@ -174,6 +175,15 @@ async def test_incompatible_worker_is_rejected_before_claiming_work(
assert "Upgrade" in error.value.detail
@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, None))
@pytest.mark.asyncio
async def test_regular_keys_cannot_poll_live_reviews(role: LitellmUserRoles | None) -> None:
auth: Final = UserAPIKeyAuth(user_role=role, team_id="team", token="hashed-test-key")
with pytest.raises(HTTPException) as error:
await read_reviews("lens", "job", auth)
assert error.value.status_code == 403
@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, None))
def test_regular_keys_cannot_read_lens_results(role: LitellmUserRoles | None) -> None:
auth: Final = UserAPIKeyAuth(user_role=role, team_id="team", token="hashed-test-key")

65
tests/unit/proxy/lens/test_repository.py Normal file
View file

@ -0,0 +1,65 @@
from datetime import datetime, timezone
from typing import Final
import pytest
from litellm.proxy.lens.models import Check, Lens, LensSettings, Scope
from litellm.proxy.lens.repository import UPDATE_ATTEMPTS, LensRepository, Row
NOW: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
STORED: Final = Lens(
id="lens",
scope=Scope(team_id="alpha"),
settings=LensSettings(
name="Swarm", model="cerebras/gpt-oss-120b", checks=(Check(id="c", instruction="Find loops"),)
),
created_at=NOW,
next_run_at=NOW,
budget_month=NOW.strftime("%Y-%m"),
)
class ContendedDatabase:
def __init__(self, losses: int) -> None:
self.losses: Final = losses
self.writes = 0 # rebind-ok: counts write attempts made under contention
async def query_raw(self, query: str, *args: object) -> object:
if query.startswith("SELECT data FROM"):
return (Row(data=STORED.model_dump(mode="json")),)
self.writes += 1
return (Row(data=1 if self.writes > self.losses else 0),)
async def execute_raw(self, query: str, *args: object) -> int:
return 0
async def no_wait(_: float) -> None:
return None
def renamed(lens: Lens) -> Lens:
return lens.model_copy(update={"settings": lens.settings.model_copy(update={"name": "Swarm (renamed)"})})
@pytest.mark.asyncio
async def test_update_survives_the_contention_of_a_fast_model_writing_every_review() -> None:
db: Final = ContendedDatabase(losses=12)
updated: Final = await LensRepository(db, sleep=no_wait).update("lens", renamed)
assert updated is not None
assert updated.settings.name == "Swarm (renamed)"
assert db.writes == 13
@pytest.mark.asyncio
async def test_update_backs_off_between_lost_writes_and_gives_up_after_the_limit() -> None:
waits: list[float] = [] # mutable-ok: records each backoff the repository requests
async def record(seconds: float) -> None:
waits.append(seconds)
db: Final = ContendedDatabase(losses=UPDATE_ATTEMPTS)
assert await LensRepository(db, sleep=record).update("lens", renamed) is None
assert db.writes == UPDATE_ATTEMPTS
assert len(waits) == UPDATE_ATTEMPTS
assert all(w >= 0 for w in waits)

142
tests/unit/proxy/lens/test_state.py
View file

@ -1,31 +1,46 @@
from datetime import datetime, timedelta, timezone
from functools import reduce
from typing import Final
from typing import Final, Literal
import pytest
from litellm.proxy.lens.models import (
MAX_REVIEWS,
MAX_STEPS,
AgentTestCase,
Check,
Evidence,
Execution,
FindingDraft,
InFlight,
IssueBrief,
Job,
Lens,
LensSettings,
MetadataFilter,
Progress,
Review,
Sample,
Scope,
Step,
Worker,
)
from litellm.proxy.lens.state import (
add_review,
add_step,
apply_progress,
can_access,
cancel_job,
claim_job,
current_job,
end_job,
merge_finding,
next_scan_start,
queue_job,
renew_budget,
replace_job,
reviews_after,
summarized,
)
NOW: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
@ -345,3 +360,128 @@ def test_calendar_overflow_is_rejected_without_the_old_history_and_interval_caps
assert getattr(accepted, field) == 100000
with pytest.raises(ValidationError, match="supported calendar range"):
LensSettings.model_validate({**lens().settings.model_dump(), field: 10**30})
def review(index: int) -> Review:
return Review(
execution_id=f"run-{index}", trace_id="t", agent="support", name="task", model="analysis", duration_ms=1, at=NOW
)
def test_reviews_keep_the_newest_window_while_counting_every_review() -> None:
job: Final = queue_job(lens(), NOW, "job").jobs[0]
grown: Final = reduce(add_review, tuple(review(i) for i in range(MAX_REVIEWS + 3)), job)
assert grown.reviewed == MAX_REVIEWS + 3
assert len(grown.reviews) == MAX_REVIEWS
assert grown.reviews[0].execution_id == "run-3"
assert grown.reviews[-1].execution_id == f"run-{MAX_REVIEWS + 2}"
def test_reclaimed_run_starts_its_review_history_over() -> None:
queued: Final = queue_job(lens(), NOW, "job")
first: Final = claim_job(queued, worker(), NOW)
reviewed: Final = replace_job(first, reduce(add_review, (review(0), review(1)), first.jobs[0]))
stalled: Final = reviewed.jobs[0].model_copy(
update={"reading": (InFlight(execution_id="run-2", trace_id="t", agent="support", started_at=NOW),)}
)
reclaimed: Final = claim_job(replace_job(reviewed, stalled), worker(identity="other"), NOW + timedelta(minutes=6))
job: Final = reclaimed.jobs[0]
assert job.worker_id == "other"
assert (job.reviews, job.reviewed, job.reading) == ((), 0, ())
replayed: Final = reduce(add_review, (review(0), review(1)), job)
assert replayed.reviewed == len(replayed.reviews) == 2
def test_progress_without_a_review_leaves_the_review_history_alone() -> None:
job: Final = add_review(queue_job(lens(), NOW, "job").jobs[0], review(0))
assert add_review(job, None) == job
def reviewed_job() -> Job:
execution: Final = Execution(
id="run-0",
source="traces",
trace_id="t",
team_id="alpha",
name="task",
start_time="2026-01-15 00:00:00",
span_count=3,
service="support",
metadata=(MetadataFilter(key="gen_ai.agent.name", value="support"),),
)
job: Final = (
queue_job(lens(), NOW, "job")
.jobs[0]
.model_copy(update={"sample": Sample(executions=(execution,), eligible=4, selected=1)})
)
timed: Final = tuple(review(i).model_copy(update={"at": NOW + timedelta(seconds=i)}) for i in range(3))
return reduce(add_review, timed, job)
def test_summary_drops_reviews_and_run_attributes_but_keeps_counts_and_run_identity() -> None:
job: Final = reviewed_job()
listed: Final = summarized(lens().model_copy(update={"jobs": (job,)})).jobs[0]
assert listed.reviews == ()
assert listed.reviewed == job.reviewed == 3
assert listed.sample is not None and job.sample is not None
assert listed.sample.executions[0].metadata == ()
assert (
listed.sample.executions[0].model_copy(update={"metadata": job.sample.executions[0].metadata})
== (job.sample.executions[0])
)
assert listed.model_copy(update={"reviews": job.reviews, "sample": job.sample}) == job
def test_review_polling_returns_only_reviews_after_the_cursor_even_when_they_finished_out_of_order() -> None:
job: Final = reduce(add_review, (review(5).model_copy(update={"at": NOW - timedelta(hours=1)}),), reviewed_job())
assert reviews_after(job, 0).reviews == job.reviews
assert [r.execution_id for r in reviews_after(job, 2).reviews] == ["run-2", "run-5"]
assert reviews_after(job, 4).reviews == ()
assert reviews_after(job, 4).reviewed == 4
def test_review_polling_after_the_window_moved_on_returns_what_is_still_kept() -> None:
job: Final = reduce(add_review, tuple(review(i) for i in range(MAX_REVIEWS + 10)), reviewed_job())
page: Final = reviews_after(job, 5)
assert page.reviews == job.reviews
assert page.reviewed == MAX_REVIEWS + 13
assert [r.execution_id for r in reviews_after(job, page.reviewed - 2).reviews] == [
f"run-{MAX_REVIEWS + 8}",
f"run-{MAX_REVIEWS + 9}",
]
def in_flight(execution: str) -> InFlight:
return InFlight(execution_id=execution, trace_id="t", agent="support", started_at=NOW)
def reading_job() -> Job:
running: Final = claim_job(queue_job(lens(), NOW, "job"), worker(), NOW).jobs[0]
return apply_progress(running, Progress(stage=running.stage, reading=(in_flight("a"), in_flight("b"))), NOW)
def test_progress_replaces_the_in_flight_runs_and_old_workers_leave_them_alone() -> None:
job: Final = reading_job()
assert [r.execution_id for r in job.reading] == ["a", "b"]
finished: Final = apply_progress(job, Progress(stage=job.stage, review=review(0), reading=(in_flight("b"),)), NOW)
assert [r.execution_id for r in finished.reading] == ["b"]
assert finished.reviewed == 1
assert apply_progress(job, Progress(stage=job.stage, review=review(1)), NOW).reading == job.reading
assert apply_progress(job, Progress(stage=job.stage, reading=()), NOW).reading == ()
@pytest.mark.parametrize("status", ("completed", "failed", "cancelled"))
def test_finished_jobs_stop_showing_runs_in_flight(status: Literal["completed", "failed", "cancelled"]) -> None:
ended: Final = end_job(reading_job(), status, NOW)
assert ended.status == status
assert ended.finished_at == NOW
assert ended.reading == ()
def test_cancel_and_repeated_disconnects_clear_runs_in_flight() -> None:
reading: Final = replace_job(queue_job(lens(), NOW, "job"), reading_job())
cancelled: Final = cancel_job(reading, NOW).jobs[0]
assert (cancelled.status, cancelled.reading) == ("cancelled", ())
abandoned: Final = reading.model_copy(update={"jobs": (reading.jobs[0].model_copy(update={"attempts": 3}),)})
expired: Final = claim_job(abandoned, worker(), NOW + timedelta(minutes=10)).jobs[0]
assert (expired.status, expired.reading) == ("failed", ())

115
tests/unit/proxy/lens/test_worker.py
View file

@ -12,12 +12,19 @@ from litellm.proxy.lens.models import (
ExecutionContent,
ModelRequest,
ModelResult,
Progress,
Result,
Sample,
TracePart,
)
from litellm.proxy.lens.state import queue_job
from litellm.proxy.lens.worker import LensWorker, failure_message
from litellm.proxy.lens.worker import (
MODEL_RETRIES,
MODEL_RETRY_MAX_SECONDS,
LensWorker,
failure_message,
retry_delay,
)
from tests.unit.proxy.lens.test_state import NOW, lens
@ -69,8 +76,44 @@ async def test_transient_retries_are_bounded() -> None:
await LensWorker(client, sleep=sleep).model_request(
"/model", ModelRequest(purpose="extract", prompt="review")
)
assert attempts.qsize() == 3
assert tuple(delays.get_nowait() for _ in range(delays.qsize())) == (1, 2)
assert attempts.qsize() == MODEL_RETRIES + 1
assert tuple(delays.get_nowait() for _ in range(delays.qsize())) == tuple(
float(min(2**n, MODEL_RETRY_MAX_SECONDS)) for n in range(MODEL_RETRIES)
)
@pytest.mark.asyncio
async def test_rate_limited_model_waits_as_long_as_the_provider_asks_then_completes() -> None:
attempts: Final = SimpleQueue[str]()
delays: Final = SimpleQueue[float]()
expected: Final = ModelResult(content='{"observations":[]}', cost=0.01)
def handle(request: httpx.Request) -> httpx.Response:
attempts.put(request.url.path)
if attempts.qsize() <= 3:
return httpx.Response(429, headers={"retry-after": "30"})
return httpx.Response(200, json=expected.model_dump())
async def sleep(delay: float) -> None:
delays.put(delay)
async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client:
result: Final = await LensWorker(client, sleep=sleep).model_request(
"/model", ModelRequest(purpose="extract", prompt="review")
)
assert result == expected
assert tuple(delays.get_nowait() for _ in range(delays.qsize())) == (30, 30, 30)
@pytest.mark.parametrize(
("retry_after", "attempt", "expected"),
(("", 1, 2), ("5", 0, 5), ("1", 3, 8), ("9999", 0, MODEL_RETRY_MAX_SECONDS), ("soon", 2, 4)),
)
def test_retry_delay_prefers_the_providers_wait_within_bounds(retry_after: str, attempt: int, expected: float) -> None:
request: Final = httpx.Request("POST", "https://proxy.test/model")
headers: Final = {"retry-after": retry_after} if retry_after else {}
error: Final = httpx.HTTPStatusError("limited", request=request, response=httpx.Response(429, headers=headers))
assert retry_delay(error, attempt) == expected
@pytest.mark.asyncio
@ -419,6 +462,72 @@ async def test_transient_heartbeat_failure_recovers_without_cancelling_analysis(
assert attempts.qsize() == 2 and saved.empty()
@pytest.mark.asyncio
async def test_worker_sends_each_runs_review_with_its_progress() -> None:
claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=())
execution: Final = Execution(
id="run", source="traces", trace_id="t", team_id="", name="task", start_time="", span_count=1
)
sent: Final = SimpleQueue[Progress]()
def handle(request: httpx.Request) -> httpx.Response:
match request.url.path.rsplit("/", 1)[-1]:
case "claim":
return httpx.Response(200, json=claim.model_dump(mode="json"))
case "sample":
return httpx.Response(200, json=Sample(executions=(execution,), eligible=1).model_dump())
case "content":
return httpx.Response(
200,
json=ExecutionContent(
execution=execution,
parts=(TracePart(execution_id="run", span_id="s", name="step", kind="agent", content="Done"),),
).model_dump(),
)
case "model":
return httpx.Response(
200, json={"content": '{"observations":[],"reasoning":"Finished the task."}', "cost": 0}
)
case "progress":
sent.put(Progress.model_validate_json(request.content))
return httpx.Response(200, json=True)
case "result":
return httpx.Response(200, json=True)
case _:
pytest.fail(f"Unexpected worker request: {request.url.path}")
async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client:
assert await LensWorker(client).run_once()
reviews: Final = tuple(p.review for p in (sent.get_nowait() for _ in range(sent.qsize())) if p.review)
assert tuple((r.execution_id, r.reasoning) for r in reviews) == (("run", "Finished the task."),)
@pytest.mark.asyncio
async def test_worker_runs_investigations_in_parallel_and_polls_quickly_when_idle() -> None:
claims: Final = SimpleQueue[str]()
running: Final = asyncio.Event()
waits: Final = SimpleQueue[float]()
class Worker(LensWorker):
async def run_once(self) -> bool:
claims.put("claim")
if claims.qsize() <= 2:
if claims.qsize() == 2:
running.set()
await running.wait()
return True
raise asyncio.CancelledError
async def sleep(delay: float) -> None:
waits.put(delay)
async with httpx.AsyncClient(base_url="https://proxy.test") as client:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(Worker(client, sleep=sleep).serve(slots=2, poll_seconds=2), timeout=1)
assert running.is_set()
assert waits.empty()
@pytest.mark.asyncio
async def test_worker_announces_release_and_waits_on_incompatible_gateway(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture

178
ui/litellm-dashboard/public/assets/logos/cerebras.svg
View file

@ -1,89 +1,89 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 800 600" style="enable-background:new 0 0 800 600;" xml:space="preserve">
<style type="text/css">
.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#F05A28;}
.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#231F20;}
</style>
<g id="Contact">
<g id="Contact-us" transform="translate(-234.000000, -1114.000000)">
<g id="map" transform="translate(-6.000000, 1027.000000)">
<g id="Contact-box" transform="translate(190.000000, 36.000000)">
<g id="Group-26" transform="translate(50.000000, 51.000000)">
<g id="Group-3">
<path id="Fill-1" class="st0" d="M220.9,421c-17,0-33.1-3.4-47.8-9.5c-22-9.2-40.8-24.6-54.1-44c-13.3-19.4-21-42.7-21-67.9
c0-16.8,3.4-32.7,9.7-47.3c9.3-21.8,24.9-40.3,44.5-53.4c19.6-13.1,43.2-20.7,68.7-20.7v-18.3c-19.5,0-38.1,3.9-55.1,11
c-25.4,10.6-47,28.3-62.2,50.6c-15.3,22.3-24.2,49.2-24.2,78.1c0,19.3,4,37.7,11.1,54.4c10.7,25.1,28.7,46.4,51.2,61.5
c22.6,15.1,49.8,23.9,79.1,23.9V421z"/>
<path id="Fill-4" class="st0" d="M157.9,374.1c-11.5-9.6-20.1-21.2-25.9-33.9c-5.8-12.7-8.8-26.4-8.8-40.2
c0-11,1.9-22,5.6-32.5c3.8-10.5,9.4-20.5,17.1-29.6c9.6-11.4,21.3-20,34-25.8c12.7-5.8,26.6-8.7,40.4-8.7
c11,0,22.1,1.9,32.6,5.6c10.6,3.8,20.6,9.4,29.7,17l11.9-14.1c-10.8-9-22.8-15.8-35.4-20.2c-12.6-4.5-25.7-6.7-38.8-6.7
c-16.5,0-32.9,3.5-48.1,10.4c-15.2,6.9-29.1,17.2-40.5,30.7c-9.1,10.8-15.8,22.7-20.3,35.2c-4.5,12.5-6.7,25.6-6.7,38.7
c0,16.4,3.5,32.8,10.4,47.9c6.9,15.1,17.3,29,30.9,40.3L157.9,374.1z"/>
<path id="Fill-6" class="st0" d="M186.4,362.2c-12.1-6.4-21.6-15.7-28.1-26.6c-6.5-10.9-9.9-23.5-9.9-36.2
c0-11.2,2.6-22.5,8.3-33c6.4-12.1,15.8-21.5,26.8-27.9c11-6.5,23.6-9.9,36.4-9.9c11.2,0,22.6,2.6,33.2,8.2l8.6-16.3
c-13.3-7-27.7-10.4-41.9-10.3c-16.1,0-32,4.3-45.8,12.4c-13.8,8.1-25.7,20.1-33.7,35.2c-7,13.3-10.4,27.6-10.4,41.6
c0,16,4.3,31.8,12.5,45.5c8.2,13.8,20.2,25.5,35.4,33.5L186.4,362.2z"/>
<path id="Fill-8" class="st0" d="M221,344.6c-6.3,0-12.3-1.3-17.7-3.6c-8.2-3.4-15.1-9.2-20-16.5c-4.9-7.3-7.8-16-7.8-25.4
c0-6.3,1.3-12.3,3.6-17.7c3.4-8.1,9.2-15.1,16.5-20c7.3-4.9,16-7.8,25.4-7.8v-18.4c-8.8,0-17.2,1.8-24.9,5
c-11.5,4.9-21.2,12.9-28.1,23.1C161,273.6,157,286,157,299.2c0,8.8,1.8,17.2,5,24.9c4.9,11.5,13,21.2,23.2,28.1
C195.4,359,207.7,363,221,363V344.6z"/>
</g>
<g id="Group" transform="translate(22.000000, 13.000000)">
<path id="Fill-10" class="st1" d="M214,271.6c-2.1-2.2-4.4-4-6.7-5.3c-2.3-1.3-4.7-2-7.2-2c-3.4,0-6.3,0.6-9,1.8
c-2.6,1.2-4.9,2.8-6.8,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.4,5.4-1.4,8.2c0,2.8,0.5,5.6,1.4,8.2c1,2.6,2.4,5,4.3,7
c1.9,2,4.1,3.7,6.8,4.9c2.6,1.2,5.6,1.8,9,1.8c2.8,0,5.5-0.6,7.9-1.7c2.4-1.2,4.5-2.9,6.2-5.1l12.2,13.1
c-1.8,1.8-3.9,3.4-6.3,4.7c-2.4,1.3-4.8,2.4-7.2,3.2s-4.8,1.4-7,1.7c-2.2,0.4-4.2,0.5-5.8,0.5c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c4.7,0,9.4,0.9,14.1,2.7
c4.7,1.8,8.9,4.6,12.4,8.4L214,271.6z"/>
<path id="Fill-12" class="st1" d="M280.4,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
c-3.2,3.1-5.2,7.4-5.9,12.6H280.4z M243,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H243z"/>
<path id="Fill-14" class="st1" d="M306.5,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
<path id="Fill-16" class="st1" d="M409,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
c-3.2,3.1-5.2,7.4-5.9,12.6H409z M371.6,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H371.6z"/>
<path id="Fill-18" class="st1" d="M494.6,286.2c0-2.8-0.5-5.6-1.5-8.2c-1-2.6-2.4-5-4.3-7c-1.9-2-4.2-3.7-6.9-4.9
c-2.7-1.2-5.7-1.8-9.1-1.8c-3.4,0-6.4,0.6-9.1,1.8c-2.7,1.2-5,2.8-6.9,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.5,5.4-1.5,8.2
c0,2.8,0.5,5.6,1.5,8.2c1,2.6,2.4,5,4.3,7c1.9,2,4.2,3.7,6.9,4.9c2.7,1.2,5.7,1.8,9.1,1.8c3.4,0,6.4-0.6,9.1-1.8
c2.7-1.2,5-2.8,6.9-4.9c1.9-2,3.3-4.4,4.3-7C494.1,291.8,494.6,289,494.6,286.2L494.6,286.2z M433.2,207.6h18.5v51.3h0.5
c0.9-1.2,2.1-2.5,3.5-3.7c1.4-1.3,3.2-2.5,5.2-3.6c2.1-1.1,4.4-2,7.1-2.7c2.7-0.7,5.8-1.1,9.3-1.1c5.2,0,10.1,1,14.5,3
c4.4,2,8.2,4.7,11.3,8.1c3.1,3.5,5.6,7.5,7.3,12.2c1.7,4.7,2.6,9.7,2.6,15.1c0,5.4-0.8,10.4-2.5,15.1
c-1.6,4.7-4.1,8.7-7.2,12.2c-3.2,3.5-7,6.2-11.6,8.1c-4.5,2-9.6,3-15.3,3c-5.2,0-10.1-1-14.7-3c-4.5-2-8.1-5.3-10.8-9.7h-0.3
v11h-17.6V207.6z"/>
<path id="Fill-20" class="st1" d="M520.9,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
<path id="Fill-22" class="st1" d="M616,290h-3.9c-2.6,0-5.5,0.1-8.7,0.3c-3.2,0.2-6.2,0.7-9.1,1.4c-2.8,0.8-5.2,1.9-7.2,3.3
c-2,1.5-2.9,3.5-2.9,6.2c0,1.7,0.4,3.2,1.2,4.3c0.8,1.2,1.8,2.2,3,3c1.2,0.8,2.6,1.4,4.2,1.8c1.5,0.4,3.1,0.5,4.6,0.5
c6.4,0,11.1-1.5,14.2-4.5c3-3,4.6-7.1,4.6-12.2V290z M617.1,312.7h-0.5c-2.7,4.2-6.1,7.2-10.2,9.1c-4.1,1.9-8.7,2.8-13.6,2.8
c-3.4,0-6.7-0.5-10-1.4s-6.1-2.3-8.7-4.1c-2.5-1.8-4.6-4.1-6.1-6.8s-2.3-5.9-2.3-9.6c0-4,0.7-7.3,2.2-10.1
c1.4-2.8,3.4-5.1,5.8-7c2.4-1.9,5.2-3.4,8.4-4.5c3.2-1.1,6.5-2,10-2.5c3.5-0.6,6.9-0.9,10.5-1.1c3.5-0.2,6.8-0.2,9.9-0.2h4.6
v-2c0-4.6-1.6-8-4.8-10.3c-3.2-2.3-7.3-3.4-12.2-3.4c-3.9,0-7.6,0.7-11,2.1c-3.4,1.4-6.4,3.2-8.8,5.6l-9.8-9.6
c4.1-4.2,9-7.1,14.5-9c5.5-1.8,11.2-2.7,17.1-2.7c5.3,0,9.7,0.6,13.3,1.7c3.6,1.2,6.6,2.7,9,4.5c2.4,1.8,4.2,3.9,5.5,6.3
c1.3,2.4,2.2,4.8,2.8,7.2c0.6,2.4,0.9,4.8,1,7.1c0.1,2.3,0.2,4.3,0.2,6v42h-16.7V312.7z"/>
<path id="Fill-24" class="st1" d="M683.6,269.9c-3.6-5-8.4-7.5-14.4-7.5c-2.5,0-4.9,0.6-7.2,1.8c-2.4,1.2-3.5,3.2-3.5,5.9
c0,2.2,1,3.9,2.9,4.9c1.9,1,4.4,1.9,7.4,2.6c3,0.7,6.2,1.4,9.6,2.2c3.4,0.8,6.6,1.9,9.6,3.5c3,1.6,5.4,3.7,7.4,6.5
c1.9,2.7,2.9,6.5,2.9,11.3c0,4.4-0.9,8-2.8,11c-1.9,3-4.3,5.4-7.4,7.2c-3,1.8-6.4,3.1-10.2,4c-3.8,0.8-7.6,1.2-11.3,1.2
c-5.7,0-11-0.8-15.8-2.4c-4.8-1.6-9.1-4.6-12.9-8.8l12.3-11.4c2.4,2.6,4.9,4.8,7.6,6.5c2.7,1.7,6,2.5,9.9,2.5
c1.3,0,2.7-0.2,4.1-0.5c1.4-0.3,2.8-0.8,4-1.5c1.2-0.7,2.2-1.6,3-2.7c0.8-1.1,1.1-2.3,1.1-3.7c0-2.5-1-4.4-2.9-5.6
c-1.9-1.2-4.4-2.2-7.4-3c-3-0.8-6.2-1.5-9.6-2.1c-3.4-0.7-6.6-1.7-9.6-3.2c-3-1.5-5.4-3.5-7.4-6.2c-1.9-2.6-2.9-6.3-2.9-11
c0-4.1,0.8-7.6,2.5-10.6c1.7-3,3.9-5.4,6.7-7.4c2.8-1.9,5.9-3.3,9.5-4.3c3.6-0.9,7.2-1.4,10.9-1.4c4.9,0,9.8,0.8,14.6,2.5
c4.8,1.7,8.7,4.5,11.7,8.6L683.6,269.9z"/>
</g>
</g>
</g>
</g>
</g>
</g>
</svg>
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="72 152 230 295" style="enable-background:new 72 152 230 295;" xml:space="preserve">
<style type="text/css">
.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#F05A28;}
.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#231F20;}
</style>
<g id="Contact">
<g id="Contact-us" transform="translate(-234.000000, -1114.000000)">
<g id="map" transform="translate(-6.000000, 1027.000000)">
<g id="Contact-box" transform="translate(190.000000, 36.000000)">
<g id="Group-26" transform="translate(50.000000, 51.000000)">
<g id="Group-3">
<path id="Fill-1" class="st0" d="M220.9,421c-17,0-33.1-3.4-47.8-9.5c-22-9.2-40.8-24.6-54.1-44c-13.3-19.4-21-42.7-21-67.9
c0-16.8,3.4-32.7,9.7-47.3c9.3-21.8,24.9-40.3,44.5-53.4c19.6-13.1,43.2-20.7,68.7-20.7v-18.3c-19.5,0-38.1,3.9-55.1,11
c-25.4,10.6-47,28.3-62.2,50.6c-15.3,22.3-24.2,49.2-24.2,78.1c0,19.3,4,37.7,11.1,54.4c10.7,25.1,28.7,46.4,51.2,61.5
c22.6,15.1,49.8,23.9,79.1,23.9V421z"/>
<path id="Fill-4" class="st0" d="M157.9,374.1c-11.5-9.6-20.1-21.2-25.9-33.9c-5.8-12.7-8.8-26.4-8.8-40.2
c0-11,1.9-22,5.6-32.5c3.8-10.5,9.4-20.5,17.1-29.6c9.6-11.4,21.3-20,34-25.8c12.7-5.8,26.6-8.7,40.4-8.7
c11,0,22.1,1.9,32.6,5.6c10.6,3.8,20.6,9.4,29.7,17l11.9-14.1c-10.8-9-22.8-15.8-35.4-20.2c-12.6-4.5-25.7-6.7-38.8-6.7
c-16.5,0-32.9,3.5-48.1,10.4c-15.2,6.9-29.1,17.2-40.5,30.7c-9.1,10.8-15.8,22.7-20.3,35.2c-4.5,12.5-6.7,25.6-6.7,38.7
c0,16.4,3.5,32.8,10.4,47.9c6.9,15.1,17.3,29,30.9,40.3L157.9,374.1z"/>
<path id="Fill-6" class="st0" d="M186.4,362.2c-12.1-6.4-21.6-15.7-28.1-26.6c-6.5-10.9-9.9-23.5-9.9-36.2
c0-11.2,2.6-22.5,8.3-33c6.4-12.1,15.8-21.5,26.8-27.9c11-6.5,23.6-9.9,36.4-9.9c11.2,0,22.6,2.6,33.2,8.2l8.6-16.3
c-13.3-7-27.7-10.4-41.9-10.3c-16.1,0-32,4.3-45.8,12.4c-13.8,8.1-25.7,20.1-33.7,35.2c-7,13.3-10.4,27.6-10.4,41.6
c0,16,4.3,31.8,12.5,45.5c8.2,13.8,20.2,25.5,35.4,33.5L186.4,362.2z"/>
<path id="Fill-8" class="st0" d="M221,344.6c-6.3,0-12.3-1.3-17.7-3.6c-8.2-3.4-15.1-9.2-20-16.5c-4.9-7.3-7.8-16-7.8-25.4
c0-6.3,1.3-12.3,3.6-17.7c3.4-8.1,9.2-15.1,16.5-20c7.3-4.9,16-7.8,25.4-7.8v-18.4c-8.8,0-17.2,1.8-24.9,5
c-11.5,4.9-21.2,12.9-28.1,23.1C161,273.6,157,286,157,299.2c0,8.8,1.8,17.2,5,24.9c4.9,11.5,13,21.2,23.2,28.1
C195.4,359,207.7,363,221,363V344.6z"/>
</g>
<g id="Group" transform="translate(22.000000, 13.000000)">
<path id="Fill-10" class="st1" d="M214,271.6c-2.1-2.2-4.4-4-6.7-5.3c-2.3-1.3-4.7-2-7.2-2c-3.4,0-6.3,0.6-9,1.8
c-2.6,1.2-4.9,2.8-6.8,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.4,5.4-1.4,8.2c0,2.8,0.5,5.6,1.4,8.2c1,2.6,2.4,5,4.3,7
c1.9,2,4.1,3.7,6.8,4.9c2.6,1.2,5.6,1.8,9,1.8c2.8,0,5.5-0.6,7.9-1.7c2.4-1.2,4.5-2.9,6.2-5.1l12.2,13.1
c-1.8,1.8-3.9,3.4-6.3,4.7c-2.4,1.3-4.8,2.4-7.2,3.2s-4.8,1.4-7,1.7c-2.2,0.4-4.2,0.5-5.8,0.5c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c4.7,0,9.4,0.9,14.1,2.7
c4.7,1.8,8.9,4.6,12.4,8.4L214,271.6z"/>
<path id="Fill-12" class="st1" d="M280.4,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
c-3.2,3.1-5.2,7.4-5.9,12.6H280.4z M243,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H243z"/>
<path id="Fill-14" class="st1" d="M306.5,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
<path id="Fill-16" class="st1" d="M409,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
c-3.2,3.1-5.2,7.4-5.9,12.6H409z M371.6,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H371.6z"/>
<path id="Fill-18" class="st1" d="M494.6,286.2c0-2.8-0.5-5.6-1.5-8.2c-1-2.6-2.4-5-4.3-7c-1.9-2-4.2-3.7-6.9-4.9
c-2.7-1.2-5.7-1.8-9.1-1.8c-3.4,0-6.4,0.6-9.1,1.8c-2.7,1.2-5,2.8-6.9,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.5,5.4-1.5,8.2
c0,2.8,0.5,5.6,1.5,8.2c1,2.6,2.4,5,4.3,7c1.9,2,4.2,3.7,6.9,4.9c2.7,1.2,5.7,1.8,9.1,1.8c3.4,0,6.4-0.6,9.1-1.8
c2.7-1.2,5-2.8,6.9-4.9c1.9-2,3.3-4.4,4.3-7C494.1,291.8,494.6,289,494.6,286.2L494.6,286.2z M433.2,207.6h18.5v51.3h0.5
c0.9-1.2,2.1-2.5,3.5-3.7c1.4-1.3,3.2-2.5,5.2-3.6c2.1-1.1,4.4-2,7.1-2.7c2.7-0.7,5.8-1.1,9.3-1.1c5.2,0,10.1,1,14.5,3
c4.4,2,8.2,4.7,11.3,8.1c3.1,3.5,5.6,7.5,7.3,12.2c1.7,4.7,2.6,9.7,2.6,15.1c0,5.4-0.8,10.4-2.5,15.1
c-1.6,4.7-4.1,8.7-7.2,12.2c-3.2,3.5-7,6.2-11.6,8.1c-4.5,2-9.6,3-15.3,3c-5.2,0-10.1-1-14.7-3c-4.5-2-8.1-5.3-10.8-9.7h-0.3
v11h-17.6V207.6z"/>
<path id="Fill-20" class="st1" d="M520.9,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
<path id="Fill-22" class="st1" d="M616,290h-3.9c-2.6,0-5.5,0.1-8.7,0.3c-3.2,0.2-6.2,0.7-9.1,1.4c-2.8,0.8-5.2,1.9-7.2,3.3
c-2,1.5-2.9,3.5-2.9,6.2c0,1.7,0.4,3.2,1.2,4.3c0.8,1.2,1.8,2.2,3,3c1.2,0.8,2.6,1.4,4.2,1.8c1.5,0.4,3.1,0.5,4.6,0.5
c6.4,0,11.1-1.5,14.2-4.5c3-3,4.6-7.1,4.6-12.2V290z M617.1,312.7h-0.5c-2.7,4.2-6.1,7.2-10.2,9.1c-4.1,1.9-8.7,2.8-13.6,2.8
c-3.4,0-6.7-0.5-10-1.4s-6.1-2.3-8.7-4.1c-2.5-1.8-4.6-4.1-6.1-6.8s-2.3-5.9-2.3-9.6c0-4,0.7-7.3,2.2-10.1
c1.4-2.8,3.4-5.1,5.8-7c2.4-1.9,5.2-3.4,8.4-4.5c3.2-1.1,6.5-2,10-2.5c3.5-0.6,6.9-0.9,10.5-1.1c3.5-0.2,6.8-0.2,9.9-0.2h4.6
v-2c0-4.6-1.6-8-4.8-10.3c-3.2-2.3-7.3-3.4-12.2-3.4c-3.9,0-7.6,0.7-11,2.1c-3.4,1.4-6.4,3.2-8.8,5.6l-9.8-9.6
c4.1-4.2,9-7.1,14.5-9c5.5-1.8,11.2-2.7,17.1-2.7c5.3,0,9.7,0.6,13.3,1.7c3.6,1.2,6.6,2.7,9,4.5c2.4,1.8,4.2,3.9,5.5,6.3
c1.3,2.4,2.2,4.8,2.8,7.2c0.6,2.4,0.9,4.8,1,7.1c0.1,2.3,0.2,4.3,0.2,6v42h-16.7V312.7z"/>
<path id="Fill-24" class="st1" d="M683.6,269.9c-3.6-5-8.4-7.5-14.4-7.5c-2.5,0-4.9,0.6-7.2,1.8c-2.4,1.2-3.5,3.2-3.5,5.9
c0,2.2,1,3.9,2.9,4.9c1.9,1,4.4,1.9,7.4,2.6c3,0.7,6.2,1.4,9.6,2.2c3.4,0.8,6.6,1.9,9.6,3.5c3,1.6,5.4,3.7,7.4,6.5
c1.9,2.7,2.9,6.5,2.9,11.3c0,4.4-0.9,8-2.8,11c-1.9,3-4.3,5.4-7.4,7.2c-3,1.8-6.4,3.1-10.2,4c-3.8,0.8-7.6,1.2-11.3,1.2
c-5.7,0-11-0.8-15.8-2.4c-4.8-1.6-9.1-4.6-12.9-8.8l12.3-11.4c2.4,2.6,4.9,4.8,7.6,6.5c2.7,1.7,6,2.5,9.9,2.5
c1.3,0,2.7-0.2,4.1-0.5c1.4-0.3,2.8-0.8,4-1.5c1.2-0.7,2.2-1.6,3-2.7c0.8-1.1,1.1-2.3,1.1-3.7c0-2.5-1-4.4-2.9-5.6
c-1.9-1.2-4.4-2.2-7.4-3c-3-0.8-6.2-1.5-9.6-2.1c-3.4-0.7-6.6-1.7-9.6-3.2c-3-1.5-5.4-3.5-7.4-6.2c-1.9-2.6-2.9-6.3-2.9-11
c0-4.1,0.8-7.6,2.5-10.6c1.7-3,3.9-5.4,6.7-7.4c2.8-1.9,5.9-3.3,9.5-4.3c3.6-0.9,7.2-1.4,10.9-1.4c4.9,0,9.8,0.8,14.6,2.5
c4.8,1.7,8.7,4.5,11.7,8.6L683.6,269.9z"/>
</g>
</g>
</g>
</g>
</g>
</g>
</svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 8 KiB

After

Width:  |  Height:  |  Size: 8 KiB

10
ui/litellm-dashboard/src/app/globals.css
View file

@ -67,6 +67,16 @@
--animate-slot-slide-in: slot-slide-in 150ms cubic-bezier(0, 0, 0.2, 1) both;
--animate-trace-drawer-in: trace-drawer-in 200ms cubic-bezier(0.25, 1, 0.5, 1) both;
--animate-trace-drawer-out: trace-drawer-out 200ms cubic-bezier(0.4, 0, 1, 1) both;
--animate-lens-shimmer: lens-shimmer 1.1s cubic-bezier(0.4, 0, 0.2, 1) infinite;
@keyframes lens-shimmer {
from {
transform: translateX(-100%);
}
to {
transform: translateX(250%);
}
}
@keyframes trace-drawer-in {
from {

116
ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx
View file

@ -1,103 +1,63 @@
"use client";
import { Tabs as TabsPrimitive } from "@base-ui/react/tabs";
import { Activity, ScanSearch, Settings } from "lucide-react";
import { Settings } from "lucide-react";
import { StatusDot } from "@/components/shared/StatusDot";
import { cn } from "@/lib/cva.config";
import type { InvestigationActivity } from "./model/status";
import { useWorkerConnected } from "./hooks/useWorkerConnected";
import type { LensList } from "./model/types";
import { LENS_TABS, type LensTab } from "./route";
import { frameCorner, frameTab } from "./ui/frame";
import { LENS_TABS } from "./route";
const MODE_ICONS = { traces: Activity, investigations: ScanSearch, settings: Settings } as const;
const ACTIVITY_DOT: Record<Exclude<InvestigationActivity, "idle">, { className: string; label: string }> = {
running: { className: "bg-info motion-safe:animate-pulse", label: "An investigation is running" },
queued: { className: "bg-muted-foreground/60", label: "An investigation is queued" },
};
function ActivityDot({ activity }: { activity: InvestigationActivity }) {
if (activity === "idle") return null;
return (
<span
aria-hidden="true"
className={cn(
"size-1.5 rounded-full animate-in fade-in-0 zoom-in-50 duration-300",
ACTIVITY_DOT[activity].className,
)}
/>
);
}
function NotchCorner({ side, demo }: { side: "left" | "right"; demo: boolean }) {
return (
<span
aria-hidden="true"
className={cn(
"pointer-events-none absolute bottom-0 hidden size-3 overflow-hidden @min-[24rem]/lens-frame:block",
side === "left" ? "-left-3" : "-right-3",
)}
>
<span className={frameCorner({ session: demo ? "demo" : "live", side })} />
</span>
);
}
const ACTIVITY_LABEL = { running: "An investigation is running", queued: "An investigation is queued" };
export function LensModeSwitch({
activity,
demo,
workers,
setup,
}: {
activity: InvestigationActivity;
demo: boolean;
workers: LensList["workers"] | null;
setup?: string;
}) {
const connected = useWorkerConnected(workers);
const settingsTitle = connected ? "Worker connected" : "Connect worker";
const tabs = Object.entries(LENS_TABS).filter(([view]) => view !== "settings" || workers);
return (
<div className={frameTab({ session: demo ? "demo" : "live" })}>
<NotchCorner side="left" demo={demo} />
<NotchCorner side="right" demo={demo} />
<TabsPrimitive.List
aria-label="Lens"
className="relative flex h-9 items-center justify-center p-1 @min-[24rem]/lens-frame:justify-start"
>
<TabsPrimitive.Indicator className="absolute top-1 bottom-1 left-(--active-tab-left) w-(--active-tab-width) rounded-full bg-muted transition-[left,width] duration-300 ease-[cubic-bezier(0.32,0.72,0,1)] motion-reduce:transition-none" />
{tabs.map(([view, label]) => {
const Icon = MODE_ICONS[view as LensTab];
const workerDisconnected = view === "settings" && workers !== null && !connected;
return (
<TabsPrimitive.Tab
key={view}
value={view}
title={view === "settings" ? settingsTitle : undefined}
aria-description={
view === "investigations" && activity !== "idle" ? ACTIVITY_DOT[activity].label : undefined
}
<TabsPrimitive.List
aria-label="Lens"
className="relative inline-flex h-8 max-w-full items-center gap-0.5 rounded-lg bg-muted/60 p-0.5"
>
<TabsPrimitive.Indicator className="absolute top-0.5 bottom-0.5 left-(--active-tab-left) w-(--active-tab-width) rounded-md bg-card shadow-sm transition-[left,width] duration-200 motion-reduce:transition-none" />
{tabs.map(([view, label]) => (
<TabsPrimitive.Tab
key={view}
value={view}
title={view === "settings" ? settingsTitle : undefined}
aria-description={view === "investigations" && activity !== "idle" ? ACTIVITY_LABEL[activity] : undefined}
className={cn(
"relative z-raised inline-flex h-full items-center gap-1.5 rounded-md text-xs font-medium text-muted-foreground outline-none hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring/50 data-active:text-foreground sm:text-sm",
view === "settings" ? "px-2" : "px-2.5 sm:px-3",
)}
>
{view === "settings" ? (
<span className="relative inline-flex">
<Settings aria-hidden="true" className="size-3.5" />
{!connected && <StatusDot state="error" className="absolute -top-0.5 -right-0.5 size-1.5" />}
<span className="sr-only">{label}</span>
</span>
) : (
label
)}
{view === "investigations" && activity !== "idle" && (
<span
aria-hidden="true"
className={cn(
"relative z-raised inline-flex h-full items-center gap-2 rounded-full text-sm font-medium text-muted-foreground outline-none transition-colors duration-200 hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring/50 data-active:text-foreground",
view === "settings" ? "px-2.5" : "px-3 sm:px-4",
"size-1.5 rounded-full",
activity === "running" ? "bg-info motion-safe:animate-pulse" : "bg-muted-foreground/60",
)}
>
<span className="relative inline-flex">
<Icon aria-hidden="true" className="size-4" />
{workerDisconnected && <StatusDot state="error" className="absolute -top-0.5 -right-0.5 size-1.5" />}
</span>
<span className={cn(view === "settings" && "sr-only")}>{label}</span>
{view === "investigations" && setup && (
<span className="absolute -top-3 right-2 rounded-full border border-border bg-card px-1.5 py-0.5 text-xs leading-none font-medium text-muted-foreground animate-in fade-in-0 duration-200">
{setup}
</span>
)}
{view === "investigations" && <ActivityDot activity={activity} />}
</TabsPrimitive.Tab>
);
})}
</TabsPrimitive.List>
</div>
/>
)}
</TabsPrimitive.Tab>
))}
</TabsPrimitive.List>
);
}

7
ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx
View file

@ -21,7 +21,7 @@ const worker = () => ({
function serve({ enabled = false, traces = false, requests = false, connected = false } = {}) {
list.mockResolvedValue({ lenses: [], workers: connected ? [worker()] : [], tracing_enabled: enabled });
network.mockImplementation(async (input, init) => {
const { path, method, body } = await readRequest(input, init);
const { path, method, body, query } = await readRequest(input, init);
if (path === "/v1/traces")
return enabled
? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] })
@ -44,6 +44,11 @@ function serve({ enabled = false, traces = false, requests = false, connected =
if (path === "/key/info") return Response.json({ info: { models: ["analysis"], max_budget: 100 } });
if (path === "/lens/agents") return Response.json(["support_agent"]);
if (path === "/lens/preview/sample") return Response.json({ eligible: 1, selected: 1, executions: [] });
if (path.endsWith("/reviews"))
return Response.json({
reviews: data.lenses[0].jobs[0].reviews.slice(Number(query.get("after") ?? 0)),
reviewed: data.lenses[0].jobs[0].reviewed,
});
if (path.endsWith("/runs")) return Response.json(data.lenses[0].jobs);
return Response.json({ data: [] });
});

52
ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx
View file

@ -41,7 +41,7 @@ describe("Lens interactive demo", () => {
expect(await screen.findByText("Where is order #1042?")).toBeVisible();
expect(screen.getByRole("switch", { name: "Demo data" })).toBeChecked();
await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true"));
expect(screen.queryByRole("button", { name: "Set up tracing" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "Set up tracing" })).toBeDisabled();
network.mockClear();
expect(screen.getByText("Where is order #1042?")).toBeVisible();
const search = screen.getByRole("combobox", { name: "Search runs" });
@ -133,7 +133,7 @@ describe("Lens interactive demo", () => {
const onUrlUpdate = vi.fn();
window.localStorage.clear();
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
searchParams: "?tab=investigations",
searchParams: "?tab=findings",
onUrlUpdate,
});
await user.click(await screen.findByRole("switch", { name: "Demo data" }));
@ -157,7 +157,7 @@ describe("Lens interactive demo", () => {
await user.click(within(finding).getByRole("button", { name: "Back to finding" }));
expect(within(finding).getByText(/The support agent retries/)).toBeVisible();
await user.click(within(finding).getByRole("button", { name: "Close finding (Esc)" }));
expect(await screen.findByRole("table", { name: "Investigations" })).toBeVisible();
expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible();
expect(network).not.toHaveBeenCalled();
await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true"));
await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false));
@ -174,6 +174,7 @@ describe("Lens interactive demo", () => {
network.mockImplementation(async (input) => {
const path = requestPath(input);
if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true });
if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed });
if (path.endsWith("/runs")) return Response.json(saved.jobs);
if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) });
return Response.json({ data: [], traces: true, requests: false });
@ -186,7 +187,7 @@ describe("Lens interactive demo", () => {
await user.click(within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Traces" }));
expect(await screen.findByText("Where is order #1042?")).toBeVisible();
expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Set up tracing" })).not.toBeInTheDocument();
expect(screen.getByRole("button", { name: "Set up tracing" })).toBeEnabled();
});
it("uses one demo switch across setup, existing investigations, and demo traces", async () => {
@ -195,6 +196,7 @@ describe("Lens interactive demo", () => {
network.mockImplementation(async (input) => {
const path = requestPath(input);
if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: false });
if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed });
if (path.endsWith("/runs")) return Response.json(saved.jobs);
if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 });
return Response.json({ data: [], traces: false, requests: false });
@ -234,12 +236,13 @@ describe("Lens interactive demo", () => {
await waitFor(() => expect(tab).toHaveAccessibleDescription(""));
});
it("marks the Investigations tab while the inline editor is open and clears it on back", async () => {
it("keeps the Investigations tab selected while editing and returns to its list", async () => {
const user = userEvent.setup();
const saved = createLensDemoData().lenses[0];
network.mockImplementation(async (input) => {
const path = requestPath(input);
if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true });
if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed });
if (path.endsWith("/runs")) return Response.json(saved.jobs);
if (path === "/lens/agents") return Response.json([]);
if (path.startsWith("/lens/preview")) return Response.json({ eligible: 0, selected: 0, executions: [] });
@ -253,10 +256,10 @@ describe("Lens interactive demo", () => {
expect(await screen.findByRole("region", { name: "New investigation" })).toBeVisible();
const tab = tabs.getByRole("tab", { name: /^Investigations/ });
expect(tab).toHaveAttribute("aria-selected", "true");
expect(within(tab).getByText("New")).toBeVisible();
expect(await screen.findByRole("region", { name: "New investigation" })).toBeVisible();
await user.click(screen.getByRole("button", { name: "Back to investigations" }));
expect(await screen.findByRole("row", { name: new RegExp(saved.settings.name) })).toBeVisible();
expect(within(tab).queryByText("New")).not.toBeInTheDocument();
expect(await screen.findByRole("table", { name: "Investigations" })).toBeVisible();
});
it("adds a quiet Settings tab that manages the worker inline and reflects its health", async () => {
@ -275,6 +278,7 @@ describe("Lens interactive demo", () => {
network.mockImplementation(async (input) => {
const path = requestPath(input);
if (path === "/lens") return Response.json({ lenses: [saved], workers: workers(), tracing_enabled: true });
if (path.endsWith("/reviews")) return Response.json({ reviews: [], reviewed: saved.jobs[0].reviewed });
if (path.endsWith("/runs")) return Response.json(saved.jobs);
if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 });
return Response.json({ data: [], traces: true, requests: false });
@ -461,3 +465,37 @@ describe("Lens interactive demo", () => {
}
});
});
it("keeps demo row actions visible and opens reviewed traces without touching live data", async () => {
const user = userEvent.setup();
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
searchParams: "?tab=investigations&demo=true",
});
const row = await screen.findByRole("row", { name: "Support quality" });
expect(within(row).getByRole("button", { name: "Run Support quality now" })).toBeDisabled();
expect(within(row).getByRole("button", { name: "Edit Support quality" })).toBeDisabled();
await user.click(row);
await user.click(await screen.findByRole("button", { name: "View run" }));
const reviews = await screen.findByRole("list", { name: "Reviewed traces" });
expect(within(reviews).getAllByRole("button").length).toBeGreaterThan(0);
expect(screen.getByRole("region", { name: "Conclusions so far" })).toHaveTextContent("Repeated lookups");
expect(network).not.toHaveBeenCalled();
});
it("keeps trace quick filters in links and clears them when leaving demo data", async () => {
const user = userEvent.setup();
const onUrlUpdate = vi.fn();
renderWithProviders(<LensWorkspace accessToken="live-token" userRole="Admin" readOnly={false} />, {
searchParams: "?tab=traces&demo=true&agent=support_agent&status=error",
onUrlUpdate,
});
const table = await screen.findByRole("table", { name: "Agent runs" });
await waitFor(() => expect(within(table).getAllByRole("row")).toHaveLength(4));
await user.click(screen.getByRole("combobox", { name: "Filter traces by status" }));
await user.click(await screen.findByRole("option", { name: "No errors" }));
await expectUrl(onUrlUpdate, (url) => expect(url.get("status")).toBe("ok"));
expect(await within(table).findByText("Can I return my headphones?")).toBeVisible();
expect(within(table).queryByText("Where is order #1042?")).not.toBeInTheDocument();
await user.click(screen.getByRole("switch", { name: "Demo data" }));
await expectUrl(onUrlUpdate, (url) => expect([...url.keys()]).toEqual(["tab"]));
});

60
ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx
View file

@ -15,7 +15,7 @@ import { LensSettings } from "./settings/LensSettings";
import { createLensDemo } from "./data/demo/createLensDemo";
import { lensQueries } from "./data/queries";
import { LensModeSwitch } from "./LensModeSwitch";
import { frameCard } from "./ui/frame";
import { FindingsView } from "./investigations/FindingsView";
import { investigationActivity, listPollInterval } from "./model/status";
import { cn } from "@/lib/cva.config";
import { useDialogRoute, useIssueRoute, useLensRoute, type LensDialog, type LensTab } from "./route";
@ -64,9 +64,6 @@ function DemoToggle({ demo, onChange }: { demo: boolean; onChange: (demo: boolea
);
}
/** The inline investigation editor marks the tab so the notch says where you are, not just which tab is open. */
const SETUP_LABELS: Partial<Record<LensDialog, string>> = { new: "New", edit: "Editing", duplicate: "Duplicate" };
/** The one always-mounted `/lens` observer; every other reader is a plain cache subscriber. */
function useLensOverview(enabled: boolean, settingsOpen: boolean) {
const api = useLensApi();
@ -156,38 +153,28 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
}}
className="@container/lens-frame min-h-0 flex-1 gap-0"
>
<div className="grid grid-cols-[minmax(0,1fr)_auto] items-start gap-x-4 gap-y-1">
<div className="contents">
<h1 className="flex items-center gap-2 text-lg font-semibold tracking-tight">
<Aperture aria-hidden="true" className="size-5" strokeWidth={2} />
Lens
</h1>
<p className="col-span-2 row-start-2 text-xs leading-5 text-muted-foreground">
Trace your agents and investigate what goes wrong.{" "}
<a
href="https://docs.litellm.ai/docs/proxy/lens"
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center gap-0.5 font-medium text-foreground/80 underline-offset-2 hover:text-foreground hover:underline"
>
Docs
<ArrowUpRight aria-hidden="true" className="size-3" />
</a>
</p>
<header className="grid shrink-0 grid-cols-[1fr_auto] items-center gap-x-4 gap-y-2 border-b pb-2 @min-[42rem]/lens-frame:grid-cols-[auto_1fr_auto]">
<h1 className="flex items-center gap-1.5 text-sm font-semibold">
<Aperture aria-hidden="true" className="size-4" strokeWidth={2} />
Lens
</h1>
<div className="col-span-2 row-start-2 min-w-0 @min-[42rem]/lens-frame:col-span-1 @min-[42rem]/lens-frame:col-start-2 @min-[42rem]/lens-frame:row-start-1">
<LensModeSwitch activity={activity} workers={workers} />
</div>
<div className="col-span-2 row-start-3 mt-2 @min-[24rem]/lens-frame:ml-8 @min-[24rem]/lens-frame:justify-self-start">
<LensModeSwitch
activity={activity}
demo={demo}
workers={workers}
setup={activeTab === "investigations" && dialog ? SETUP_LABELS[dialog] : undefined}
/>
</div>
<div className="col-start-2 row-start-1 flex items-center justify-end pt-1">
<div className="col-start-2 row-start-1 flex items-center justify-end gap-4 @min-[42rem]/lens-frame:col-start-3">
<a
href="https://docs.litellm.ai/docs/proxy/lens"
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center gap-0.5 text-xs text-muted-foreground hover:text-foreground"
>
Docs
<ArrowUpRight aria-hidden="true" className="size-3" />
</a>
<DemoToggle demo={demo} onChange={toggleDemo} />
</div>
</div>
<div className={frameCard({ session: demo ? "demo" : "live" })}>
</header>
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-card">
{showSetup ? (
<TabsContent value={activeTab} keepMounted className={cn(PANEL, "p-3 sm:p-5")}>
{setupState.loading ? (
@ -209,6 +196,13 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
canMintTracingKey={isAdmin}
/>
</TabsContent>
<TabsContent value="findings" className={PANEL}>
{canViewInvestigations ? (
<FindingsView readOnly={readOnly || !isAdmin} />
) : (
<p className="py-6 text-sm text-muted-foreground">Findings require proxy administrator access.</p>
)}
</TabsContent>
<TabsContent value="investigations" className={PANEL}>
{canViewInvestigations ? (
<InvestigationsView readOnly={readOnly || !isAdmin} />

12
ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.test.ts
View file

@ -70,3 +70,15 @@ describe("Lens demo data", () => {
network.mockRestore();
});
});
it("serves review pages locally with the same cursor contract as live runs", async () => {
const api = createLensDemo().lens;
const { lenses } = await api.lenses();
const owner = lenses[0];
const job = owner.jobs[0];
const first = await api.reviews(owner.id, job.id, 0);
expect(first.reviews).toHaveLength(job.reviewed);
expect(first.reviews.some((review) => review.verdicts.some((verdict) => verdict.kind === "issue"))).toBe(true);
expect(await api.reviews(owner.id, job.id, first.reviewed)).toEqual({ reviews: [], reviewed: first.reviewed });
await expect(api.reviews(owner.id, "missing", 0)).rejects.toMatchObject({ status: 404 });
});

7
ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts
View file

@ -18,6 +18,13 @@ function demoLensApi(data: LensDemoData): LensApi {
activity: async () => ({ traces: true, requests: false }),
runs: (lensId, offset) => found(jobs(lensId)?.slice(offset)),
run: (lensId, jobId) => found(jobs(lensId)?.find((job) => job.id === jobId)),
reviews: async (lensId, jobId, after) => {
const job = await found(jobs(lensId)?.find((item) => item.id === jobId));
return {
reviews: job.reviews.slice(Math.max(0, after - (job.reviewed - job.reviews.length))),
reviewed: job.reviewed,
};
},
execution: notInDemo,
sample: notInDemo,
agents: notInDemo,

19
ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts
View file

@ -291,6 +291,25 @@ export function createLensDemoData(now = Date.now()) {
})),
attempts: 1,
steps: [],
reviews: sample.map((execution, index) => ({
execution_id: execution.id,
trace_id: execution.trace_id,
name: execution.name,
agent: definition.agent,
model: settings.model,
at: iso(now - 320_000 - day * 60_000 + index * 1000),
duration_ms: 1200,
cannot_assess: false,
reasoning:
snapshot.find((finding) => finding.occurrences.includes(execution.id))?.description ??
"The recorded response is consistent with the available information and follows the review criteria.",
spans: [],
verdicts: snapshot
.filter((finding) => finding.occurrences.includes(execution.id))
.map((finding) => ({ check_id: finding.check_id, kind: finding.kind, summary: finding.title })),
})),
reviewed: sample.length,
reading: [],
trigger: "schedule" as const,
error: "",
cost: sample.length * 0.012,

21
ui/litellm-dashboard/src/components/lens/data/queries.ts
View file

@ -6,6 +6,7 @@ import type { KeyPage, LensApi } from "./service";
export type { Key } from "./service";
type Activity = Awaited<ReturnType<LensApi["activity"]>>;
const LIVE_REVIEW_POLL_MS = 1500;
export const lensKeys = {
all: ["lens"] as const,
@ -17,6 +18,8 @@ export const lensKeys = {
runs: () => [...lensKeys.all, "run"] as const,
run: (scope: string, lensId: string | undefined, batchId: string) =>
[...lensKeys.runs(), { scope, lensId, batchId }] as const,
reviews: (scope: string, page: { lensId: string; jobId: string; attempt: number; after: number }) =>
[...lensKeys.all, "reviews", { scope, ...page }] as const,
evidence: (scope: string, lensId: string | undefined, evidenceId: string | undefined, offset: number) =>
[...lensKeys.all, "evidence", { scope, lensId, evidenceId, offset }] as const,
models: (scope: string) => [...lensKeys.all, "models", { scope }] as const,
@ -71,6 +74,24 @@ export const lensQueries = {
};
return queryOptions(options);
},
reviews(
api: LensApi,
{
lensId,
jobId,
attempt,
after,
live,
}: { lensId: string; jobId: string; attempt: number; after: number; live: boolean },
) {
const cursor = { lensId, jobId, attempt, after };
const options = {
queryKey: lensKeys.reviews(api.scope, cursor),
queryFn: () => api.reviews(lensId, jobId, after),
refetchInterval: live ? LIVE_REVIEW_POLL_MS : (false as const),
};
return queryOptions(options);
},
evidence(
api: LensApi,
{

10
ui/litellm-dashboard/src/components/lens/data/service.ts
View file

@ -35,6 +35,8 @@ export type Key = z.infer<typeof keySchema>;
export type KeyPage = z.infer<typeof keyPageSchema>;
export type KeyInfo = z.infer<typeof keyInfoSchema>["info"];
export type ReviewPage = components["schemas"]["ReviewPage"];
export interface AnalysisKeyRequest {
readonly model: string;
readonly budget: number;
@ -47,6 +49,7 @@ export interface LensApi {
activity(): Promise<components["schemas"]["ActivityAvailability"]>;
runs(lensId: string, offset: number): Promise<Job[]>;
run(lensId: string, jobId: string): Promise<Job>;
reviews(lensId: string, jobId: string, after: number): Promise<ReviewPage>;
execution(lensId: string, executionId: string, offset: number): Promise<ExecutionContent>;
sample(selection: ActivitySelection, offset: number, asOf: string): Promise<Sample>;
agents(): Promise<string[]>;
@ -97,6 +100,13 @@ export function liveLensApi(client: LensClient, apiClient: ApiClient, accessToke
params: { path: { lens_id: lensId, job_id: jobId } },
}),
),
reviews: (lensId, jobId, after) =>
required(
client.GET("/lens/{lens_id}/runs/{job_id}/reviews", {
headers,
params: { path: { lens_id: lensId, job_id: jobId }, query: { after } },
}),
),
execution: (lensId, executionId, offset) =>
required(
client.GET("/lens/{lens_id}/executions/{execution_id}", {

117
ui/litellm-dashboard/src/components/lens/investigations/FindingsView.integration.test.tsx Normal file
View file

@ -0,0 +1,117 @@
import { fireEvent, screen, waitFor, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { beforeEach, expect, it, vi } from "vitest";
import { renderWithLens, stubGateway } from "@/../tests/lens-test-utils";
import { testQueryClient } from "@/../tests/test-utils";
import { createLensDemoData } from "../data/demo/fixtures";
import { findingKey, inboxRows } from "../model/inbox";
import type { Lens } from "../model/types";
import { FindingsView } from "./FindingsView";
let proxy = stubGateway();
const data = createLensDemoData();
const support = data.lenses[0];
const twin: Lens = { ...support, id: "twin", settings: { ...support.settings, name: "Second support review" } };
const lenses = [support, twin, data.lenses[1]];
const issue = support.findings.find((finding) => finding.kind === "issue")!;
beforeEach(() => {
testQueryClient.clear();
proxy = stubGateway();
proxy.get.mockImplementation(async (path) =>
path === "/lens" ? { lenses, workers: [], tracing_enabled: true } : { data: [] },
);
});
it("deduplicates findings across investigations and applies feedback to every source", async () => {
const user = userEvent.setup();
const onUrlUpdate = vi.fn();
renderWithLens(<FindingsView />, { searchParams: "?tab=findings", onUrlUpdate });
const rows = await screen.findAllByRole("row", { name: issue.title });
expect(rows).toHaveLength(1);
expect(within(rows[0]).getByRole("cell", { name: "2", exact: true })).toBeVisible();
await user.click(rows[0]);
const panel = await screen.findByRole("complementary", { name: "Finding details" });
fireEvent.change(within(panel).getByRole("textbox"), { target: { value: "A handoff now handles failures" } });
await user.click(within(panel).getByRole("button", { name: "Mark resolved" }));
await waitFor(() => expect(proxy.patch).toHaveBeenCalledTimes(2));
expect(proxy.patch.mock.calls.map(([path, request]) => [path, request.body])).toEqual([
[`/lens/support/findings/${issue.id}`, { status: "resolved", reason: "A handoff now handles failures" }],
[`/lens/twin/findings/${issue.id}`, { status: "resolved", reason: "A handoff now handles failures" }],
]);
await waitFor(() => expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).has("issue")).toBe(false));
});
it("restores inbox filters from a link and keeps filter changes in the URL", async () => {
const user = userEvent.setup();
const onUrlUpdate = vi.fn();
renderWithLens(<FindingsView />, {
searchParams: "?tab=findings&inbox_agent=support_agent&priority=high",
onUrlUpdate,
});
expect(await screen.findByRole("row", { name: issue.title })).toBeVisible();
expect(screen.queryByRole("row", { name: data.lenses[1].findings[0].title })).not.toBeInTheDocument();
await user.click(screen.getByRole("combobox", { name: "Filter by priority" }));
await user.click(screen.getByRole("option", { name: "Low", exact: true }));
expect(await screen.findByText("No findings match these filters.")).toBeVisible();
await waitFor(() =>
expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).get("priority")).toBe("low"),
);
});
it("reopens an inbox finding from a link and navigates between findings with the keyboard", async () => {
const user = userEvent.setup();
const onUrlUpdate = vi.fn();
const rows = inboxRows(lenses);
renderWithLens(<FindingsView readOnly />, {
searchParams: `?tab=findings&issue=${encodeURIComponent(rows[0].key)}`,
onUrlUpdate,
});
const panel = await screen.findByRole("complementary", { name: "Finding details" });
expect(within(panel).getByRole("heading", { name: rows[0].title })).toBeVisible();
expect(within(panel).queryByRole("button", { name: "Mark resolved" })).not.toBeInTheDocument();
await user.keyboard("j");
expect(await within(panel).findByRole("heading", { name: rows[1].title })).toBeVisible();
await user.keyboard("{Escape}");
await waitFor(() => expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).has("issue")).toBe(false));
});
it("retains feedback and the open finding when a grouped review fails", async () => {
const user = userEvent.setup();
proxy.patch.mockImplementation(async (path) => {
if (path.startsWith("/lens/twin/")) throw new Error("Review could not be saved");
});
renderWithLens(<FindingsView />, { searchParams: "?tab=findings" });
await user.click(await screen.findByRole("row", { name: issue.title }));
fireEvent.change(screen.getByRole("textbox"), { target: { value: "Keep this feedback" } });
await user.click(screen.getByRole("button", { name: "Mark resolved" }));
expect(await screen.findByText("Review could not be saved")).toBeVisible();
expect(screen.getByRole("textbox")).toHaveValue("Keep this feedback");
expect(screen.getByRole("button", { name: "Mark resolved" })).toBeEnabled();
});
it("reviews only the selected check when two findings have the same title", async () => {
const other = { ...issue, id: "different-check", check_id: "different-check" };
proxy.get.mockImplementation(async (path) =>
path === "/lens"
? { lenses: [{ ...support, findings: [issue, other] }], workers: [], tracing_enabled: true }
: { data: [] },
);
const user = userEvent.setup();
renderWithLens(<FindingsView />, { searchParams: "?tab=findings" });
const rows = await screen.findAllByRole("row", { name: issue.title });
expect(rows).toHaveLength(2);
await user.click(rows[0]);
await user.click(await screen.findByRole("button", { name: "Mark resolved" }));
await waitFor(() => expect(proxy.patch).toHaveBeenCalledTimes(1));
expect(proxy.patch.mock.calls[0][0]).toBe(`/lens/support/findings/${issue.id}`);
});
it("opens a grouped finding from a link to any of its owning investigations", async () => {
renderWithLens(<FindingsView readOnly />, {
searchParams: `?tab=findings&issue=${encodeURIComponent(findingKey(twin, issue))}`,
});
const panel = await screen.findByRole("complementary", { name: "Finding details" });
expect(within(panel).getByRole("heading", { name: issue.title })).toBeVisible();
expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "true");
});

241
ui/litellm-dashboard/src/components/lens/investigations/FindingsView.tsx Normal file
View file

@ -0,0 +1,241 @@
"use client";
import { useQuery } from "@tanstack/react-query";
import { ChevronRight } from "lucide-react";
import { Inspector } from "@/components/shared/Inspector";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useNow } from "@/hooks/useNow";
import { cn } from "@/lib/cva.config";
import { formatActivityTimestamp } from "@/utils/activityTimestamp";
import { useLensApi } from "../data/LensServices";
import { useLensUpdate } from "../data/mutations";
import { lensQueries } from "../data/queries";
import { agoLabel } from "../model/format";
import {
ALL_AGENTS,
filterInbox,
findingKey,
inboxAgents,
inboxFinding,
inboxRows,
sampledExecutions,
type InboxRow,
type Priority,
} from "../model/inbox";
import type { Finding } from "../model/types";
import { useEvidenceRoute, useInboxFilters, useIssueRoute } from "../route";
import { FINDING_PANEL_WIDTH_KEY } from "../storage";
import { EvidenceView } from "./Evidence";
import { FindingDetails } from "./FindingDetails";
import { InvestigationError, InvestigationsLoading } from "./InvestigationStates";
const PRIORITY_DOT = { high: "bg-destructive", medium: "bg-warning", low: "bg-muted-foreground/50" } as const;
const PRIORITIES: { value: Priority | "all"; label: string }[] = [
{ value: "all", label: "All priorities" },
{ value: "high", label: "High" },
{ value: "medium", label: "Medium" },
{ value: "low", label: "Low" },
];
function FilterSelect<T extends string>({
label,
value,
items,
onChange,
}: {
label: string;
value: T;
items: { value: T; label: string }[];
onChange: (value: T) => void;
}) {
return (
<Select items={items} value={value} onValueChange={(next: T | null) => next !== null && onChange(next)}>
<SelectTrigger size="sm" className="h-7 min-w-32 text-xs" aria-label={label}>
<SelectValue />
</SelectTrigger>
<SelectContent>
{items.map((item) => (
<SelectItem key={item.value} value={item.value}>
{item.label}
</SelectItem>
))}
</SelectContent>
</Select>
);
}
function InboxDetail({
row,
readOnly,
busy,
onReview,
}: {
row: InboxRow;
readOnly: boolean;
busy: boolean;
onReview: (row: InboxRow, status: Finding["status"], reason: string) => void;
}) {
const { evidence, setEvidence } = useEvidenceRoute();
const owner =
row.sources.find(({ finding }) => finding.evidence.some((quote) => quote.execution_id === evidence?.id)) ??
row.sources[0];
return (
<>
<div hidden={evidence !== null} className={evidence ? undefined : "flex min-h-0 flex-1 flex-col"}>
<FindingDetails
key={row.key}
finding={inboxFinding(row)}
agents={row.agents}
sampledRuns={row.sources.flatMap(({ lens }) => sampledExecutions(lens))}
readOnly={readOnly}
busy={busy}
onOpenEvidence={setEvidence}
onReview={(status, reason) => onReview(row, status, reason)}
/>
</div>
{evidence && (
<EvidenceView
lensId={owner.lens.id}
evidence={evidence}
backLabel="Back to finding"
onBack={() => setEvidence(null)}
/>
)}
</>
);
}
export function FindingsView({ readOnly = false }: { readOnly?: boolean }) {
const api = useLensApi();
const list = useQuery(lensQueries.list(api));
const update = useLensUpdate();
const { issueKey, setIssueKey } = useIssueRoute();
const filters = useInboxFilters();
const now = useNow(30000);
const all = inboxRows(list.data?.lenses ?? []);
const rows = filterInbox(all, filters);
const selected =
all.find((row) => row.sources.some(({ lens, finding }) => findingKey(lens, finding) === issueKey)) ?? null;
const agents = [
{ value: ALL_AGENTS, label: "All agents" },
...inboxAgents(all).map((agent) => ({ value: agent, label: agent })),
];
const review = (row: InboxRow, status: Finding["status"], reason: string) => {
update.mutate(
async (api) => {
const results = await Promise.allSettled(
row.sources.map(({ lens, finding }) => api.reviewFinding(lens.id, finding.id, status, reason)),
);
const failed = results.find((result) => result.status === "rejected");
if (failed) throw failed.reason;
},
{ onSuccess: () => setIssueKey(null) },
);
};
if (list.isPending) return <InvestigationsLoading />;
return (
<Inspector.Root
items={rows}
itemKey={(row) => row.key}
selected={selected}
onSelectedChange={(row) => setIssueKey(row?.key ?? null)}
noun="finding"
storageKey={FINDING_PANEL_WIDTH_KEY}
>
<section aria-label="Findings" className="flex min-h-0 flex-1 flex-col">
{(list.error || update.error) && (
<InvestigationError
message={(list.error ?? update.error)!.message}
refresh={() => {
update.reset();
void list.refetch();
}}
/>
)}
<div className="flex min-h-10 shrink-0 flex-wrap items-center gap-2 border-b px-3 py-1.5">
<FilterSelect label="Filter by agent" value={filters.agent} items={agents} onChange={filters.setAgent} />
<FilterSelect
label="Filter by priority"
value={filters.priority}
items={PRIORITIES}
onChange={filters.setPriority}
/>
</div>
<div className="min-h-0 flex-1 overflow-auto">
<table aria-label="Findings" className="w-full table-fixed border-collapse text-left text-xs">
<thead className="sticky top-0 z-sticky bg-muted/40 backdrop-blur">
<tr className="h-8 border-b text-xs tracking-wider text-muted-foreground uppercase">
<th className="w-20 px-3 font-medium">Priority</th>
<th className="px-3 font-medium">Finding</th>
<th className="hidden w-40 px-3 font-medium md:table-cell">Agent</th>
<th className="hidden w-16 px-3 text-right font-medium sm:table-cell">Runs</th>
<th className="hidden w-24 px-3 font-medium lg:table-cell">Last seen</th>
<th className="w-7">
<span className="sr-only">Details</span>
</th>
</tr>
</thead>
<tbody>
{rows.map((row) => (
<Inspector.Row
key={row.key}
item={row}
render={
<tr
tabIndex={0}
aria-label={row.title}
className="h-9 cursor-pointer border-b border-border/60 hover:bg-trace-row-hover focus-visible:outline-2 focus-visible:outline-ring data-[state=selected]:bg-trace-row-selected"
/>
}
>
<td className="px-3">
<span className="inline-flex items-center gap-1.5 text-muted-foreground">
<span aria-hidden="true" className={cn("size-1.5 rounded-full", PRIORITY_DOT[row.priority])} />
{row.priority}
</span>
</td>
<td className="px-3 py-2 sm:py-0" title={row.suggestion || undefined}>
<span className="line-clamp-2 text-foreground sm:block sm:truncate">{row.title}</span>
<span className="mt-1 block text-xs text-muted-foreground md:hidden">
{row.agents.join(", ")} · {row.runs} {row.runs === 1 ? "run" : "runs"}
</span>
</td>
<td
className="hidden truncate px-3 text-muted-foreground md:table-cell"
title={row.agents.join(", ")}
>
{row.agents.join(", ")}
</td>
<td className="hidden px-3 text-right font-mono tabular-nums sm:table-cell">{row.runs}</td>
<td
className="hidden px-3 tabular-nums text-muted-foreground lg:table-cell"
title={formatActivityTimestamp(row.lastSeen)}
>
{agoLabel(Date.parse(row.lastSeen), now)}
</td>
<td>
<ChevronRight aria-hidden="true" className="size-3 text-muted-foreground/60" />
</td>
</Inspector.Row>
))}
</tbody>
</table>
{!rows.length && !list.error && (
<p className="px-4 py-16 text-center text-xs text-muted-foreground">
{all.length
? "No findings match these filters."
: "No open findings yet. New problems show up here as soon as an investigation spots them."}
</p>
)}
</div>
<footer className="flex h-8 shrink-0 items-center border-t bg-muted/30 px-3 text-xs text-muted-foreground">
{rows.length} {rows.length === 1 ? "finding" : "findings"}
{rows.length !== all.length && ` of ${all.length}`}
</footer>
</section>
<Inspector.Panel label="Finding details" testId="finding-panel">
{(row: InboxRow) => <InboxDetail row={row} readOnly={readOnly} busy={update.isPending} onReview={review} />}
</Inspector.Panel>
</Inspector.Root>
);
}

187
ui/litellm-dashboard/src/components/lens/investigations/InvestigationList.tsx
View file

@ -2,25 +2,15 @@
import {
getCoreRowModel,
getExpandedRowModel,
useReactTable,
type CellContext,
type ColumnDef,
type TableOptions,
} from "@tanstack/react-table";
import { createContext, useContext, type ReactNode } from "react";
import {
ChevronRight,
Circle,
CircleCheck,
CircleDashed,
CircleDot,
CircleSlash,
CircleX,
Pencil,
Play,
} from "lucide-react";
import { ChevronRight, Pencil, Play } from "lucide-react";
import { useMediaQuery } from "usehooks-ts";
import { useNow } from "@/hooks/useNow";
import { Inspector } from "@/components/shared/Inspector";
import { InspectorTable } from "@/components/shared/InspectorTable";
@ -28,21 +18,18 @@ import { formatActivityTimestamp } from "@/utils/activityTimestamp";
import { cn } from "@/lib/cva.config";
import { agoLabel, scopeLabel } from "../model/format";
import { findingAgents, findingKey, openFindings, scheduleLabel } from "../model/inbox";
import { findingKey, openFindings, scheduleLabel } from "../model/inbox";
import { lensStatus } from "../model/status";
import { SearchBox } from "@/components/shared/search/SearchBox";
import { itemValues } from "@/components/shared/search/valueSource";
import { type Finding, type Lens } from "../model/types";
import { useListSearchRoute } from "../route";
import { useLensRoute, useListSearchRoute } from "../route";
import { FINDING_PANEL_WIDTH_KEY } from "../storage";
import { filterInvestigations, INVESTIGATION_INDEX, INVESTIGATION_QUERY } from "./investigationQuery";
const PRIORITY_COLOR = { high: "text-destructive", medium: "text-warning", low: "text-muted-foreground" } as const;
const META = "truncate text-xs text-muted-foreground";
const ACTION =
"inline-flex size-7 items-center justify-center rounded text-muted-foreground hover:bg-muted hover:text-foreground";
const INVESTIGATION_HEIGHT = 56;
const FINDING_HEIGHT = 48;
"inline-flex size-7 items-center justify-center rounded text-muted-foreground hover:bg-muted hover:text-foreground disabled:cursor-not-allowed disabled:opacity-50";
const INVESTIGATION_HEIGHT = 36;
/** One row of the list: an investigation, or an open finding shown under the investigation that owns it. */
export type InvestigationRow =
@ -52,15 +39,11 @@ export type InvestigationRow =
export const investigationRowKey = (row: InvestigationRow): string =>
row.kind === "investigation" ? `investigation:${row.lens.id}` : `finding:${findingKey(row.lens, row.finding)}`;
const findingRows = (row: InvestigationRow): InvestigationRow[] | undefined =>
row.kind === "investigation"
? openFindings(row.lens).map((finding) => ({ kind: "finding", lens: row.lens, finding }))
: undefined;
interface ListContextValue {
readonly now: number;
readonly connected: boolean;
readonly readOnly: boolean;
readonly demo: boolean;
readonly onEdit: (id: string) => void;
readonly onRunNow: (id: string) => void;
}
@ -77,76 +60,44 @@ type Cell = CellContext<InvestigationRow, unknown>;
const ROW_LABEL = { investigation: "Investigation details", finding: "Finding details" } as const;
function JobIcon({ lens }: { lens: Lens }) {
const status = lens.jobs[0]?.status;
const className = "size-4 shrink-0";
if (status === "queued" || status === "running")
return <CircleDashed aria-hidden="true" className={cn(className, "text-info")} />;
if (status === "failed") return <CircleX aria-hidden="true" className={cn(className, "text-destructive")} />;
if (status === "cancelled")
return <CircleSlash aria-hidden="true" className={cn(className, "text-muted-foreground")} />;
if (status === "completed") return <CircleCheck aria-hidden="true" className={cn(className, "text-success")} />;
return <Circle aria-hidden="true" className={cn(className, "text-muted-foreground")} />;
}
function TwoLine({ meta, title, className }: { meta: ReactNode; title: ReactNode; className?: string }) {
function NameCell({ row: { original: item } }: Cell) {
return (
<span className="flex min-w-0 flex-col gap-0.5">
<span className={META}>{meta}</span>
<span className={cn("truncate text-sm text-foreground", className)}>{title}</span>
<span className="flex min-w-0 flex-col">
<span className="truncate font-medium text-foreground">{item.lens.settings.name}</span>
<span className="truncate text-xs text-muted-foreground md:hidden">{scopeLabel(item.lens.settings)}</span>
</span>
);
}
function NameCell({ row }: Cell) {
function AgentCell({ row: { original: item } }: Cell) {
return <span className="block truncate text-muted-foreground">{scopeLabel(item.lens.settings)}</span>;
}
function ScheduleCell({ row: { original: item } }: Cell) {
const { now } = useList();
const item = row.original;
if (item.kind === "investigation")
return (
<span className="flex min-w-0 items-center gap-2">
<InspectorTable.Indent
row={row}
toggleLabel={(expanded) => `${expanded ? "Hide" : "Show"} findings for ${item.lens.settings.name}`}
/>
<JobIcon lens={item.lens} />
<TwoLine
meta={`${scopeLabel(item.lens.settings)} · ${scheduleLabel(item.lens, now)}`}
title={item.lens.settings.name}
className="font-semibold"
/>
</span>
);
const priority = item.finding.priority ?? "medium";
return (
<span
className="flex min-w-0 items-center gap-2"
title={item.finding.suggestion ? `Fix: ${item.finding.suggestion}` : undefined}
>
<InspectorTable.Indent row={row} className="h-12" />
<span aria-hidden="true" className="w-4 shrink-0" />
<CircleDot aria-hidden="true" className={cn("size-4 shrink-0", PRIORITY_COLOR[priority])} />
<TwoLine
meta={`${priority} priority · ${findingAgents(item.lens, item.finding).join(", ")}`}
title={item.finding.title}
<span className="inline-flex items-center gap-1.5 whitespace-nowrap text-muted-foreground">
<span
aria-hidden="true"
className={cn("size-1.5 rounded-full", item.lens.settings.enabled ? "bg-info" : "bg-muted-foreground/40")}
/>
{scheduleLabel(item.lens, now)}
</span>
);
}
function StatusCell({ row: { original: item } }: Cell) {
const { connected } = useList();
if (item.kind === "finding") {
const runs = item.finding.occurrences.length;
return (
<span className="text-muted-foreground">
{runs} {runs === 1 ? "run" : "runs"}
</span>
);
}
const failed = item.lens.jobs[0]?.status === "failed";
const { connected, now } = useList();
const latest = item.lens.jobs[0];
return (
<span className={cn("block truncate", failed ? "text-destructive" : "text-muted-foreground")}>
{lensStatus(item.lens, connected)}
<span
className="flex items-center gap-1.5 truncate"
title={latest ? formatActivityTimestamp(latest.created_at) : undefined}
>
<span className={cn(latest?.status === "failed" ? "text-destructive" : "text-muted-foreground")}>
{lensStatus(item.lens, connected)}
</span>
{latest && <span className="text-muted-foreground">· {agoLabel(Date.parse(latest.created_at), now)}</span>}
</span>
);
}
@ -165,34 +116,19 @@ function FindingCount({ row: { original: item } }: Cell) {
);
}
function ActivityCell({ row: { original: item } }: Cell) {
const { now } = useList();
if (item.kind === "finding")
return (
<span title={formatActivityTimestamp(item.finding.last_seen)}>
{agoLabel(Date.parse(item.finding.last_seen), now)}
</span>
);
const latest = item.lens.jobs[0];
return (
<span title={latest ? formatActivityTimestamp(latest.created_at) : undefined}>
{latest ? agoLabel(Date.parse(latest.created_at), now) : "never run"}
</span>
);
}
function ActionsCell({ row: { original: item } }: Cell) {
const { readOnly, onEdit, onRunNow } = useList();
const { readOnly, demo, onEdit, onRunNow } = useList();
if (item.kind === "finding")
return <ChevronRight aria-hidden="true" className="mr-2 ml-auto size-3.5 text-muted-foreground/60" />;
if (readOnly) return null;
if (readOnly && !demo) return null;
const { id, settings } = item.lens;
return (
<span className="flex items-center justify-end gap-0.5">
<button
type="button"
aria-label={`Run ${settings.name} now`}
title="Run now"
title={demo ? "Turn off Demo data to run an investigation" : "Run now"}
disabled={readOnly}
onClick={(event) => {
event.stopPropagation();
onRunNow(id);
@ -204,7 +140,8 @@ function ActionsCell({ row: { original: item } }: Cell) {
<button
type="button"
aria-label={`Edit ${settings.name}`}
title="Edit"
title={demo ? "Turn off Demo data to edit an investigation" : "Edit"}
disabled={readOnly}
onClick={(event) => {
event.stopPropagation();
onEdit(id);
@ -218,17 +155,18 @@ function ActionsCell({ row: { original: item } }: Cell) {
}
const COLUMNS: ColumnDef<InvestigationRow>[] = [
{ id: "name", header: "Investigation", cell: NameCell, meta: { className: "pl-2 pr-0" } },
{ id: "status", size: 180, header: "Status", cell: StatusCell, meta: { className: "text-right text-xs" } },
{ id: "findings", size: 72, header: "Open findings", cell: FindingCount, meta: { className: "text-right" } },
{ id: "name", header: "Investigation", cell: NameCell },
{ id: "agent", size: 160, header: "Agent", cell: AgentCell },
{ id: "schedule", size: 180, header: "Schedule", cell: ScheduleCell },
{ id: "status", size: 200, header: "Last run", cell: StatusCell },
{ id: "findings", size: 64, header: "Open", cell: FindingCount, meta: { numeric: true } },
{
id: "activity",
size: 120,
header: "Last activity",
cell: ActivityCell,
meta: { className: "text-right text-xs text-muted-foreground" },
id: "actions",
size: 76,
header: "Actions",
cell: ActionsCell,
meta: { headerClassName: "sr-only", className: "pl-0 pr-3" },
},
{ id: "actions", size: 76, header: "Actions", cell: ActionsCell, meta: { className: "pl-0 pr-3" } },
];
export interface InvestigationListProps {
@ -244,7 +182,6 @@ export interface InvestigationListProps {
readonly children: (row: InvestigationRow) => ReactNode;
}
/** Investigations with their open findings; any row opens in the side panel and J/K walk the visible rows. */
export function InvestigationList({
lenses,
connected,
@ -257,18 +194,18 @@ export function InvestigationList({
children,
}: InvestigationListProps) {
const [search, setSearch] = useListSearchRoute();
const { demo } = useLensRoute();
const now = useNow(15000);
const desktop = useMediaQuery("(min-width: 768px)");
const shown = filterInvestigations([...lenses], search);
const tableOptions: TableOptions<InvestigationRow> = {
data: shown.map((lens): InvestigationRow => ({ kind: "investigation", lens })),
columns: COLUMNS,
defaultColumn: { size: undefined },
state: { columnVisibility: { agent: desktop, schedule: desktop, status: desktop } },
getRowId: investigationRowKey,
getSubRows: findingRows,
initialState: { expanded: true },
autoResetAll: false,
getCoreRowModel: getCoreRowModel(),
getExpandedRowModel: getExpandedRowModel(),
};
const table = useReactTable(tableOptions);
const rows = table.getRowModel().rows.map((row) => row.original);
@ -282,30 +219,26 @@ export function InvestigationList({
noun={noun}
storageKey={FINDING_PANEL_WIDTH_KEY}
>
<div className="flex min-h-[420px] flex-1 flex-col overflow-hidden bg-card">
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-card">
<div className="flex min-h-11 shrink-0 flex-wrap items-center gap-2 border-b border-border bg-card p-2">
<SearchBox.Root
className="sm:max-w-96"
language={INVESTIGATION_QUERY}
values={itemValues(INVESTIGATION_INDEX, lenses)}
value={search}
onValueChange={setSearch}
label="Search investigations"
>
<SearchBox.Input
className="rounded-lg"
placeholder="Search investigations, or filter like status:failed schedule:watching"
/>
<SearchBox.Input className="rounded-md" placeholder="Search investigations" />
<SearchBox.Suggestions />
</SearchBox.Root>
{actions && <div className="ml-auto flex items-center gap-2">{actions}</div>}
</div>
<ListContext.Provider value={{ now, connected, readOnly, onEdit, onRunNow }}>
<ListContext.Provider value={{ now, connected, readOnly, demo, onEdit, onRunNow }}>
<InspectorTable.Root table={table}>
<InspectorTable.Grid aria-label="Investigations" className="min-w-[720px]">
<InspectorTable.Header hidden />
<InspectorTable.Body<InvestigationRow>
rowHeight={(row) => (row.depth ? FINDING_HEIGHT : INVESTIGATION_HEIGHT)}
>
<InspectorTable.Grid aria-label="Investigations" className="text-xs md:min-w-[860px]">
<InspectorTable.Header />
<InspectorTable.Body<InvestigationRow> rowHeight={() => (desktop ? INVESTIGATION_HEIGHT : 48)}>
{(row) => (
<InspectorTable.Row
row={row}
@ -314,7 +247,7 @@ export function InvestigationList({
aria-label={
row.original.kind === "finding" ? row.original.finding.title : row.original.lens.settings.name
}
className={row.depth ? "h-12" : "group h-14"}
className="group h-12 md:h-9"
/>
)}
</InspectorTable.Body>
@ -326,6 +259,10 @@ export function InvestigationList({
)}
</InspectorTable.Root>
</ListContext.Provider>
<footer className="flex h-8 shrink-0 items-center border-t bg-muted/30 px-3 text-xs text-muted-foreground">
{shown.length} {shown.length === 1 ? "investigation" : "investigations"} ·{" "}
{shown.filter((lens) => lens.settings.enabled).length} watching
</footer>
</div>
<Inspector.Panel label={ROW_LABEL[noun]} testId="investigation-panel">
{children}

20
ui/litellm-dashboard/src/components/lens/investigations/InvestigationProgress.tsx
View file

@ -16,18 +16,30 @@ import { useProgressSamples } from "./useProgressSamples";
import { durationText } from "../model/format";
import { type Job } from "../model/types";
import { cn } from "@/lib/cva.config";
import { QueueReasonText } from "./QueueReasonText";
import { useQueueReason, type QueueContext } from "./useQueueReason";
const steps = ["review runs", "find patterns", "check evidence"];
const markers = { done: "✓", active: "▸", todo: "·" };
const blocks = 32;
const QUEUED_TITLE = { busy: "Waiting in line", no_worker: "Waiting for a worker", starting: "Picking up" } as const;
function stageState(index: number, current: number): keyof typeof markers {
if (index < current) return "done";
return index === current ? "active" : "todo";
}
export function InvestigationProgress({ job, onCancel }: { job: Job; onCancel?: () => void }) {
export function InvestigationProgress({
job,
queue,
onCancel,
}: {
job: Job;
queue?: QueueContext;
onCancel?: () => void;
}) {
const now = useNow(1000);
const reason = useQueueReason(job, queue);
const progress = analysisProgress(job);
const fraction = analysisFraction(progress);
const samples = useProgressSamples(progress);
@ -47,7 +59,7 @@ export function InvestigationProgress({ job, onCancel }: { job: Job; onCancel?:
<section aria-label="Analysis progress" className="space-y-3 rounded-md border bg-muted/40 px-4 py-3 text-sm">
<div className="flex items-center justify-between gap-2">
<span className="truncate" role="status">
<span className="font-medium">{progress.title}</span>
<span className="font-medium">{reason ? QUEUED_TITLE[reason.kind] : progress.title}</span>
{!queued && <span className="text-muted-foreground"> · {progress.detail}</span>}
</span>
{onCancel && (
@ -111,7 +123,9 @@ export function InvestigationProgress({ job, onCancel }: { job: Job; onCancel?:
</ol>
<div className="flex flex-wrap gap-x-5 gap-y-1 pt-1 tabular-nums">
{queued ? (
<span className="text-muted-foreground">{progress.detail}</span>
<span className="text-muted-foreground">
{reason ? <QueueReasonText reason={reason} onConnect={queue?.onConnect} /> : progress.detail}
</span>
) : (
stats.map(([key, value]) => (
<span key={key}>

165
ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx
View file

@ -93,6 +93,9 @@ const lens: Lens = {
findings: [pattern, issue],
assessments: [],
steps: [],
reviews: [],
reviewed: 0,
reading: [],
trigger: "schedule",
attempts: 0,
error: "",
@ -525,7 +528,7 @@ it("keeps saved investigations accessible when tracing is disabled", async () =>
return { data: [] };
});
renderWithProviders(<InvestigationsView readOnly />);
expect(await screen.findByRole("row", { name: issue.title })).toBeVisible();
expect(await screen.findByRole("row", { name: lens.settings.name })).toBeVisible();
expect(screen.queryByRole("region", { name: "Get Lens running" })).not.toBeInTheDocument();
});
@ -588,97 +591,17 @@ it("reopens a finding and a results section from shared links", async () => {
});
const panel = await screen.findByRole("complementary", { name: "Finding details" });
expect(within(panel).getByRole("heading", { name: issue.title })).toBeVisible();
expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "true");
await user.click(within(panel).getByRole("button", { name: "Close finding (Esc)" }));
await waitFor(() =>
expect(new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString)).has("issue")).toBe(false),
);
expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "false");
unmount();
renderWithProviders(<InvestigationsView />, { searchParams: `?lens=${lens.id}&section=checks` });
expect(await screen.findByRole("tab", { name: "Criteria", selected: true })).toBeVisible();
});
it("steps across findings and investigations with J and K, skipping hidden findings, and closes with Escape", async () => {
window.history.replaceState({}, "", "/lens/");
testQueryClient.clear();
const twinIssue: Finding = { ...issue, id: "twin-issue", title: "Twin review used the wrong defect rate" };
const twin: Lens = {
...lens,
id: "twin",
settings: { ...lens.settings, name: "Twin reviews" },
findings: [twinIssue],
jobs: lens.jobs.map((job) => ({ ...job, findings: [twinIssue] })),
};
proxy.get.mockImplementation(async (path) => {
if (path === "/lens") return { lenses: [lens, twin], tracing_enabled: true, workers: [] };
if (path === "/lens/activity/available") return { traces: true, requests: false };
if (path.endsWith("/runs")) return [];
return { data: [] };
});
const onUrlUpdate = vi.fn();
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />, { onUrlUpdate });
const lastIssue = () => new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")).get("issue");
await user.click(await screen.findByRole("button", { name: `Hide findings for ${twin.settings.name}` }));
expect(screen.queryByRole("row", { name: twinIssue.title })).not.toBeInTheDocument();
await user.click(screen.getByRole("row", { name: issue.title }));
const panel = await screen.findByRole("complementary", { name: "Finding details" });
expect(panel).toHaveTextContent("2 / 3");
await waitFor(() => expect(lastIssue()).toBe(findingKey(lens, issue)));
await user.keyboard("j");
await waitFor(() => expect(lastIssue()).toBeNull());
expect(screen.getByRole("row", { name: twin.settings.name })).toHaveAttribute("aria-selected", "true");
expect(screen.queryByRole("row", { name: twinIssue.title })).not.toBeInTheDocument();
const twinPanel = screen.getByRole("complementary", { name: "Investigation details" });
expect(within(twinPanel).getByRole("heading", { level: 2, name: twin.settings.name })).toBeVisible();
expect(twinPanel).toHaveTextContent("3 / 3");
expect(screen.getByRole("button", { name: "Next investigation (J)" })).toBeDisabled();
await user.keyboard("k");
await waitFor(() => expect(lastIssue()).toBe(findingKey(lens, issue)));
await user.keyboard("{Escape}");
await waitFor(() => expect(lastIssue()).toBeNull());
expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "false");
});
it("opens an investigation beside the list and walks from it into its findings with J and K", async () => {
window.history.replaceState({}, "", "/lens/");
testQueryClient.clear();
proxy.get.mockImplementation(async (path) => {
if (path === "/lens") return { lenses: [lens], tracing_enabled: true, workers: [] };
if (path === "/lens/activity/available") return { traces: true, requests: false };
if (path.endsWith("/runs")) return [];
return { data: [] };
});
const onUrlUpdate = vi.fn();
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />, { onUrlUpdate });
const lastUrl = () => new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? ""));
await user.click(await screen.findByRole("row", { name: lens.settings.name }));
const panel = await screen.findByRole("complementary", { name: "Investigation details" });
expect(within(panel).getByRole("heading", { level: 2, name: lens.settings.name })).toBeVisible();
expect(screen.getByRole("table", { name: "Investigations" })).toBeVisible();
expect(panel).toHaveTextContent("1 / 2");
await waitFor(() => expect(lastUrl().get("lens")).toBe(lens.id));
await user.keyboard("j");
await waitFor(() => expect(lastUrl().get("issue")).toBe(findingKey(lens, issue)));
expect(lastUrl().has("lens")).toBe(false);
expect(screen.getByRole("complementary", { name: "Finding details" })).toHaveTextContent("2 / 2");
expect(screen.getByRole("row", { name: issue.title })).toHaveAttribute("aria-selected", "true");
await user.keyboard("k");
await waitFor(() => expect(lastUrl().get("lens")).toBe(lens.id));
expect(lastUrl().has("issue")).toBe(false);
expect(screen.getByRole("complementary", { name: "Investigation details" })).toHaveTextContent("1 / 2");
});
it("lists each finding under the investigation that owns it and resolves only that copy", async () => {
window.history.replaceState({}, "", "/lens/");
it("walks the compact investigation list with J and K and closes with Escape", async () => {
testQueryClient.clear();
const twin: Lens = { ...lens, id: "twin", settings: { ...lens.settings, name: "Twin reviews" } };
proxy.get.mockImplementation(async (path) => {
@ -687,15 +610,35 @@ it("lists each finding under the investigation that owns it and resolves only th
if (path.endsWith("/runs")) return [];
return { data: [] };
});
const onUrlUpdate = vi.fn();
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />, { searchParams: "", onUrlUpdate });
const lastLens = () => new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")).get("lens");
await user.click(await screen.findByRole("row", { name: lens.settings.name }));
expect(screen.getByRole("complementary", { name: "Investigation details" })).toHaveTextContent("1 / 2");
expect(screen.queryByRole("row", { name: issue.title })).not.toBeInTheDocument();
await user.keyboard("j");
await waitFor(() => expect(lastLens()).toBe(twin.id));
expect(screen.getByRole("row", { name: twin.settings.name })).toHaveAttribute("aria-selected", "true");
expect(screen.getByRole("button", { name: "Next investigation (J)" })).toBeDisabled();
await user.keyboard("k");
await waitFor(() => expect(lastLens()).toBe(lens.id));
await user.keyboard("{Escape}");
await waitFor(() => expect(lastLens()).toBeNull());
});
it("reviews only the owning investigation when following an existing finding link", async () => {
testQueryClient.clear();
const twin: Lens = { ...lens, id: "twin", settings: { ...lens.settings, name: "Twin reviews" } };
proxy.get.mockImplementation(async (path) => {
if (path === "/lens") return { lenses: [lens, twin], tracing_enabled: true, workers: [] };
return { data: [] };
});
proxy.patch.mockResolvedValue(undefined);
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />);
const rows = await screen.findAllByRole("row", { name: issue.title });
expect(rows).toHaveLength(2);
await user.click(await screen.findByRole("button", { name: `Hide findings for ${lens.settings.name}` }));
const remaining = screen.getByRole("row", { name: issue.title });
expect(remaining.previousElementSibling).toBe(screen.getByRole("row", { name: twin.settings.name }));
await user.click(remaining);
renderWithProviders(<InvestigationsView />, {
searchParams: `?issue=${encodeURIComponent(findingKey(twin, issue))}`,
});
await user.click(await screen.findByRole("button", { name: "Mark resolved" }));
await waitFor(() => expect(proxy.patch).toHaveBeenCalledTimes(1));
expect(proxy.patch.mock.calls[0][0]).toBe("/lens/twin/findings/issue");
@ -816,9 +759,9 @@ it("keeps a finding open to retry when its update fails", async () => {
if (String(path).startsWith("/lens/twin/")) throw new Error("Twin reviews could not be updated");
});
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />);
await user.click(await screen.findByRole("button", { name: `Hide findings for ${lens.settings.name}` }));
await user.click(screen.getByRole("row", { name: issue.title }));
renderWithProviders(<InvestigationsView />, {
searchParams: `?issue=${encodeURIComponent(findingKey(twin, issue))}`,
});
await user.click(await screen.findByRole("button", { name: "Mark resolved" }));
expect(await screen.findByText("Twin reviews could not be updated")).toBeInTheDocument();
expect(screen.getByRole("button", { name: "Mark resolved" })).toBeVisible();
@ -845,6 +788,7 @@ it("cancels the running job from the progress banner", async () => {
testQueryClient.clear();
const running = { ...lens.jobs[0], id: "live", status: "running" as const, stage: "Reading executions" };
proxy.get.mockImplementation(async (path) => {
if (path.endsWith("/reviews")) return { reviews: [], reviewed: 0 };
if (path === "/lens")
return { lenses: [{ ...lens, jobs: [running, lens.jobs[0]] }], workers: [], tracing_enabled: true };
if (path === "/lens/lens/runs") return [running, lens.jobs[0]];
@ -860,6 +804,7 @@ it("refreshes run history as soon as the list reports a job the scheduler starte
testQueryClient.clear();
const runs = vi.fn().mockResolvedValue(lens.jobs);
proxy.get.mockImplementation(async (path) => {
if (path.endsWith("/reviews")) return { reviews: [], reviewed: 0 };
if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true };
if (path === "/lens/lens/runs") return runs();
return { data: [] };
@ -883,3 +828,39 @@ it("refreshes run history as soon as the list reports a job the scheduler starte
expect(await history.findByText("queued")).toBeVisible();
expect(runs.mock.calls.length).toBeGreaterThan(fetched);
});
it("lists recorded agents in Run now and runs the one picked from the list", async () => {
testQueryClient.clear();
proxy.get.mockImplementation(async (path) => {
if (path === "/lens")
return {
lenses: [lens],
tracing_enabled: true,
workers: [
{
id: "worker",
name: "Worker",
revoked: false,
analysis_key_id: "a".repeat(64),
scope: lens.scope,
last_seen: new Date().toISOString(),
},
],
};
if (path === "/lens/lens/runs") return lens.jobs;
if (path === "/lens/agents") return ["billing-agent", "support-bot"];
if (path === "/lens/activity/available") return { traces: true, requests: false };
return { data: [] };
});
proxy.post.mockResolvedValue(lens);
const user = userEvent.setup();
renderWithProviders(<InvestigationsView />);
await user.click(await screen.findByRole("button", { name: "Run now" }));
const dialog = within(await screen.findByRole("dialog", { name: "Run now" }));
await user.click(dialog.getByRole("combobox", { name: "Agent" }));
expect(await screen.findByRole("option", { name: "support-bot" })).toBeVisible();
await user.click(await screen.findByRole("option", { name: "billing-agent" }));
expect(dialog.getByRole("combobox", { name: "Agent" })).toHaveValue("billing-agent");
await user.click(dialog.getByRole("button", { name: "Run now" }));
await waitFor(() => expect(sentBody(proxy.post, "/lens/lens/runs")).toEqual([{ agent_name: "billing-agent" }]));
});

17
ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.tsx
View file

@ -63,13 +63,20 @@ export function InvestigationsView({ readOnly = false }: InvestigationsViewProps
const actions = useInvestigationActions();
const { dialog, target, openDialog, closeDialog } = useDialogRoute();
const { issueKey, setIssueKey } = useIssueRoute();
const { lensId, setLensId } = useLensRoute();
const { lensId, setLensId, setTab } = useLensRoute();
const list = useQuery(lensQueries.list(api));
const status = useLensReadiness(true);
const { connected } = status;
const screenInput = { list, lensId, dialog, target };
const screen = investigationScreen(screenInput);
const lenses = list.data?.lenses ?? [];
const queue = {
api,
lenses,
workers: list.data?.workers ?? [],
onConnect: () => setTab("settings"),
onOpenLens: setLensId,
};
const lens = lenses.find((candidate) => candidate.id === lensId);
const peeked = issueKey ? findFinding(lenses, issueKey) ?? null : null;
const selectedRow = (open: Lens | undefined): InvestigationRow | null => {
@ -156,6 +163,7 @@ export function InvestigationsView({ readOnly = false }: InvestigationsViewProps
ready={status.ready}
busy={actions.busy}
connected={connected}
queue={queue}
onEdit={() => openDialog("edit")}
onDuplicate={() => openDialog("duplicate")}
onPause={() => void actions.pause(row.lens)}
@ -205,8 +213,11 @@ export function InvestigationsView({ readOnly = false }: InvestigationsViewProps
busy={actions.busy}
onClose={closeDialog}
onRun={async (request) => {
await actions.startRun(dialogLens, request);
closeDialog();
const started = await actions.startRun(dialogLens, request);
if (started) {
closeDialog();
setLensId(dialogLens.id);
}
}}
/>
)}

37
ui/litellm-dashboard/src/components/lens/investigations/QueueReasonText.tsx Normal file
View file

@ -0,0 +1,37 @@
"use client";
import { queueReasonText, workerTaskText, type QueueReason } from "../model/status";
import type { QueueContext } from "./useQueueReason";
export function QueueReasonText({ reason, onConnect }: { reason: QueueReason; onConnect?: () => void }) {
if (reason.kind !== "no_worker" || !onConnect) return <>{queueReasonText(reason)}</>;
return (
<>
No worker connected. Start one from{" "}
<button type="button" onClick={onConnect} className="font-medium text-foreground underline underline-offset-2">
Connect worker
</button>
.
</>
);
}
export function WorkerTasks({ reason, queue }: { reason: QueueReason; queue?: QueueContext }) {
if (reason.kind === "no_worker" || !reason.tasks.length) return null;
return (
<ol aria-label="What the worker is doing" className="flex flex-col">
{reason.tasks.map((task) => (
<li key={task.lensId}>
<button
type="button"
disabled={!queue}
onClick={() => queue?.onOpenLens(task.lensId)}
className="w-full truncate rounded px-1.5 py-0.5 text-left tabular-nums hover:bg-muted hover:text-foreground"
>
{workerTaskText(task)}
</button>
</li>
))}
</ol>
);
}

72
ui/litellm-dashboard/src/components/lens/investigations/StepFeed.tsx
View file

@ -1,72 +0,0 @@
"use client";
import { ChevronDown } from "lucide-react";
import { useState } from "react";
import { cn } from "@/lib/cva.config";
import { modelsUsed, stepLine, windowLabel } from "../model/inbox";
import type { Job } from "../model/types";
const STEP_MARK = { model: "✓", stage: "▸", error: "!" } as const;
const STEP_TONE = {
model: "text-foreground",
stage: "text-info",
error: "text-destructive",
} as const;
export function StepFeed({ job }: { job: Job }) {
const [open, setOpen] = useState(true);
const steps = job.steps ?? [];
const models = modelsUsed(steps);
const calls = steps.filter((s) => s.kind === "model").length;
return (
<section aria-label="Live investigation steps" className="rounded-md border bg-background font-mono text-xs">
<button
type="button"
onClick={() => setOpen(!open)}
aria-expanded={open}
className="flex w-full flex-wrap items-center justify-between gap-x-4 gap-y-1 px-3 py-2 text-left hover:bg-muted/40"
>
<span className="flex flex-wrap items-center gap-x-3 gap-y-1 text-muted-foreground">
<span className="font-semibold text-foreground">
{job.trigger === "manual" ? "manual run" : "scheduled run"}
</span>
<span>{windowLabel(job)}</span>
<span>
{calls} model {calls === 1 ? "call" : "calls"} · ${job.cost.toFixed(4)}
</span>
{models.length > 0 && (
<span data-testid="step-feed-models">
using <span className="text-foreground">{models.join(", ")}</span>
</span>
)}
</span>
<ChevronDown className={cn("size-3.5 text-muted-foreground transition-transform", open && "rotate-180")} />
</button>
{open && (
<ol className="max-h-56 space-y-0.5 overflow-y-auto border-t px-3 py-2" data-testid="step-feed">
{steps.length === 0 && <li className="text-muted-foreground">Waiting for the worker to start…</li>}
{[...steps].reverse().map((step, index) => (
<li
key={`${step.at}-${index}`}
className={cn(
"grid grid-cols-[1rem_4.5rem_minmax(0,1fr)_auto] items-baseline gap-2",
index === 0 && "motion-safe:animate-in motion-safe:fade-in",
)}
>
<span aria-hidden="true" className={STEP_TONE[step.kind]}>
{index === 0 && step.kind === "stage" ? "▸" : STEP_MARK[step.kind]}
</span>
<span className="tabular-nums text-muted-foreground">
{new Date(step.at).toLocaleTimeString(undefined, { hour12: false })}
</span>
<span className={cn("truncate", STEP_TONE[step.kind])}>{stepLine(step)}</span>
<span className="truncate text-muted-foreground">{step.model}</span>
</li>
))}
</ol>
)}
</section>
);
}

18
ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationDetail.tsx
View file

@ -5,7 +5,9 @@ import { Button } from "@/components/ui/button";
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { RunsTab } from "./RunsTab";
import { InvestigationProgress } from "../InvestigationProgress";
import { StepFeed } from "../StepFeed";
import { LiveRunLoader } from "../live/LiveRunLoader";
import type { QueueContext } from "../useQueueReason";
import { liveJob } from "../../model/live";
import { InvestigationSummary } from "./InvestigationSummary";
import { InvestigationFailure } from "./InvestigationFailure";
import { scopeLabel, sourceLabels } from "../../model/format";
@ -28,6 +30,7 @@ export type InvestigationDetailProps = InvestigationIntents & {
readonly ready: boolean;
readonly busy: boolean;
readonly connected: boolean;
readonly queue: QueueContext;
readonly onReviewFinding: (owned: OwnedFinding, status: Finding["status"], reason: string) => void;
};
@ -37,6 +40,7 @@ export function InvestigationDetail({
ready,
busy,
connected,
queue,
onCancelRun,
onReviewFinding,
...intents
@ -45,6 +49,7 @@ export function InvestigationDetail({
const snapshot = useRunSnapshot(lens);
const { job, batchId, batchSettings, batchFindings, missingSnapshot } = snapshot;
const active = activeJob(lens.jobs);
const live = liveJob(lens.jobs);
return (
<div>
<section className="min-w-0 space-y-5">
@ -58,8 +63,15 @@ export function InvestigationDetail({
{!readOnly && <InvestigationActions lens={lens} ready={ready} busy={busy} {...intents} />}
</div>
<InvestigationSummary lens={lens} connected={connected} />
{active && <InvestigationProgress key={active.id} job={active} onCancel={readOnly ? undefined : onCancelRun} />}
{active && <StepFeed job={active} />}
{active && (
<InvestigationProgress
key={active.id}
job={active}
queue={queue}
onCancel={readOnly ? undefined : onCancelRun}
/>
)}
{live && <LiveRunLoader key={live.id} lensId={lens.id} job={live} name={lens.settings.name} queue={queue} />}
{job?.error && <InvestigationFailure job={job} connected={connected} />}
<Tabs value={section} onValueChange={setSection} key={lens.id}>
<div className="flex flex-wrap items-center justify-between gap-x-4 gap-y-2 border-b">

39
ui/litellm-dashboard/src/components/lens/investigations/detail/RunNowDialog.tsx
View file

@ -4,6 +4,14 @@ import { useState } from "react";
import { useQuery } from "@tanstack/react-query";
import { Button } from "@/components/ui/button";
import {
Combobox,
ComboboxContent,
ComboboxEmpty,
ComboboxInput,
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import {
Dialog,
DialogContent,
@ -64,18 +72,25 @@ export function RunNowDialog({
<div className="space-y-4">
<label className="grid gap-1.5 text-sm font-medium">
Agent
<Input
list="run-now-agents"
value={agent}
placeholder="All agents"
onChange={(e) => setAgent(e.target.value)}
aria-label="Agent"
/>
<datalist id="run-now-agents">
{agents.map((name) => (
<option key={name} value={name} />
))}
</datalist>
<Combobox
items={agents}
value={agent || null}
inputValue={agent}
onInputValueChange={setAgent}
onValueChange={(name) => setAgent(name ?? "")}
>
<ComboboxInput aria-label="Agent" placeholder="All agents" showClear={!!agent} className="h-9 w-full" />
<ComboboxContent>
<ComboboxEmpty>No matches. You can enter a recorded name.</ComboboxEmpty>
<ComboboxList>
{(name: string) => (
<ComboboxItem key={name} value={name}>
{name}
</ComboboxItem>
)}
</ComboboxList>
</ComboboxContent>
</Combobox>
</label>
<fieldset className="space-y-2">
<legend className="text-sm font-medium">Traces to review</legend>

105
ui/litellm-dashboard/src/components/lens/investigations/live/ConclusionsPanel.tsx Normal file
View file

@ -0,0 +1,105 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { cn } from "@/lib/cva.config";
import { grownGroups, share, type Conclusion } from "../../model/live";
const FLASH_MS = 150;
function useFlashing(groups: readonly Conclusion[]): ReadonlySet<string> {
const previous = useRef(groups);
const [flashing, setFlashing] = useState<ReadonlySet<string>>(new Set());
useEffect(() => {
const grown = grownGroups(previous.current, groups);
previous.current = groups;
if (!grown.size) return;
setFlashing(grown);
const timer = window.setTimeout(() => setFlashing(new Set()), FLASH_MS);
return () => window.clearTimeout(timer);
}, [groups]);
return flashing;
}
export function ConclusionsPanel({
groups,
total,
scope,
selected,
onSelect,
}: {
groups: readonly Conclusion[];
total: number;
scope: string;
selected: string | null;
onSelect: (key: string | null) => void;
}) {
const flashing = useFlashing(groups);
if (!groups.length) return <p className="text-xs text-muted-foreground">Nothing concluded yet.</p>;
return (
<div className="flex flex-col gap-1.5">
<ol aria-label="Conclusions" className="flex flex-col gap-1.5">
{groups.map((group) => {
const active = selected === group.key;
return (
<li key={group.key}>
<button
type="button"
aria-pressed={active}
title={group.latest}
onClick={() => onSelect(active ? null : group.key)}
className={cn(
"flex w-full flex-col gap-1.5 rounded-lg px-3 py-2 text-left transition-colors duration-150 active:scale-[0.99] motion-reduce:transition-none",
active ? "bg-background ring-[1.5px] ring-inset ring-foreground" : "bg-muted/50 hover:bg-muted",
flashing.has(group.key) && "bg-muted",
)}
>
<span className="flex items-start justify-between gap-3">
<span className="flex min-w-0 items-start gap-2">
<span
aria-hidden="true"
className={cn(
"mt-[0.4rem] size-1.5 shrink-0 rounded-full",
group.issue ? "bg-destructive" : "bg-muted-foreground/40",
)}
/>
<span className="flex min-w-0 flex-col gap-0.5">
<span className="line-clamp-2 text-sm leading-snug font-medium text-foreground">
{group.label}
</span>
<span className="line-clamp-1 text-xs text-muted-foreground">{group.latest}</span>
</span>
</span>
<span className="flex shrink-0 flex-col items-end gap-0.5">
<span
className={cn(
"text-xl leading-none font-semibold tabular-nums",
group.count ? "text-foreground" : "text-muted-foreground",
)}
>
{group.count}
</span>
{group.noted > 0 && (
<span className="text-xs tabular-nums text-muted-foreground">{group.noted} noted</span>
)}
</span>
</span>
<span aria-hidden="true" className="h-1 w-full overflow-hidden rounded-full bg-muted">
<span
className={cn(
"block h-full rounded-full transition-[width] duration-300 motion-reduce:transition-none",
group.issue ? "bg-destructive/70" : "bg-muted-foreground/40",
)}
style={{ width: `${share(group.count, total) * 100}%` }}
/>
</span>
</button>
</li>
);
})}
</ol>
{scope && <p className="text-xs text-muted-foreground">{scope}</p>}
</div>
);
}

158
ui/litellm-dashboard/src/components/lens/investigations/live/LiveDrawer.tsx Normal file
View file

@ -0,0 +1,158 @@
"use client";
import { useState, type ReactNode } from "react";
import { StatusBadge, type StatusTone } from "@/components/shared/table_cells";
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet";
import { conclusions } from "../../model/live";
import { releasedReviews } from "../../model/stage";
import type { InFlight, Job, Review, Settings } from "../../model/types";
import { ConclusionsPanel } from "./ConclusionsPanel";
import { ModelName } from "./LiveStrip";
import { NowReading } from "./NowReading";
import { TraceList } from "./TraceList";
import { useStage } from "./useStage";
const STATUS: Record<Job["status"], { label: string; tone: StatusTone }> = {
queued: { label: "Queued", tone: "neutral" },
running: { label: "Running", tone: "neutral" },
completed: { label: "Done", tone: "neutral" },
failed: { label: "Failed", tone: "error" },
cancelled: { label: "Cancelled", tone: "neutral" },
};
const PANE_TITLE = "text-xs font-semibold text-foreground";
function Stage({
reviews,
reading,
running,
slots,
model,
counter,
checks,
children,
}: {
reviews: readonly Review[];
reading: readonly InFlight[];
running: boolean;
slots: number;
model: string;
counter: string;
checks: Settings["checks"];
children: (listed: readonly Review[], groups: ReturnType<typeof conclusions>, nowReading: ReactNode) => ReactNode;
}) {
const { stage, now, charMs } = useStage(reviews, reading, running, slots);
const listed = releasedReviews(reviews, stage);
const nowReading = running && (
<NowReading model={model} counter={counter} lanes={stage.lanes} now={now} charMs={charMs} />
);
return <>{children(listed, conclusions(listed, checks), nowReading)}</>;
}
export function LiveDrawer({
open,
onClose,
name,
model,
status,
reviewed,
reviews,
reading,
counter,
done,
slots,
checks,
scope,
waiting,
}: {
open: boolean;
onClose: () => void;
name: string;
model: string;
status: Job["status"];
reviewed: number;
reviews: readonly Review[];
reading: readonly InFlight[];
counter: string;
done: string | null;
slots: number;
checks: Settings["checks"];
scope: string;
waiting?: ReactNode;
}) {
const [group, setGroup] = useState<string | null>(null);
const badge = STATUS[status];
const running = status === "running";
return (
<Sheet open={open} onOpenChange={(value) => !value && onClose()}>
<SheetContent className="flex h-full flex-col gap-0 data-[side=right]:w-full data-[side=right]:max-w-full data-[side=right]:sm:max-w-[min(1200px,94vw)]">
<SheetHeader className="shrink-0 gap-1.5 border-b px-5">
<SheetTitle className="pr-10 text-base">{name}</SheetTitle>
<SheetDescription render={<div />} className="flex flex-wrap items-center gap-2.5 pr-10">
<StatusBadge tone={badge.tone} label={badge.label} />
<ModelName model={model} size="md" />
</SheetDescription>
</SheetHeader>
{open && (
<Stage
reviews={reviews}
reading={reading}
running={running}
slots={slots}
model={model}
counter={counter}
checks={checks}
>
{(listed, groups, nowReading) => (
<div className="grid min-h-0 flex-1 grid-cols-1 grid-rows-2 md:grid-cols-[minmax(0,55fr)_minmax(0,45fr)] md:grid-rows-1">
<section
aria-label="Traces"
className="min-h-0 overflow-y-auto border-b px-4 py-4 md:border-r md:border-b-0"
>
<div className="mb-2 flex flex-wrap items-baseline justify-between gap-x-3 gap-y-1 px-2">
<h2 className={PANE_TITLE}>Traces</h2>
<span className="text-xs leading-relaxed tabular-nums text-muted-foreground">
{done ?? `${reviewed} reviewed`}
{done && (
<>
{" "}
<ModelName model={model} />
</>
)}
{reviewed > reviews.length && reviews.length ? ` · showing latest ${reviews.length}` : ""}
</span>
</div>
{nowReading}
{listed.length || reviews.length ? (
<TraceList reviews={listed} group={group} />
) : (
!running && (
<div className="flex flex-col gap-1 px-2 py-6 text-xs text-muted-foreground">
<p className="text-sm text-foreground">Waiting for the first trace…</p>
{waiting}
</div>
)
)}
</section>
<section
aria-label="Conclusions so far"
className="flex min-h-0 flex-col gap-3 overflow-y-auto px-5 py-4"
>
<h2 className={PANE_TITLE}>Conclusions so far</h2>
<ConclusionsPanel
groups={groups}
total={listed.length}
scope={scope}
selected={group}
onSelect={setGroup}
/>
</section>
</div>
)}
</Stage>
)}
</SheetContent>
</Sheet>
);
}

87
ui/litellm-dashboard/src/components/lens/investigations/live/LiveRun.tsx Normal file
View file

@ -0,0 +1,87 @@
"use client";
import { useState } from "react";
import { modelsUsed } from "../../model/inbox";
import { analysisModel, doneLine, inFlight, issueCount, nowLine, stripState } from "../../model/live";
import { queueReasonText } from "../../model/status";
import type { Job, Review } from "../../model/types";
import { QueueReasonText, WorkerTasks } from "../QueueReasonText";
import { useQueueReason, type QueueContext } from "../useQueueReason";
import { LiveDrawer } from "./LiveDrawer";
import { LiveStrip } from "./LiveStrip";
import { useStripOpen } from "./useLivePanels";
const MAX_LANES = 4;
export function LiveRun({
job,
reviews,
name,
queue,
}: {
job: Job;
reviews: readonly Review[];
name: string;
queue?: QueueContext;
}) {
const reason = useQueueReason(job, queue);
const [stripOpen, setStripOpen] = useStripOpen();
const [drawerOpen, setDrawerOpen] = useState(false);
const reading = inFlight(job);
const withReviews = { ...job, reviews: [...reviews] };
const model = analysisModel([...reviews.map((r) => r.model), ...modelsUsed(job.steps), job.settings.model]);
const issues = issueCount(withReviews);
const open = () => setDrawerOpen(true);
const waiting = reason && (
<>
<QueueReasonText reason={reason} onConnect={queue?.onConnect} />
<WorkerTasks reason={reason} queue={queue} />
</>
);
return (
<>
{stripOpen ? (
<LiveStrip
model={model}
state={stripState(withReviews, model, reason ? queueReasonText(reason) : undefined)}
waiting={waiting}
reviews={reviews}
reviewed={job.reviewed}
selected={job.coverage.selected}
issues={issues}
cost={job.cost}
onOpen={open}
onClose={() => setStripOpen(false)}
/>
) : (
<button
type="button"
onClick={() => setStripOpen(true)}
className="self-start text-xs text-muted-foreground hover:text-foreground"
>
Show live trace results
</button>
)}
<LiveDrawer
open={drawerOpen}
onClose={() => setDrawerOpen(false)}
name={name}
model={model}
status={job.status}
reviewed={job.reviewed}
reviews={reviews}
reading={reading}
counter={nowLine(job, reading.length)}
done={job.status === "completed" ? doneLine(job) : null}
slots={Math.max(1, Math.min(job.settings.concurrency, MAX_LANES))}
checks={job.settings.checks}
scope={
job.reviewed > reviews.length ? `From the latest ${reviews.length} of ${job.reviewed} reviewed traces` : ""
}
waiting={waiting}
/>
</>
);
}

21
ui/litellm-dashboard/src/components/lens/investigations/live/LiveRunLoader.tsx Normal file
View file

@ -0,0 +1,21 @@
"use client";
import type { Job } from "../../model/types";
import type { QueueContext } from "../useQueueReason";
import { LiveRun } from "./LiveRun";
import { useJobReviews } from "./useJobReviews";
export function LiveRunLoader({
lensId,
job,
name,
queue,
}: {
lensId: string;
job: Job;
name: string;
queue: QueueContext;
}) {
const reviews = useJobReviews(queue.api, lensId, job);
return <LiveRun job={job} reviews={reviews} name={name} queue={queue} />;
}

158
ui/litellm-dashboard/src/components/lens/investigations/live/LiveStrip.tsx Normal file
View file

@ -0,0 +1,158 @@
"use client";
import { ArrowRight, PanelRight, X } from "lucide-react";
import type { ReactNode } from "react";
import { useLensApi } from "../../data/LensServices";
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
import { ProviderLogo } from "@/components/molecules/models/ProviderLogo";
import { agoLabel } from "../../model/format";
import { Button } from "@/components/ui/button";
import { useNow } from "@/hooks/useNow";
import { cn } from "@/lib/cva.config";
import { money } from "../../model/format";
import {
newestFirst,
outcome,
providerOf,
reviewKey,
shortVerdict,
type IssueCount,
type StripState,
} from "../../model/live";
import type { Review } from "../../model/types";
const RECENT = 3;
const MARK = { issue: "●", clear: "✓", unknown: "–" } as const;
const LOGO = { sm: "size-3.5", md: "size-[18px]" } as const;
export function ModelName({ model, size = "sm" }: { model: string; size?: keyof typeof LOGO }) {
const api = useLensApi();
const { data: catalog } = useModelCostMap(api.scope !== "demo");
if (!model) return null;
const provider = providerOf(model, catalog ?? {});
return (
<span data-testid="live-model" className="inline-flex shrink-0 items-center gap-1.5 text-foreground">
{provider && <ProviderLogo provider={provider} className={cn("shrink-0", LOGO[size])} />}
<span className={cn("font-mono", size === "md" ? "text-xs" : "text-xs")}>{model}</span>
</span>
);
}
function RecentLine({ review, now, onOpen }: { review: Review; now: number; onOpen: () => void }) {
const result = outcome(review);
const issue = result === "issue";
return (
<li className="motion-safe:animate-in motion-safe:fade-in motion-safe:slide-in-from-top-1 motion-safe:duration-300">
<button
type="button"
onClick={(event) => {
event.stopPropagation();
onOpen();
}}
className="grid w-full grid-cols-[0.75rem_minmax(0,9rem)_minmax(0,1fr)_auto] items-center gap-2 rounded px-1.5 py-0.5 text-left hover:bg-muted"
>
<span aria-label={result} className={issue ? "text-destructive" : ""}>
{MARK[result]}
</span>
<span className="truncate text-foreground">{review.agent || review.name}</span>
<span className={cn("truncate", issue && "text-destructive")}>{shortVerdict(review)}</span>
<span className="tabular-nums">{agoLabel(Date.parse(review.at), now)}</span>
</button>
</li>
);
}
function issueLabel({ count, scope }: IssueCount): string {
const noun = count === 1 ? "issue" : "issues";
if (scope === "findings") return `${count} ${noun} found`;
return scope ? `${count} ${noun} ${scope}` : `${count} ${noun}`;
}
export function LiveStrip({
model,
state,
reviews,
reviewed,
selected,
issues,
cost,
waiting,
onOpen,
onClose,
}: {
model: string;
state: StripState;
waiting?: ReactNode;
reviews: readonly Review[];
reviewed: number;
selected: number;
issues: IssueCount;
cost: number;
onOpen: () => void;
onClose: () => void;
}) {
const now = useNow(5000);
const recent = newestFirst(reviews, RECENT);
return (
<section
aria-label="Live trace results"
onClick={onOpen}
className="flex cursor-pointer flex-wrap items-start gap-x-6 gap-y-2 rounded-md bg-muted/30 px-3 py-2 text-xs text-muted-foreground transition-colors hover:bg-muted/50"
>
<div className="flex min-w-[16rem] items-start gap-3">
<div className="flex flex-col gap-1">
<ModelName model={model} />
<span className="tabular-nums">
{reviewed} of {selected} traces · {issueLabel(issues)} · {money(cost)}
</span>
</div>
<Button
variant="outline"
size="xs"
className="mt-0.5 shrink-0"
onClick={(event) => {
event.stopPropagation();
onOpen();
}}
>
<PanelRight />
View run
<ArrowRight />
</Button>
</div>
<div className="min-w-0 flex-1">
{state.kind === "failed" && (
<p role="alert" className="py-0.5 text-destructive">
{state.message}
</p>
)}
{state.kind === "waiting" && (
<div role="status" className="flex flex-col gap-0.5 py-0.5" onClick={(event) => event.stopPropagation()}>
{waiting ?? state.message}
</div>
)}
{recent.length > 0 && state.kind !== "waiting" && (
<ol aria-label="Recently reviewed traces" className="flex flex-col">
{recent.map((review) => (
<RecentLine key={reviewKey(review)} review={review} now={now} onOpen={onOpen} />
))}
</ol>
)}
</div>
<Button
variant="ghost"
size="icon-xs"
aria-label="Hide live trace results"
className="shrink-0"
onClick={(event) => {
event.stopPropagation();
onClose();
}}
>
<X />
</Button>
</section>
);
}

94
ui/litellm-dashboard/src/components/lens/investigations/live/NowReading.tsx Normal file
View file

@ -0,0 +1,94 @@
"use client";
import { cn } from "@/lib/cva.config";
import { durationLabel, outcome, shortVerdict } from "../../model/live";
import { laneText, typedChars, type Lane } from "../../model/stage";
import { ModelName } from "./LiveStrip";
const RED = "text-destructive";
function dotTone(reading: boolean, issueShown: boolean): string {
if (reading) return "bg-foreground motion-safe:animate-pulse";
return issueShown ? "bg-destructive" : "bg-foreground/50";
}
function LaneRow({ lane, now, charMs }: { lane: Lane; now: number; charMs: number }) {
const { review } = lane;
const text = review ? laneText(review.reasoning) : "";
const typed = typedChars(lane, now, charMs);
const typing = !!review && typed < text.length;
const issue = review ? outcome(review) === "issue" : false;
return (
<li className="flex flex-col gap-1 rounded-lg bg-background/70 px-2.5 py-2 text-xs ring-1 ring-border/60 motion-safe:animate-in motion-safe:fade-in motion-safe:slide-in-from-top-1 motion-safe:duration-200">
<div className="flex items-center gap-2">
<span aria-hidden="true" className={cn("size-1.5 shrink-0 rounded-full", dotTone(!review, issue && !typing))} />
<span className="truncate font-medium text-foreground">{lane.agent || "trace"}</span>
<span className="truncate font-mono text-xs text-muted-foreground">{lane.traceId.slice(0, 8)}</span>
<span className="ml-auto shrink-0 text-xs tabular-nums text-muted-foreground">
{review ? durationLabel(review.duration_ms) : `reading · ${durationLabel(Math.max(0, now - lane.startedAt))}`}
</span>
</div>
{!review ? (
<div aria-hidden="true" className="h-0.5 overflow-hidden rounded-full bg-muted">
<div className="h-full w-2/5 rounded-full bg-foreground/30 motion-safe:animate-lens-shimmer" />
</div>
) : (
<>
<p className="min-h-[1.25rem] leading-relaxed text-muted-foreground">
{text.slice(0, typed)}
{typing && (
<span aria-hidden="true" className="ml-px inline-block h-3 w-px translate-y-0.5 bg-foreground" />
)}
</p>
{!typing && (
<p
className={cn(
"truncate font-medium motion-safe:animate-in motion-safe:fade-in",
issue ? RED : "text-foreground",
)}
>
{shortVerdict(review)}
</p>
)}
</>
)}
</li>
);
}
export function NowReading({
model,
counter,
lanes,
now,
charMs,
}: {
model: string;
counter: string;
lanes: readonly Lane[];
now: number;
charMs: number;
}) {
return (
<section aria-label="Now reading" className="mb-3 flex flex-col gap-2 rounded-xl bg-muted p-2.5 ring-1 ring-border">
<header className="flex items-center justify-between gap-3 px-0.5">
<ModelName model={model} size="md" />
<span role="status" className="text-xs tabular-nums text-muted-foreground">
{counter}
</span>
</header>
{lanes.length ? (
<ol className="flex flex-col gap-1.5">
{lanes.map((lane) => (
<LaneRow key={lane.key} lane={lane} now={now} charMs={charMs} />
))}
</ol>
) : (
<div aria-hidden="true" className="h-0.5 overflow-hidden rounded-full bg-background/60">
<div className="h-full w-2/5 rounded-full bg-foreground/30 motion-safe:animate-lens-shimmer" />
</div>
)}
</section>
);
}

110
ui/litellm-dashboard/src/components/lens/investigations/live/TraceList.tsx Normal file
View file

@ -0,0 +1,110 @@
"use client";
import { useState } from "react";
import { agoLabel } from "../../model/format";
import { useNow } from "@/hooks/useNow";
import { cn } from "@/lib/cva.config";
import { briefReasoning, durationLabel, inGroup, newestFirst, outcome, shortVerdict } from "../../model/live";
import type { Review } from "../../model/types";
const LIMIT = 200;
const RED = "text-destructive";
const ROW =
"grid h-9 w-full grid-cols-[0.75rem_minmax(0,5rem)_minmax(0,1fr)_auto] sm:grid-cols-[0.75rem_minmax(0,8rem)_minmax(0,1fr)_auto] items-center gap-2 px-2 text-left text-xs";
function Expanded({ review }: { review: Review }) {
const verdicts = review.verdicts.length
? review.verdicts
: [
{
check_id: "",
kind: "pattern" as const,
summary: review.cannot_assess ? "Not enough evidence to judge" : "No issue observed",
},
];
return (
<div className="flex flex-col gap-1.5 px-7 pt-0.5 pb-2.5 text-xs leading-relaxed">
{review.reasoning && <p className="text-muted-foreground">{briefReasoning(review.reasoning)}</p>}
<ul className="flex flex-col gap-1">
{verdicts.map((verdict, index) => (
<li
key={`${verdict.check_id}-${index}`}
className={cn("flex gap-2", verdict.kind === "issue" ? RED : "text-foreground")}
>
<span
aria-hidden="true"
className={cn(
"mt-[0.45rem] size-1.5 shrink-0 rounded-full",
verdict.kind === "issue" ? "bg-destructive" : "bg-muted-foreground/40",
)}
/>
<span className="min-w-0">{verdict.summary}</span>
</li>
))}
</ul>
</div>
);
}
function DoneRow({
review,
now,
open,
onToggle,
}: {
review: Review;
now: number;
open: boolean;
onToggle: () => void;
}) {
const result = outcome(review);
return (
<>
<button type="button" aria-expanded={open} onClick={onToggle} className={cn(ROW, "rounded-lg hover:bg-muted/50")}>
<span
aria-label={result}
className={cn("size-1.5 rounded-full", result === "issue" ? "bg-destructive" : "bg-muted-foreground/40")}
/>
<span className="truncate text-foreground">{review.agent || review.name}</span>
<span
className={cn(
"truncate motion-safe:animate-in motion-safe:fade-in motion-safe:duration-300",
result === "issue" ? RED : "text-muted-foreground",
)}
>
{shortVerdict(review)}
</span>
<span className="text-xs tabular-nums text-muted-foreground">
{durationLabel(review.duration_ms)} · {agoLabel(Date.parse(review.at), now)}
</span>
</button>
{open && <Expanded review={review} />}
</>
);
}
export function TraceList({ reviews, group }: { reviews: readonly Review[]; group: string | null }) {
const now = useNow(5000);
const [expanded, setExpanded] = useState<string | null>(null);
const rows = newestFirst(reviews, LIMIT).filter((review) => inGroup(review, group));
if (!rows.length) return <p className="px-2 py-2 text-xs text-muted-foreground">No traces here yet.</p>;
return (
<ol aria-label="Reviewed traces" className="flex flex-col">
{rows.map((review) => (
<li
key={review.execution_id}
className="motion-safe:animate-in motion-safe:fade-in motion-safe:slide-in-from-top-3 motion-safe:duration-300"
>
<DoneRow
review={review}
now={now}
open={expanded === review.execution_id}
onToggle={() => setExpanded(expanded === review.execution_id ? null : review.execution_id)}
/>
</li>
))}
</ol>
);
}

46
ui/litellm-dashboard/src/components/lens/investigations/live/useJobReviews.integration.test.tsx Normal file
View file

@ -0,0 +1,46 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { render, screen } from "@testing-library/react";
import { expect, it, vi } from "vitest";
import { createLensDemo } from "../../data/demo/createLensDemo";
import { createLensDemoData } from "../../data/demo/fixtures";
import type { LensApi } from "../../data/service";
import type { Job } from "../../model/types";
import { useJobReviews } from "./useJobReviews";
function Reviews({ api, job }: { api: LensApi; job: Job }) {
const reviews = useJobReviews(api, "support", job);
return (
<ul>
{reviews.map((review) => (
<li key={review.execution_id}>{review.reasoning}</li>
))}
</ul>
);
}
it("replaces cached reviews when a worker reclaims the same run", async () => {
const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
const job = createLensDemoData().lenses[0].jobs[0];
const previous = { ...job.reviews[0], reasoning: "Previous attempt's conclusion" };
const current = { ...job.reviews[0], reasoning: "Current attempt's conclusion", at: "2026-10-05T12:00:00Z" };
const reviews = vi.fn<LensApi["reviews"]>().mockImplementation(async (_lens, _job, after) => ({
reviews: after === 0 ? [previous] : [],
reviewed: 1,
}));
const api = { ...createLensDemo().lens, reviews };
const view = (attempts: number) => (
<QueryClientProvider client={client}>
<Reviews api={api} job={{ ...job, attempts, reviewed: 1 }} />
</QueryClientProvider>
);
const { rerender } = render(view(1));
expect(await screen.findByText(previous.reasoning)).toBeVisible();
reviews.mockImplementation(async (_lens, _job, after) => ({
reviews: after === 0 ? [current] : [],
reviewed: 1,
}));
rerender(view(2));
expect(await screen.findByText(current.reasoning)).toBeVisible();
expect(screen.queryByText(previous.reasoning)).not.toBeInTheDocument();
});

24
ui/litellm-dashboard/src/components/lens/investigations/live/useJobReviews.ts Normal file
View file

@ -0,0 +1,24 @@
"use client";
import { useQuery } from "@tanstack/react-query";
import { useState } from "react";
import { lensQueries } from "../../data/queries";
import type { LensApi } from "../../data/service";
import { appendPage, EMPTY_FEED, polling, type ReviewFeed } from "../../model/live";
import type { Job, Review } from "../../model/types";
export function useJobReviews(
api: LensApi,
lensId: string,
job: Pick<Job, "id" | "status" | "reviewed" | "attempts">,
): readonly Review[] {
const attempt = job.attempts ?? 0;
const [state, setState] = useState<{ attempt: number; feed: ReviewFeed }>({ attempt, feed: EMPTY_FEED });
const feed = state.attempt === attempt ? state.feed : EMPTY_FEED;
const page = { lensId, jobId: job.id, attempt, after: feed.cursor, live: polling(job, feed) };
const { data } = useQuery(lensQueries.reviews(api, page));
const next = data ? appendPage(feed, data) : feed;
if (state.attempt !== attempt || next !== state.feed) setState({ attempt, feed: next });
return next.reviews;
}

30
ui/litellm-dashboard/src/components/lens/investigations/live/useLivePanels.ts Normal file
View file

@ -0,0 +1,30 @@
"use client";
import { useCallback, useState } from "react";
const STRIP_KEY = "litellm.lens.liveStrip";
function read(key: string): string | null {
try {
return window.localStorage.getItem(key);
} catch {
return null;
}
}
function write(key: string, value: string): void {
try {
window.localStorage.setItem(key, value);
} catch {
return;
}
}
export function useStripOpen(): [boolean, (open: boolean) => void] {
const [open, setOpen] = useState(() => read(STRIP_KEY) !== "closed");
const update = useCallback((next: boolean) => {
setOpen(next);
write(STRIP_KEY, next ? "open" : "closed");
}, []);
return [open, update];
}

33
ui/litellm-dashboard/src/components/lens/investigations/live/useStage.ts Normal file
View file

@ -0,0 +1,33 @@
"use client";
import { useEffect, useState } from "react";
import { stageTick, startStage, stepStage, type Stage } from "../../model/stage";
import type { InFlight, Review } from "../../model/types";
const CHAR_MS = 3;
function prefersReducedMotion(): boolean {
return typeof window !== "undefined" && window.matchMedia?.("(prefers-reduced-motion: reduce)").matches === true;
}
export function useStage(
reviews: readonly Review[],
reading: readonly InFlight[],
running: boolean,
slots: number,
): { stage: Stage; now: number; charMs: number } {
const [charMs] = useState(() => (prefersReducedMotion() ? 0 : CHAR_MS));
const [stage, setStage] = useState(() => startStage(reviews));
const [now, setNow] = useState(Date.now);
const input = { reading, reviews, now, slots, running, charMs };
const next = stepStage(stage, input);
if (next !== stage) setStage(next);
const tick = stageTick(next, running, now, charMs);
useEffect(() => {
if (!tick) return;
const timer = window.setInterval(() => setNow(Date.now()), tick);
return () => window.clearInterval(timer);
}, [tick]);
return { stage: next, now, charMs };
}

21
ui/litellm-dashboard/src/components/lens/investigations/useQueueReason.ts Normal file
View file

@ -0,0 +1,21 @@
"use client";
import { useNow } from "@/hooks/useNow";
import type { LensApi } from "../data/service";
import { queueReason, type QueueReason } from "../model/status";
import type { Job, Lens, LensList } from "../model/types";
export interface QueueContext {
api: LensApi;
lenses: readonly Lens[];
workers: readonly LensList["workers"][number][];
onConnect: () => void;
onOpenLens: (id: string) => void;
}
export function useQueueReason(job: Job, queue: QueueContext | undefined): QueueReason | null {
const now = useNow(1000);
if (job.status !== "queued" || !queue) return null;
return queueReason(job, queue.lenses, queue.workers, now);
}

68
ui/litellm-dashboard/src/components/lens/model/inbox.test.ts
View file

@ -2,6 +2,9 @@ import { describe, expect, it } from "vitest";
import {
UNKNOWN_AGENT,
inboxRows,
inboxFinding,
filterInbox,
findFinding,
findingAgents,
findingKey,
@ -42,7 +45,16 @@ const lens = (
findings,
jobs: [{ sample: { executions: runs } }],
next_run_at: "2026-10-03T12:10:00Z",
settings: { name: id, agent_name: agent, service: "", enabled: true, interval_minutes: 15, ...settings },
scope: { all_teams: true, team_id: "", api_key_hash: "" },
settings: {
name: id,
agent_name: agent,
service: "",
enabled: true,
interval_minutes: 15,
checks: [{ id: "c", instruction: "Check tool errors" }],
...settings,
},
}) as unknown as Lens;
describe("findingAgents", () => {
@ -149,3 +161,57 @@ describe("scheduleLabel", () => {
expect(scheduleLabel(lens("a", "x", []), Date.parse("2026-10-03T12:30:00Z"))).toBe("every 15m · due now");
});
});
it("groups matching findings by agent, combines distinct runs and evidence, and takes the highest priority", () => {
const quote = { execution_id: "run-1", span_id: "step", quote: "Failed", role: "support" as const };
const firstInput: Partial<Finding> = {
title: "Tool errors swallowed",
occurrences: ["run-1"],
evidence: [quote],
priority: "low",
};
const first = finding(firstInput);
const secondInput: Partial<Finding> = {
title: " tool ERRORS swallowed ",
occurrences: ["run-1", "run-2"],
evidence: [quote],
priority: "high",
last_seen: "2026-10-02T00:00:00Z",
};
const second = finding(secondInput);
const rows = inboxRows([
lens("a", "support", [first]),
lens("b", "support", [second]),
lens("c", "billing", [first]),
]);
expect(rows).toHaveLength(2);
expect(rows[0]).toMatchObject({ priority: "high", runs: 2, lastSeen: second.last_seen });
expect(rows[0].sources.map(({ lens }) => lens.id)).toEqual(["a", "b"]);
expect(inboxFinding(rows[0])).toMatchObject({ occurrences: ["run-1", "run-2"], evidence: [quote], priority: "high" });
expect(filterInbox(rows, { agent: "support", priority: "high" })).toEqual([rows[0]]);
expect(filterInbox(rows, { agent: "billing", priority: "high" })).toEqual([]);
});
it("keeps only open issues in the cross-investigation inbox", () => {
expect(
inboxRows([
lens("a", "support", [
finding({ kind: "pattern" }),
finding({ status: "resolved" }),
finding({ status: "dismissed" }),
]),
]),
).toEqual([]);
});
it("keeps identical titles separate when their checks or visibility scopes differ", () => {
const first = lens("a", "support", [finding({})]);
const otherCheck = lens("b", "support", [finding({ check_id: "other" })]);
const otherInstruction = lens("c", "support", [finding({})], {
settings: { checks: [{ id: "c", instruction: "A different review criterion" }] },
});
const otherTeam = { ...first, id: "d", scope: { ...first.scope, team_id: "another-team" } };
const rows = inboxRows([first, otherCheck, otherInstruction, otherTeam]);
expect(rows).toHaveLength(4);
expect(rows.map((row) => row.sources.map(({ lens }) => lens.id))).toEqual([["a"], ["b"], ["c"], ["d"]]);
});

82
ui/litellm-dashboard/src/components/lens/model/inbox.ts
View file

@ -3,6 +3,23 @@ import type { Finding, Job, Lens } from "./types";
export type Step = Job["steps"][number];
export type Priority = NonNullable<Finding["priority"]>;
export interface InboxRow {
key: string;
title: string;
agents: readonly string[];
priority: Priority;
suggestion: string;
runs: number;
lastSeen: string;
sources: readonly { lens: Lens; finding: Finding }[];
}
export interface InboxFilter {
agent: string;
priority: Priority | "all";
}
export const ALL_AGENTS = "all";
export const UNKNOWN_AGENT = "unknown agent";
const priorityRank = { high: 0, medium: 1, low: 2 } as const;
@ -12,6 +29,7 @@ export function sampledExecutions(lens: Lens) {
}
export function findingAgents(lens: Lens, finding: Finding): readonly string[] {
if (lens.settings.agent_name) return [lens.settings.agent_name];
const services = new Map(sampledExecutions(lens).map((run) => [run.id, run.service] as const));
const seen = new Set(finding.occurrences.map((id) => services.get(id)).filter((s): s is string => !!s));
if (seen.size) return [...seen].sort();
@ -26,6 +44,58 @@ function groupBy<T>(items: readonly T[], key: (item: T) => string): Map<string,
}, new Map<string, T[]>());
}
function inboxGroupKey(lens: Lens, finding: Finding, agents: readonly string[]): string {
const check = lens.settings.checks.find((candidate) => candidate.id === finding.check_id);
return JSON.stringify([
lens.scope.team_id,
lens.scope.api_key_hash,
lens.scope.all_teams,
lens.settings.source,
agents,
finding.check_id,
check?.instruction ?? lens.id,
finding.title.trim().toLowerCase(),
]);
}
export function inboxRows(lenses: readonly Lens[]): InboxRow[] {
const open = lenses.flatMap((lens) =>
lens.findings
.filter((f) => f.status === "open" && f.kind === "issue")
.map((finding) => ({ lens, finding, agents: findingAgents(lens, finding) })),
);
const grouped = groupBy(open, ({ lens, agents, finding }) => inboxGroupKey(lens, finding, agents));
return [...grouped.values()]
.map((sources): InboxRow => {
const best = sources.reduce((a, b) =>
priorityRank[a.finding.priority ?? "medium"] <= priorityRank[b.finding.priority ?? "medium"] ? a : b,
);
return {
key: findingKey(sources[0].lens, sources[0].finding),
title: best.finding.title,
agents: best.agents,
priority: best.finding.priority ?? "medium",
suggestion: best.finding.suggestion,
runs: new Set(sources.flatMap((s) => s.finding.occurrences)).size,
lastSeen: sources.map((s) => s.finding.last_seen).reduce((a, b) => (Date.parse(a) > Date.parse(b) ? a : b)),
sources: sources.map(({ lens, finding }) => ({ lens, finding })),
};
})
.sort(
(a, b) => priorityRank[a.priority] - priorityRank[b.priority] || Date.parse(b.lastSeen) - Date.parse(a.lastSeen),
);
}
export function filterInbox(rows: readonly InboxRow[], { agent, priority }: InboxFilter): InboxRow[] {
const agentMatches = (row: InboxRow) => agent === ALL_AGENTS || row.agents.includes(agent);
const priorityMatches = (row: InboxRow) => priority === "all" || row.priority === priority;
return rows.filter((row) => agentMatches(row) && priorityMatches(row));
}
export function inboxAgents(rows: readonly InboxRow[]): string[] {
return [...new Set(rows.flatMap((row) => row.agents))].sort();
}
export function openFindings(lens: Lens): Finding[] {
return lens.findings
.filter((f) => f.status === "open" && f.kind === "issue")
@ -89,3 +159,15 @@ export function windowLabel(job: Job): string {
const fmt = (iso: string) => new Date(iso).toLocaleString(undefined, WINDOW_FORMAT);
return `${fmt(job.start)} → ${fmt(job.end)}`;
}
export function inboxFinding(row: InboxRow): Finding {
const primary = row.sources.find(({ finding }) => finding.priority === row.priority) ?? row.sources[0];
const evidence = row.sources.flatMap(({ finding }) => finding.evidence);
return {
...primary.finding,
priority: row.priority,
last_seen: row.lastSeen,
occurrences: [...new Set(row.sources.flatMap(({ finding }) => finding.occurrences))],
evidence: [...new Map(evidence.map((quote) => [JSON.stringify(quote), quote])).values()],
};
}

332
ui/litellm-dashboard/src/components/lens/model/live.test.ts Normal file
View file

@ -0,0 +1,332 @@
import { describe, expect, it } from "vitest";
import {
analysisModel,
briefReasoning,
checkLabel,
conclusions,
doneLine,
durationLabel,
newestFirst,
nowLine,
inFlight,
grownGroups,
inGroup,
share,
issueCount,
appendPage,
EMPTY_FEED,
shortVerdict,
stripState,
liveJob,
outcome,
providerOf,
polling,
} from "./live";
import type { Job, Review } from "./types";
function review(id: string, overrides: Partial<Review> = {}): Review {
return {
execution_id: id,
trace_id: `trace-${id}`,
agent: "support-bot",
name: id,
spans: [],
reasoning: "",
verdicts: [],
cannot_assess: false,
model: "cerebras/gpt-oss-120b",
duration_ms: 100,
at: "2026-10-03T16:00:00Z",
...overrides,
};
}
const issue = (check: string, summary = check) => ({ check_id: check, kind: "issue" as const, summary });
const pattern = (check: string, summary = check) => ({ check_id: check, kind: "pattern" as const, summary });
describe("provider from model", () => {
it("takes the lowercase prefix before the first slash", () => {
expect(providerOf("Cerebras/gpt-oss-120b")).toBe("cerebras");
expect(providerOf("openrouter/meta/llama")).toBe("openrouter");
});
it("has no provider for a bare model alias", () => {
expect(providerOf("analysis")).toBe("");
expect(providerOf("/weird")).toBe("");
});
it("resolves a bare model name through the model catalog", () => {
const catalog = {
"gpt-5.6": { litellm_provider: "openai" },
"cerebras/gpt-oss-120b": { litellm_provider: "cerebras" },
};
expect(providerOf("gpt-5.6", catalog)).toBe("openai");
expect(providerOf("my-alias", catalog)).toBe("");
expect(analysisModel(["", "gpt-5.6"], catalog)).toBe("gpt-5.6");
});
it("prefers a model that names its provider over a bare alias", () => {
expect(analysisModel(["analysis", "", "cerebras/gpt-oss-120b"])).toBe("cerebras/gpt-oss-120b");
expect(analysisModel(["", "analysis"])).toBe("analysis");
expect(analysisModel([])).toBe("");
});
});
describe("review outcome", () => {
it("is an issue when any verdict is an issue", () => {
expect(outcome(review("a", { verdicts: [pattern("p"), issue("i")] }))).toBe("issue");
});
it("is clear when only patterns are seen", () => {
expect(outcome(review("a", { verdicts: [pattern("p")] }))).toBe("clear");
});
it("is unknown when the model cannot assess, even with an issue", () => {
expect(outcome(review("a", { cannot_assess: true, verdicts: [issue("i")] }))).toBe("unknown");
});
it("keeps the first few sentences of the reasoning", () => {
expect(briefReasoning("One. Two? Three! Four. Five.")).toBe("One. Two? Three!");
expect(briefReasoning("Saw 1.5 percent drop. Fine")).toBe("Saw 1.5 percent drop. Fine");
expect(briefReasoning(" ")).toBe("");
});
});
describe("conclusions", () => {
it("makes one group per check, counting issue traces and noting pattern traces", () => {
const reviews = [
review("a", { verdicts: [pattern("calm"), issue("invented", "first")] }),
review("b", { verdicts: [pattern("calm"), pattern("calm")] }),
review("c", { verdicts: [issue("unhappy_user")] }),
review("d", { verdicts: [issue("invented", "latest"), pattern("invented", "fine")] }),
review("e", { verdicts: [pattern("invented", "fine")] }),
];
const result = conclusions(reviews, [{ id: "invented", instruction: "Invents answers", enabled: true }]);
expect(result.map((c) => [c.checkId, c.count, c.noted, c.issue])).toEqual([
["invented", 2, 1, true],
["unhappy_user", 1, 0, true],
["calm", 0, 2, false],
]);
expect(new Set(result.map((c) => c.key)).size).toBe(result.length);
expect(result[0].label).toBe("Invents answers");
expect(result[0].latest).toBe("latest");
expect(result[1].label).toBe("Unhappy user");
});
it("labels a check by its humanized id when the instruction is long", () => {
const long = "Agent takes a risky action (refund over limit, prod deploy) without required approval";
expect(checkLabel("no_approval", long)).toBe("No approval");
expect(checkLabel("expected_behavior", undefined)).toBe("Expected behavior");
expect(checkLabel("x", "Invents answers")).toBe("Invents answers");
});
it("is empty when nothing was flagged", () => {
expect(conclusions([review("a"), review("b")])).toEqual([]);
});
it("filters traces to a group and lets everything through without one", () => {
const [invented] = conclusions([review("a", { verdicts: [issue("invented")] })]);
expect(inGroup(review("a", { verdicts: [issue("invented")] }), invented.key)).toBe(true);
expect(inGroup(review("b", { verdicts: [pattern("other")] }), invented.key)).toBe(false);
expect(inGroup(review("c"), null)).toBe(true);
});
it("reports which groups gained a trace so they can flash", () => {
const before = conclusions([review("a", { verdicts: [issue("x"), pattern("y")] })]);
const after = conclusions([
review("a", { verdicts: [issue("x"), pattern("y")] }),
review("b", { verdicts: [issue("x"), issue("z")] }),
]);
expect([...grownGroups(before, after)].sort()).toEqual(["x", "z"]);
expect(grownGroups(after, after).size).toBe(0);
});
it("flashes a group that only gained a pattern trace", () => {
const before = conclusions([review("a", { verdicts: [pattern("y")] })]);
const after = conclusions([review("a", { verdicts: [pattern("y")] }), review("b", { verdicts: [pattern("y")] })]);
expect([...grownGroups(before, after)]).toEqual(["y"]);
});
it("gives a bar share bounded to the total", () => {
expect(share(3, 12)).toBe(0.25);
expect(share(5, 0)).toBe(0);
expect(share(9, 4)).toBe(1);
});
});
describe("which job the live run shows", () => {
const job = (id: string, status: Job["status"], reviews: Review[]) =>
({ id, status, reviews: [], reviewed: reviews.length }) as unknown as Job;
it("shows the active job once it has reviews", () => {
expect(liveJob([job("new", "running", [review("a")]), job("old", "completed", [review("b")])])?.id).toBe("new");
});
it("shows an active job before its first review so the wait is explained", () => {
expect(liveJob([job("new", "queued", []), job("old", "completed", [review("b")])])?.id).toBe("new");
});
it("keeps the latest finished job viewable only when it has reviews", () => {
expect(liveJob([job("done", "completed", [review("a")])])?.id).toBe("done");
expect(liveJob([job("done", "completed", []), job("older", "completed", [review("b")])])).toBeUndefined();
expect(liveJob([])).toBeUndefined();
});
});
describe("strip state", () => {
const base = {
status: "running" as Job["status"],
error: "",
stage: "Reading executions",
steps: [] as Job["steps"],
coverage: { selected: 328 } as Job["coverage"],
reviews: [] as Review[],
};
const MODEL = "cerebras/gpt-oss-120b";
it("says what is happening before the first review instead of a silent spinner", () => {
expect(stripState(base, MODEL)).toEqual({
kind: "waiting",
message: "Reading 328 traces with cerebras/gpt-oss-120b…",
});
expect(stripState({ ...base, stage: "Grouping observations" }, "")).toEqual({
kind: "waiting",
message: "Grouping observations…",
});
expect(stripState({ ...base, status: "queued" }, MODEL, "No worker connected.")).toEqual({
kind: "waiting",
message: "No worker connected.",
});
});
it("shows the job error plainly when the run failed", () => {
expect(stripState({ ...base, status: "failed", error: "Anthropic: credit balance too low" }, MODEL)).toEqual({
kind: "failed",
message: "Anthropic: credit balance too low",
});
expect(stripState({ ...base, status: "failed" }, MODEL)).toEqual({
kind: "failed",
message: "The investigation failed",
});
});
it("surfaces a model error step while still running with nothing reviewed", () => {
const steps = [{ kind: "error", label: "Model call failed: 400 credit balance too low" }] as Job["steps"];
expect(stripState({ ...base, steps }, MODEL)).toEqual({
kind: "failed",
message: "Model call failed: 400 credit balance too low",
});
expect(stripState({ ...base, steps, reviews: [review("a")] }, MODEL).kind).toBe("reviewing");
});
it("is done for finished runs and reviewing once reviews arrive", () => {
expect(stripState({ ...base, status: "completed" }, MODEL).kind).toBe("done");
expect(stripState({ ...base, reviews: [review("a")] }, MODEL).kind).toBe("reviewing");
});
});
describe("issue count", () => {
const issueReview = review("x", { verdicts: [issue("i")] });
it("uses the job's issue findings once the run completes", () => {
const findings = [{ kind: "issue" }, { kind: "pattern" }, { kind: "issue" }] as Job["findings"];
const completed = { status: "completed" as const, findings, reviews: [issueReview], reviewed: 328 };
expect(issueCount(completed)).toEqual({ count: 2, scope: "findings" });
});
it("labels counts honestly when only the last reviews are kept", () => {
const reviews = [issueReview, review("y"), issueReview];
const running = { status: "running" as const, findings: null, reviews };
expect(issueCount({ ...running, reviewed: 3 })).toEqual({ count: 2, scope: "" });
expect(issueCount({ ...running, reviewed: 120 })).toEqual({
count: 2,
scope: "in last 3 reviewed",
});
});
});
describe("incremental reviews", () => {
const at = (id: string, minute: number) => review(id, { at: `2026-10-03T16:${String(minute).padStart(2, "0")}:00Z` });
it("appends pages in arrival order, not by time, and advances the cursor to the reviewed count", () => {
const first = appendPage(EMPTY_FEED, { reviews: [at("a", 5), at("b", 2)], reviewed: 2 });
expect(first).toEqual({ reviews: [at("a", 5), at("b", 2)], cursor: 2 });
const second = appendPage(first, { reviews: [at("c", 1)], reviewed: 3 });
expect(second.reviews.map((r) => r.execution_id)).toEqual(["a", "b", "c"]);
expect(second.cursor).toBe(3);
});
it("keeps the same feed for an empty page and never duplicates a re-sent review", () => {
const feed = appendPage(EMPTY_FEED, { reviews: [at("a", 1)], reviewed: 1 });
expect(appendPage(feed, { reviews: [], reviewed: 1 })).toBe(feed);
expect(appendPage(feed, { reviews: [at("a", 1)], reviewed: 1 })).toBe(feed);
});
it("caps how many reviews are kept, dropping the oldest", () => {
const many = Array.from({ length: 250 }, (_, n) => review(`r${n}`));
const feed = appendPage(EMPTY_FEED, { reviews: many, reviewed: 250 });
expect(feed.reviews).toHaveLength(200);
expect(feed.reviews[0].execution_id).toBe("r50");
expect(feed.cursor).toBe(250);
});
it("keeps polling a finished run until every review it reported has arrived", () => {
const fetched = { reviews: [], cursor: 28 };
expect(polling({ status: "running", reviewed: 28 }, fetched)).toBe(true);
expect(polling({ status: "completed", reviewed: 30 }, fetched)).toBe(true);
expect(polling({ status: "completed", reviewed: 30 }, { ...fetched, cursor: 30 })).toBe(false);
});
});
describe("short verdict", () => {
it("prefers the issue, otherwise says no issues or not enough evidence", () => {
expect(shortVerdict(review("a", { verdicts: [pattern("p"), issue("i", "made it up")] }))).toBe("made it up");
expect(shortVerdict(review("a", { verdicts: [pattern("p")] }))).toBe("no issues");
expect(shortVerdict(review("a", { cannot_assess: true }))).toBe("not enough evidence");
});
});
describe("honest live list", () => {
it("lists completed reviews newest first in the order they finished, never re-sorted by time", () => {
const reviews = [
review("a", { at: "2026-10-03T16:05:00Z" }),
review("b", { at: "2026-10-03T16:01:00Z" }),
review("c"),
];
expect(newestFirst(reviews, 10).map((r) => r.execution_id)).toEqual(["c", "b", "a"]);
expect(newestFirst(reviews, 2).map((r) => r.execution_id)).toEqual(["c", "b"]);
expect(reviews.map((r) => r.execution_id)).toEqual(["a", "b", "c"]);
});
it("says how many traces are in flight and how many are done", () => {
const job = (reviewed: number, selected: number) => ({ reviewed, coverage: { selected } }) as unknown as Job;
expect(nowLine(job(18, 30), 4)).toBe("18 of 30 · 4 in flight");
expect(nowLine(job(30, 30), 0)).toBe("30 of 30");
expect(nowLine(job(3, 0), 1)).toBe("3 done · 1 in flight");
});
it("only shows in-flight traces while the job runs", () => {
const item = { execution_id: "x", trace_id: "t", agent: "bot", started_at: "2026-10-03T16:00:00Z" };
const job = (status: Job["status"]) => ({ status, reading: [item] }) as unknown as Job;
expect(inFlight(job("running"))).toEqual([item]);
expect(inFlight(job("completed"))).toEqual([]);
expect(inFlight({ status: "running" } as Job)).toEqual([]);
});
it("sums up a finished run from when reading started", () => {
const job = {
reviewed: 30,
created_at: "2026-10-03T16:00:00Z",
finished_at: "2026-10-03T16:00:41Z",
steps: [{ kind: "stage", label: "Reading executions", at: "2026-10-03T16:00:10Z" }],
} as unknown as Job;
expect(doneLine(job)).toBe("Reviewed 30 traces in 31s with");
expect(doneLine({ ...job, reviewed: 1, finished_at: null })).toBe("Reviewed 1 trace with");
});
it("formats review time", () => {
expect(durationLabel(420)).toBe("420ms");
expect(durationLabel(1420)).toBe("1.4s");
expect(durationLabel(83_000)).toBe("1m 23s");
});
});

198
ui/litellm-dashboard/src/components/lens/model/live.ts Normal file
View file

@ -0,0 +1,198 @@
import { durationText } from "./format";
import { activeJob, isActive, readingStart } from "./status";
import type { InFlight, Job, Review, ReviewVerdict, Settings } from "./types";
export type Outcome = "issue" | "clear" | "unknown";
export interface Conclusion {
key: string;
checkId: string;
label: string;
latest: string;
count: number;
noted: number;
issue: boolean;
}
export function liveJob(jobs: readonly Job[]): Job | undefined {
const active = activeJob(jobs);
if (active) return active;
const latest = jobs[0];
return latest && latest.reviewed > 0 ? latest : undefined;
}
export function reviewKey(review: Pick<Review, "execution_id" | "at">): string {
return `${review.execution_id}@${review.at}`;
}
export type ProviderCatalog = Readonly<Record<string, { litellm_provider?: string } | undefined>>;
export function providerOf(model: string, catalog: ProviderCatalog = {}): string {
const known = catalog[model]?.litellm_provider;
if (known) return known.toLowerCase();
const slash = model.indexOf("/");
return slash > 0 ? model.slice(0, slash).toLowerCase() : "";
}
export function outcome(review: Pick<Review, "cannot_assess" | "verdicts">): Outcome {
if (review.cannot_assess) return "unknown";
return review.verdicts.some((v) => v.kind === "issue") ? "issue" : "clear";
}
export function shortVerdict(review: Pick<Review, "cannot_assess" | "verdicts">): string {
const issue = review.verdicts.find((v) => v.kind === "issue");
if (issue) return issue.summary;
return review.cannot_assess ? "not enough evidence" : "no issues";
}
export type StripState =
| { kind: "failed"; message: string }
| { kind: "waiting"; message: string }
| { kind: "reviewing" }
| { kind: "done" };
export function stripState(
job: Pick<Job, "status" | "error" | "stage" | "steps" | "coverage" | "reviews">,
model: string,
queued = "Queued, waiting for a worker to pick this up",
): StripState {
if (job.status === "failed") return { kind: "failed", message: job.error || "The investigation failed" };
const stepError = job.steps.findLast((step) => step.kind === "error");
if (stepError && !job.reviews.length) return { kind: "failed", message: stepError.label };
if (job.status === "completed" || job.status === "cancelled") return { kind: "done" };
if (job.reviews.length) return { kind: "reviewing" };
if (job.status === "queued") return { kind: "waiting", message: queued };
const { selected } = job.coverage;
const using = model ? ` with ${model}` : "";
if (job.stage === "Reading executions" && selected) {
return { kind: "waiting", message: `Reading ${selected} ${selected === 1 ? "trace" : "traces"}${using}…` };
}
return { kind: "waiting", message: `${job.stage || "Starting"}${using}…` };
}
export interface IssueCount {
count: number;
scope: string;
}
export function issueCount(job: Pick<Job, "status" | "findings" | "reviews" | "reviewed">): IssueCount {
const findings = job.findings?.filter((f) => f.kind === "issue").length;
if (job.status === "completed" && findings !== undefined) return { count: findings, scope: "findings" };
const count = job.reviews.filter((r) => outcome(r) === "issue").length;
if (job.reviewed > job.reviews.length) return { count, scope: `in last ${job.reviews.length} reviewed` };
return { count, scope: "" };
}
const KEPT_REVIEWS = 200;
export interface ReviewFeed {
reviews: readonly Review[];
cursor: number;
}
export const EMPTY_FEED: ReviewFeed = { reviews: [], cursor: 0 };
export function appendPage(feed: ReviewFeed, page: { reviews: readonly Review[]; reviewed: number }): ReviewFeed {
const added = unseen(page.reviews, new Set(feed.reviews.map(reviewKey)));
if (page.reviewed === feed.cursor && !added.length) return feed;
return { reviews: [...feed.reviews, ...added].slice(-KEPT_REVIEWS), cursor: page.reviewed };
}
export function polling(job: Pick<Job, "status" | "reviewed">, feed: ReviewFeed): boolean {
return isActive(job) || feed.cursor < job.reviewed;
}
export function unseen(reviews: readonly Review[], seen: ReadonlySet<string>): Review[] {
return reviews.filter((review) => !seen.has(reviewKey(review)));
}
export function analysisModel(candidates: readonly string[], catalog: ProviderCatalog = {}): string {
return candidates.find((model) => providerOf(model, catalog)) ?? candidates.find(Boolean) ?? "";
}
const SHORT_LABEL = 48;
function humanize(checkId: string): string {
const words = checkId.replace(/[_-]+/g, " ").trim();
return words ? words[0].toUpperCase() + words.slice(1) : checkId;
}
export function checkLabel(checkId: string, instruction: string | undefined): string {
return instruction && instruction.length <= SHORT_LABEL ? instruction : humanize(checkId);
}
function verdictsByCheck(review: Pick<Review, "verdicts">): Map<string, ReviewVerdict> {
const ranked = [...review.verdicts].sort((a, b) => Number(a.kind === "issue") - Number(b.kind === "issue"));
return new Map(ranked.map((verdict) => [verdict.check_id, verdict]));
}
export function conclusions(reviews: readonly Review[], checks: Settings["checks"] = []): Conclusion[] {
const instructions = new Map(checks.map((check) => [check.id, check.instruction]));
const grouped = reviews.reduce((groups, review) => {
return [...verdictsByCheck(review).values()].reduce((next, verdict) => {
const prior = next.get(verdict.check_id);
const issue = verdict.kind === "issue";
const merged: Conclusion = {
key: verdict.check_id,
checkId: verdict.check_id,
label: checkLabel(verdict.check_id, instructions.get(verdict.check_id)),
latest: issue || !prior ? verdict.summary : prior.latest,
count: (prior?.count ?? 0) + Number(issue),
noted: (prior?.noted ?? 0) + Number(!issue),
issue: (prior?.issue ?? false) || issue,
};
return new Map(next).set(verdict.check_id, merged);
}, groups);
}, new Map<string, Conclusion>());
return [...grouped.values()].sort((a, b) => b.count - a.count || b.noted - a.noted);
}
export function share(count: number, total: number): number {
return total > 0 ? Math.min(1, count / total) : 0;
}
export function inGroup(review: Pick<Review, "verdicts">, checkId: string | null): boolean {
return checkId === null || review.verdicts.some((verdict) => verdict.check_id === checkId);
}
const BRIEF_SENTENCES = 3;
export function briefReasoning(reasoning: string): string {
const sentences = reasoning.trim().split(/(?<=[.!?])\s+/);
return sentences.slice(0, BRIEF_SENTENCES).join(" ").trim();
}
export function grownGroups(before: readonly Conclusion[], after: readonly Conclusion[]): Set<string> {
const prior = new Map(before.map((group) => [group.key, group.count + group.noted]));
return new Set(
after.filter((group) => group.count + group.noted > (prior.get(group.key) ?? 0)).map((group) => group.key),
);
}
export function newestFirst(reviews: readonly Review[], limit: number): Review[] {
return [...reviews].reverse().slice(0, limit);
}
export function inFlight(job: Pick<Job, "status" | "reading">): readonly InFlight[] {
return job.status === "running" ? job.reading ?? [] : [];
}
export function nowLine(job: Pick<Job, "coverage" | "reviewed">, reading: number): string {
const { selected } = job.coverage;
const done = selected ? `${Math.min(job.reviewed, selected)} of ${selected}` : `${job.reviewed} done`;
return reading ? `${done} · ${reading} in flight` : done;
}
export function durationLabel(ms: number): string {
if (ms < 1000) return `${Math.max(0, Math.round(ms))}ms`;
return ms < 60_000
? `${(ms / 1000).toFixed(1)}s`
: `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
}
export function doneLine(job: Pick<Job, "reviewed" | "created_at" | "finished_at" | "steps">): string {
const end = job.finished_at ? Date.parse(job.finished_at) : Number.NaN;
const seconds = Math.round((end - Date.parse(readingStart(job))) / 1000);
const took = Number.isFinite(seconds) && seconds >= 0 ? ` in ${durationText(seconds)}` : "";
return `Reviewed ${job.reviewed} ${job.reviewed === 1 ? "trace" : "traces"}${took} with`;
}

2
ui/litellm-dashboard/src/components/lens/model/progress.test.ts
View file

@ -25,6 +25,8 @@ const coverage: Job["coverage"] = {
const job: Job = {
assessments: [],
steps: [],
reviews: [],
reviewed: 0,
trigger: "schedule",
coverage,
attempts: 0,

127
ui/litellm-dashboard/src/components/lens/model/stage.test.ts Normal file
View file

@ -0,0 +1,127 @@
import { describe, expect, it } from "vitest";
import {
holdMs,
laneText,
releasedReviews,
stageTick,
startStage,
stepStage,
typedChars,
type StageInput,
} from "./stage";
import type { Review } from "./types";
function review(id: string, reasoning = "Checked the tool output. It matched."): Review {
return {
execution_id: id,
trace_id: `trace-${id}`,
agent: "support-bot",
name: id,
spans: [],
reasoning,
verdicts: [],
cannot_assess: false,
model: "cerebras/gpt-oss-120b",
duration_ms: 900,
at: "2026-10-03T16:00:00Z",
};
}
const reading = (id: string) => ({
execution_id: id,
trace_id: `trace-${id}`,
agent: "bot",
started_at: "2026-10-03T16:00:00Z",
});
const input = (overrides: Partial<StageInput>): StageInput => ({
reading: [],
reviews: [],
now: 0,
slots: 4,
running: true,
charMs: 3,
...overrides,
});
const keys = (stage: ReturnType<typeof startStage>) => stage.lanes.map((lane) => lane.key);
describe("lane text", () => {
it("keeps the first two sentences and caps the length", () => {
expect(laneText("One. Two. Three.")).toBe("One. Two.");
const long = laneText(`${"word ".repeat(80)}end.`);
expect(long.length).toBeLessThanOrEqual(220);
expect(long.endsWith("…")).toBe(true);
});
it("types at the given speed and holds the verdict after typing", () => {
expect(typedChars({ landedAt: 1000 }, 1030, 3)).toBe(10);
expect(typedChars({ landedAt: null }, 1030, 3)).toBe(0);
expect(typedChars({ landedAt: 1000 }, 1000, 0)).toBe(Number.MAX_SAFE_INTEGER);
expect(holdMs(review("a", "Short."), 3)).toBe(6 * 3 + 1200);
});
});
describe("now reading stage", () => {
it("does not replay reviews that already existed when the view opened", () => {
const stage = startStage([review("old")]);
const next = stepStage(stage, input({ reviews: [review("old")] }));
expect(next).toBe(stage);
expect(releasedReviews([review("old")], next).map((r) => r.execution_id)).toEqual(["old"]);
});
it("shows real in-flight traces, lands the review in the same lane, then hands it to the list", () => {
const opened = stepStage(startStage([]), input({ reading: [reading("x"), reading("y")] }));
expect(keys(opened)).toEqual(["x", "y"]);
expect(opened.lanes.every((lane) => lane.review === null)).toBe(true);
const landed = stepStage(opened, input({ reading: [reading("y")], reviews: [review("x")], now: 500 }));
expect(keys(landed)).toEqual(["x", "y"]);
expect(landed.lanes[0]).toMatchObject({ landedAt: 500 });
expect(landed.released.has("x")).toBe(false);
const hold = holdMs(review("x"), 3);
const still = stepStage(landed, input({ reading: [reading("y")], reviews: [review("x")], now: 500 + hold - 1 }));
expect(keys(still)).toEqual(["x", "y"]);
const handed = stepStage(still, input({ reading: [reading("y")], reviews: [review("x")], now: 500 + hold }));
expect(keys(handed)).toEqual(["y"]);
expect(handed.released.has("x")).toBe(true);
});
it("gives newly arrived reviews a lane when the worker does not report what it is reading", () => {
const stage = stepStage(startStage([]), input({ reviews: [review("a"), review("b")], now: 10 }));
expect(keys(stage)).toEqual(["a", "b"]);
expect(stage.lanes.every((lane) => lane.landedAt === 10)).toBe(true);
});
it("never lets a burst back up: extra reviews go straight to the list, oldest first", () => {
const burst = ["a", "b", "c", "d", "e", "f"].map((id) => review(id));
const stage = stepStage(startStage([]), input({ reviews: burst, slots: 2 }));
expect(keys(stage)).toEqual(["e", "f"]);
expect([...stage.released].sort()).toEqual(["a", "b", "c", "d"]);
});
it("drops a lane whose trace vanished without a review after a grace period", () => {
const opened = stepStage(startStage([]), input({ reading: [reading("x")] }));
const gone = stepStage(opened, input({ now: 1000 }));
expect(keys(gone)).toEqual(["x"]);
expect(keys(stepStage(gone, input({ now: 5000 })))).toEqual([]);
expect(stepStage(gone, input({ now: 5000 })).released.has("x")).toBe(false);
});
it("releases everything once the run stops", () => {
const opened = stepStage(startStage([]), input({ reading: [reading("x")], reviews: [review("a")] }));
const stopped = stepStage(opened, input({ running: false, reviews: [review("a"), review("x")] }));
expect(stopped.lanes).toEqual([]);
expect([...stopped.released].sort()).toEqual(["a", "x"]);
});
});
describe("stage tick", () => {
it("ticks fast only while reasoning is typing, slowly for live timers, and stops when idle", () => {
const landed = stepStage(startStage([]), input({ reviews: [review("a")], now: 0, charMs: 3 }));
expect(stageTick(landed, true, 1, 3)).toBe(30);
expect(stageTick(landed, true, 10_000, 3)).toBe(500);
expect(stageTick(startStage([]), true, 0, 3)).toBe(500);
expect(stageTick(startStage([]), false, 0, 3)).toBeNull();
});
});

135
ui/litellm-dashboard/src/components/lens/model/stage.ts Normal file
View file

@ -0,0 +1,135 @@
import type { InFlight } from "./types";
import type { Review } from "./types";
export interface Lane {
key: string;
agent: string;
traceId: string;
startedAt: number;
review: Review | null;
landedAt: number | null;
goneAt: number | null;
}
export interface Stage {
lanes: readonly Lane[];
released: ReadonlySet<string>;
}
export interface StageInput {
reading: readonly InFlight[];
reviews: readonly Review[];
now: number;
slots: number;
running: boolean;
charMs: number;
}
const VERDICT_HOLD_MS = 1200;
const GONE_MS = 4000;
const LANE_CHARS = 220;
const LANE_SENTENCES = 2;
export function laneText(reasoning: string): string {
const brief = reasoning
.trim()
.split(/(?<=[.!?])\s+/)
.slice(0, LANE_SENTENCES)
.join(" ");
return brief.length > LANE_CHARS ? `${brief.slice(0, LANE_CHARS - 1).trimEnd()}…` : brief;
}
export function holdMs(review: Pick<Review, "reasoning">, charMs: number): number {
return laneText(review.reasoning).length * charMs + VERDICT_HOLD_MS;
}
export function typedChars(lane: Pick<Lane, "landedAt">, now: number, charMs: number): number {
if (lane.landedAt === null) return 0;
return charMs > 0 ? Math.floor((now - lane.landedAt) / charMs) : Number.MAX_SAFE_INTEGER;
}
export function startStage(reviews: readonly Review[]): Stage {
return { lanes: [], released: new Set(reviews.map((review) => review.execution_id)) };
}
function settle(lane: Lane, reviews: ReadonlyMap<string, Review>, live: ReadonlySet<string>, now: number): Lane {
const review = lane.review ?? reviews.get(lane.key) ?? null;
const landedAt = lane.landedAt ?? (review ? now : null);
const goneAt = review || live.has(lane.key) ? null : lane.goneAt ?? now;
if (review === lane.review && landedAt === lane.landedAt && goneAt === lane.goneAt) return lane;
return { ...lane, review, landedAt, goneAt };
}
function expired(lane: Lane, now: number, charMs: number): boolean {
if (lane.review && lane.landedAt !== null) return now - lane.landedAt >= holdMs(lane.review, charMs);
return lane.goneAt !== null && now - lane.goneAt >= GONE_MS;
}
function landedLane(review: Review, now: number): Lane {
return {
key: review.execution_id,
agent: review.agent || review.name,
traceId: review.trace_id,
startedAt: now,
review,
landedAt: now,
goneAt: null,
};
}
function readingLane(item: InFlight): Lane {
return {
key: item.execution_id,
agent: item.agent,
traceId: item.trace_id,
startedAt: Date.parse(item.started_at),
review: null,
landedAt: null,
goneAt: null,
};
}
export function stepStage(stage: Stage, input: StageInput): Stage {
const { now, charMs } = input;
if (!input.running) {
if (!stage.lanes.length && input.reviews.every((r) => stage.released.has(r.execution_id))) return stage;
return { lanes: [], released: new Set([...stage.released, ...input.reviews.map((r) => r.execution_id)]) };
}
const reviews = new Map(input.reviews.map((review) => [review.execution_id, review]));
const live = new Set(input.reading.map((item) => item.execution_id));
const settled = stage.lanes.map((lane) => settle(lane, reviews, live, now));
const done = settled.filter((lane) => expired(lane, now, charMs));
const kept = settled.filter((lane) => !done.includes(lane));
const taken = new Set([...settled.map((lane) => lane.key), ...stage.released]);
const fresh = input.reviews.filter((review) => !taken.has(review.execution_id));
const free = Math.max(0, input.slots - kept.length);
const landing = free ? fresh.slice(-free) : [];
const overflow = fresh.slice(0, fresh.length - landing.length);
const room = free - landing.length;
const waiting = input.reading.filter((item) => !taken.has(item.execution_id) && !reviews.has(item.execution_id));
const starting = room > 0 ? waiting.slice(0, room) : [];
const nothingMoved = !done.length && !fresh.length && !starting.length;
if (nothingMoved && settled.every((lane, n) => lane === stage.lanes[n])) return stage;
const releasedNow = [...done.filter((lane) => lane.review), ...overflow].map((item) =>
"key" in item ? item.key : item.execution_id,
);
return {
lanes: [...kept, ...landing.map((review) => landedLane(review, now)), ...starting.map(readingLane)],
released: releasedNow.length ? new Set([...stage.released, ...releasedNow]) : stage.released,
};
}
const TYPING_TICK_MS = 30;
const CLOCK_TICK_MS = 500;
export function stageTick(stage: Stage, running: boolean, now: number, charMs: number): number | null {
const typing = stage.lanes.some(
(lane) => lane.review && typedChars(lane, now, charMs) < laneText(lane.review.reasoning).length,
);
if (typing) return TYPING_TICK_MS;
return running || stage.lanes.length ? CLOCK_TICK_MS : null;
}
export function releasedReviews(reviews: readonly Review[], stage: Stage): Review[] {
return reviews.filter((review) => stage.released.has(review.execution_id));
}

110
ui/litellm-dashboard/src/components/lens/model/status.test.ts
View file

@ -1,4 +1,14 @@
import { activeJob, investigationActivity, listPollInterval, nextCheckStatus, workerConnected } from "./status";
import {
activeJob,
investigationActivity,
listPollInterval,
nextCheckStatus,
queueReason,
secondsToFinishReading,
queueReasonText,
workerConnected,
workerTaskText,
} from "./status";
import { describe, expect, it } from "vitest";
import type { Job, Lens, LensList } from "./types";
@ -19,6 +29,8 @@ const coverage: Job["coverage"] = {
const job: Job = {
assessments: [],
steps: [],
reviews: [],
reviewed: 0,
trigger: "schedule",
coverage,
attempts: 0,
@ -71,6 +83,84 @@ describe("Worker readiness", () => {
});
});
describe("Why a run is queued", () => {
const now = Date.parse("2026-10-01T12:00:00Z");
const worker: LensList["workers"][number] = {
id: "w1",
name: "Worker",
revoked: false,
analysis_key_id: "key",
last_seen: "2026-10-01T11:59:59Z",
scope: { all_teams: true, api_key_hash: "", team_id: "" },
};
const queued = { ...job, id: "mine", status: "queued" as const, worker_id: null, created_at: "2026-10-01T11:59:48Z" };
const running = (lensId: string, workerId: string, reviewed: number, status: Job["status"] = "running") => ({
id: lensId,
settings: { ...job.settings, name: lensId },
jobs: [
{
...job,
id: `job-${lensId}`,
status,
worker_id: workerId,
reviewed,
created_at: "2026-10-01T11:58:20Z",
coverage: { ...coverage, selected: 328 },
},
],
});
const three = [running("swarm", "w1", 200), running("billing", "w1", 300), running("research", "w1", 100)];
it("lists what a busy worker is running and when this run should start", () => {
const reason = queueReason(queued, three, [worker], now);
expect(reason.kind).toBe("busy");
if (reason.kind !== "busy") return;
expect(reason.tasks.map(workerTaskText)).toEqual([
"swarm · Reading executions · 200 of 328 traces",
"billing · Reading executions · 300 of 328 traces",
"research · Reading executions · 100 of 328 traces",
]);
expect(reason.tasks[0].lensId).toBe("swarm");
expect(reason.startsIn).toBe(10);
expect(queueReasonText(reason)).toBe("Worker is busy with 3 investigations · starts in ~10s");
});
it("says no worker is connected when every heartbeat is stale or revoked", () => {
const stale = { ...worker, last_seen: "2026-10-01T11:50:00Z" };
expect(queueReason(queued, three, [stale, { ...worker, revoked: true }], now)).toEqual({ kind: "no_worker" });
expect(queueReasonText({ kind: "no_worker" })).toBe("No worker connected. Start one from Connect worker.");
});
it("is picking up when nothing else is running, counting seconds waited", () => {
const done = [running("swarm", "w1", 328, "completed")];
expect(queueReason(queued, done, [worker], now)).toEqual({ kind: "starting", seconds: 12, tasks: [] });
expect(queueReasonText({ kind: "starting", seconds: 12, tasks: [] })).toBe("Picking up… 12s");
});
it("gives the worker a moment to pick up before calling it busy", () => {
const fresh = { ...queued, created_at: "2026-10-01T11:59:57Z" };
expect(queueReason(fresh, three, [worker], now).kind).toBe("starting");
});
it("only counts work on the worker this run is assigned to, and never itself", () => {
const free = { ...worker, id: "w2" };
const elsewhere = [running("swarm", "w2", 200)];
expect(queueReason({ ...queued, worker_id: "w1" }, elsewhere, [worker, free], now).kind).toBe("starting");
const self = {
id: "mine",
settings: job.settings,
jobs: [{ ...queued, status: "running" as const, worker_id: "w1" }],
};
expect(queueReason(queued, [self], [worker], now).kind).toBe("starting");
});
it("has no start estimate before any busy run has a rate", () => {
const reason = queueReason(queued, [running("swarm", "w1", 0)], [worker], now);
expect(reason).toMatchObject({ kind: "busy", startsIn: null });
expect(queueReasonText(reason)).toBe("Worker is busy with 1 investigation");
});
});
const lens: Lens = {
version: 0,
spent: 0,
@ -157,3 +247,21 @@ describe("activeJob", () => {
expect(activeJob([done])).toBeUndefined();
});
});
describe("time left reading", () => {
const started = {
created_at: "2026-10-03T16:00:00Z",
steps: [{ kind: "stage", label: "Reading executions", at: "2026-10-03T16:00:20Z" }] as Job["steps"],
coverage: { selected: 328 } as Job["coverage"],
};
const now = Date.parse("2026-10-03T16:01:00Z");
it("projects the remaining traces at the rate since reading started", () => {
expect(secondsToFinishReading({ ...started, reviewed: 80 }, now)).toBe(124);
});
it("has no estimate before the first review or once every trace is read", () => {
expect(secondsToFinishReading({ ...started, reviewed: 0 }, now)).toBeNull();
expect(secondsToFinishReading({ ...started, reviewed: 328 }, now)).toBeNull();
});
});

102
ui/litellm-dashboard/src/components/lens/model/status.ts
View file

@ -5,8 +5,108 @@ export function workerConnected(worker: LensList["workers"][number], now = Date.
return !worker.revoked && !!worker.analysis_key_id && now - Date.parse(worker.last_seen) < 120000;
}
export function readingStart(job: Pick<Job, "steps" | "created_at">): string {
return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at;
}
export function secondsToFinishReading(
job: Pick<Job, "steps" | "created_at" | "reviewed" | "coverage">,
now: number,
): number | null {
const remaining = job.coverage.selected - job.reviewed;
const elapsed = (now - Date.parse(readingStart(job))) / 1000;
if (remaining <= 0 || job.reviewed <= 0 || elapsed <= 0) return null;
return Math.ceil(remaining / (job.reviewed / elapsed));
}
const PICKUP_GRACE_SECONDS = 6;
export interface WorkerTask {
lensId: string;
name: string;
stage: string;
reviewed: number;
selected: number;
secondsLeft: number | null;
}
export type QueueReason =
| { kind: "busy"; tasks: readonly WorkerTask[]; startsIn: number | null }
| { kind: "no_worker" }
| { kind: "starting"; seconds: number; tasks: readonly WorkerTask[] };
type QueuedJob = Pick<Job, "id" | "worker_id" | "created_at">;
function workerTasks(
job: QueuedJob,
lenses: readonly Pick<Lens, "id" | "jobs" | "settings">[],
workerIds: ReadonlySet<string>,
now: number,
) {
return lenses.flatMap((lens) =>
lens.jobs
.filter((other) => other.status === "running" && other.id !== job.id && workerIds.has(other.worker_id ?? ""))
.map((other) => ({
workerId: other.worker_id ?? "",
task: {
lensId: lens.id,
name: lens.settings.name,
stage: other.stage,
reviewed: other.reviewed,
selected: other.coverage.selected,
secondsLeft: secondsToFinishReading(other, now),
},
})),
);
}
export function queueReason(
job: QueuedJob,
lenses: readonly Pick<Lens, "id" | "jobs" | "settings">[],
workers: readonly LensList["workers"][number][],
now: number,
): QueueReason {
const connected = workers.filter((worker) => workerConnected(worker, now)).map((worker) => worker.id);
if (!connected.length) return { kind: "no_worker" };
const eligible = new Set(job.worker_id ? [job.worker_id] : connected);
const running = workerTasks(job, lenses, eligible, now);
const tasks = running.map((entry) => entry.task);
const seconds = Math.max(0, Math.floor((now - Date.parse(job.created_at)) / 1000));
if (!tasks.length || seconds < PICKUP_GRACE_SECONDS) return { kind: "starting", seconds, tasks };
const estimates = tasks.flatMap((task) => (task.secondsLeft === null ? [] : [task.secondsLeft]));
return { kind: "busy", tasks, startsIn: estimates.length ? Math.min(...estimates) : null };
}
function waitLabel(seconds: number): string {
if (seconds < 60) return `~${Math.max(5, Math.ceil(seconds / 5) * 5)}s`;
return `~${Math.ceil(seconds / 60)}m`;
}
export function queueReasonText(reason: QueueReason): string {
switch (reason.kind) {
case "busy": {
const count = reason.tasks.length;
const when = reason.startsIn === null ? "" : ` · starts in ${waitLabel(reason.startsIn)}`;
return `Worker is busy with ${count} ${count === 1 ? "investigation" : "investigations"}${when}`;
}
case "no_worker":
return "No worker connected. Start one from Connect worker.";
case "starting":
return `Picking up… ${reason.seconds}s`;
}
}
export function workerTaskText(task: WorkerTask): string {
const progress = task.selected ? ` · ${task.reviewed} of ${task.selected} traces` : "";
return `${task.name} · ${task.stage || "starting"}${progress}`;
}
export function isActive(job: Pick<Job, "status">): boolean {
return job.status === "queued" || job.status === "running";
}
export function activeJob(jobs: readonly Job[]): Job | undefined {
return jobs.find((job) => job.status === "queued" || job.status === "running");
return jobs.find(isActive);
}
export function hasActiveJob(jobs: readonly Job[]): boolean {

6
ui/litellm-dashboard/src/components/lens/model/types.ts
View file

@ -33,6 +33,12 @@ export type ActivitySelection = Pick<Settings, "source"> &
export type Worker = LensList["workers"][number];
export type Review = components["schemas"]["Review"];
export type InFlight = components["schemas"]["InFlight"];
export type ReviewVerdict = components["schemas"]["ReviewVerdict"];
export interface RunWindow {
agent_name?: string;
start?: string;

23
ui/litellm-dashboard/src/components/lens/route.ts
View file

@ -4,7 +4,12 @@ import { parseAsBoolean, parseAsString, parseAsStringLiteral, useQueryStates } f
import { useCallback } from "react";
import { OPEN_TRACE_PARSERS, RUN_FILTER_PARSERS } from "@/components/lens/traces/routing";
export const LENS_TABS = { traces: "Traces", investigations: "Investigations", settings: "Settings" } as const;
export const LENS_TABS = {
traces: "Traces",
findings: "Findings",
investigations: "Investigations",
settings: "Settings",
} as const;
export type LensTab = keyof typeof LENS_TABS;
const lensTabs = Object.keys(LENS_TABS) as LensTab[];
@ -23,6 +28,11 @@ const LENS_PARSERS = {
const ISSUE_PARSERS = { issue: parseAsString };
const INBOX_PARSERS = {
inbox_agent: parseAsString.withDefault("all"),
priority: parseAsStringLiteral(["all", "high", "medium", "low"]).withDefault("all"),
};
const LIST_PARSERS = { search: parseAsString.withDefault("") };
const RESULT_PARSERS = {
@ -44,6 +54,7 @@ const SESSION_PARSERS = {
...RUN_FILTER_PARSERS,
...ISSUE_PARSERS,
...LIST_PARSERS,
...INBOX_PARSERS,
...RESULT_PARSERS,
...DIALOG_PARSERS,
};
@ -177,3 +188,13 @@ export function useEvidenceRoute() {
const current: EvidenceRef | null = evidence === null ? null : { id: evidence, span: evidence_span ?? "" };
return { evidence: current, setEvidence };
}
export function useInboxFilters() {
const [{ inbox_agent, priority }, setParams] = useQueryStates(INBOX_PARSERS);
return {
agent: inbox_agent,
priority,
setAgent: (agent: string) => void setParams({ inbox_agent: agent }),
setPriority: (priority: "all" | "high" | "medium" | "low") => void setParams({ priority }),
};
}

72
ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx
View file

@ -1,6 +1,8 @@
"use client";
import moment from "moment";
import { RefreshCw } from "lucide-react";
import { traceAgentNames } from "../utils";
import { useMemo, useState } from "react";
import { filterRuns } from "./runSearch/runQuery";
@ -94,7 +96,7 @@ export function AgentTracesSection({
}: AgentTracesSectionProps) {
const live = useTracesLive();
const { trace: openTrace, openTrace: openRun, selection, fullScreen, setFullScreen } = useOpenTraceRouting();
const { query, setQuery } = useRunFilterRouting();
const { query, setQuery, agent, status } = useRunFilterRouting();
const [showSetup, setShowSetup] = useState(false);
const [zoom, setZoom] = useZoomRouting();
const [rangeChanged, setRangeChanged] = useState(false);
@ -115,7 +117,10 @@ export function AgentTracesSection({
() => ({ startMs: traceWindowStartMs(startTime, endTime, isCustomDate, endMs), endMs }),
[startTime, endTime, isCustomDate, endMs],
);
const filtered = useMemo(() => filterRuns(traces.traces, query), [traces.traces, query]);
const filtered = useMemo(
() => filterRuns(traces.traces, query, { agent, status }),
[traces.traces, query, agent, status],
);
const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
const runRefs = useMemo(() => runs.map(traceRefOf), [runs]);
@ -170,7 +175,7 @@ export function AgentTracesSection({
fullScreen={fullScreen}
onFullScreenChange={setFullScreen}
>
<div className="flex min-h-[560px] flex-1 flex-col overflow-hidden bg-card">
<div className="flex min-h-0 flex-1 flex-col overflow-hidden bg-card">
{checkHistory && <TraceHistoryError history={history} />}
<TracesReceived received={setup.received} />
<Inspector.Panel label="Trace details" testId="run-drawer">
@ -186,17 +191,31 @@ export function AgentTracesSection({
)}
</Inspector.Panel>
<RunsToolbar query={query} onQueryChange={setQuery} runs={traces.traces} range={zoom ?? range}>
<TracingSetupAction available={traces.traces.length > 0} live={live} onSetup={() => setShowSetup(true)} />
<Button
variant="outline"
size="icon-sm"
aria-label="Refresh traces"
title="Refresh traces"
disabled={traces.isFetching}
onClick={checkTraces}
>
<RefreshCw className="size-3.5" />
</Button>
{timeControls && (
<TimeRangeControls
fixedRange={zoom ?? (isLiveTail ? null : range)}
rangeHours={timeControls.rangeHours}
onRangeHoursChange={(hours) => changeRange(hours, timeControls.onRangeHoursChange)}
live={isLiveTail}
showLive={live}
onLiveChange={timeControls.onLiveChange}
/>
<div className="h-8 min-w-0 overflow-hidden rounded-md border [&>div]:h-full [&>div]:border-l-0">
<TimeRangeControls
fixedRange={zoom ?? (isLiveTail ? null : range)}
rangeHours={timeControls.rangeHours}
onRangeHoursChange={(hours) => changeRange(hours, timeControls.onRangeHoursChange)}
live={isLiveTail}
showLive={live}
onLiveChange={timeControls.onLiveChange}
/>
</div>
)}
</RunsToolbar>
<TraceCounts runs={filtered} />
<TracesTimeline runs={filtered} range={range} selection={zoom} onSelect={setZoom} />
<AgentTracesTable
traces={runs}
@ -209,6 +228,7 @@ export function AgentTracesSection({
rangeEmpty={traces.traces.length === 0}
onSetUpTracing={() => setShowSetup(true)}
/>
<TraceFooter runs={runs} hasMore={traces.hasMore} />
</div>
</Inspector.Root>
);
@ -222,3 +242,33 @@ function TracesReceived({ received }: { received: boolean }) {
</p>
);
}
function TraceCounts({ runs }: { runs: readonly TraceSummary[] }) {
const failed = runs.filter((run) => run.error_count > 0).length;
return (
<div className="flex shrink-0 flex-wrap items-center gap-x-3 gap-y-1 px-3 pt-2 text-xs text-muted-foreground">
<span>
{runs.length} {runs.length === 1 ? "run" : "runs"} from {new Set(runs.flatMap(traceAgentNames)).size} agents
</span>
{failed > 0 && <span className="text-destructive">{failed} with errors</span>}
</div>
);
}
function TraceFooter({ runs, hasMore }: { runs: readonly TraceSummary[]; hasMore: boolean }) {
return (
<footer className="flex h-8 shrink-0 items-center border-t bg-muted/30 px-3 text-xs text-muted-foreground">
{runs.length} {runs.length === 1 ? "run" : "runs"}
{hasMore ? " loaded" : ""}
</footer>
);
}
function TracingSetupAction({ available, live, onSetup }: { available: boolean; live: boolean; onSetup: () => void }) {
if (!available) return null;
return (
<Button variant="outline" size="sm" className="h-8 text-xs" disabled={!live} onClick={onSetup}>
Set up tracing
</Button>
);
}

2
ui/litellm-dashboard/src/components/lens/traces/list/runSearch/RunSearch.test.tsx
View file

@ -12,7 +12,7 @@ describe("RunSearch", () => {
it("offers the run fields, then the agents seen in the loaded runs", async () => {
const user = userEvent.setup();
render(<RunSearch value="" onChange={vi.fn()} runs={runs} />);
expect(screen.getByText("Search runs, or filter like agent:researcher status:error")).toBeVisible();
expect(screen.getByText("Search input or trace ID")).toBeVisible();
await user.click(box());
expect(
within(listbox())

6
ui/litellm-dashboard/src/components/lens/traces/list/runSearch/RunSearch.tsx
View file

@ -31,11 +31,11 @@ export function RunSearch({ value, onChange, runs, range }: RunSearchProps) {
value={value}
onValueChange={onChange}
label="Search runs"
className="h-11 min-w-0"
className="min-w-40 basis-full sm:basis-72 sm:max-w-96"
>
<SearchBox.Input
className="h-full overflow-hidden rounded-none border-0 px-3 whitespace-nowrap focus-within:bg-muted/40 focus-within:ring-0 dark:bg-transparent"
placeholder="Search runs, or filter like agent:researcher status:error"
className="h-8 overflow-hidden rounded-md px-3 whitespace-nowrap"
placeholder="Search input or trace ID"
/>
<SearchBox.Suggestions>
<SearchBox.CopyCommand title={COPY_HINT} command={command} />

55
ui/litellm-dashboard/src/components/lens/traces/list/runSearch/RunsToolbar.tsx
View file

@ -3,6 +3,9 @@
import type { TraceSummary } from "../../types";
import type { TimeWindow } from "@/components/shared/timeline/Timeline";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useRunFilterRouting } from "../../routing";
import { traceAgentNames } from "../../utils";
import { RunSearch } from "./RunSearch";
interface RunsToolbarProps {
@ -16,16 +19,50 @@ interface RunsToolbarProps {
}
export function RunsToolbar({ query, onQueryChange, runs, range, children }: RunsToolbarProps) {
const { agent, status, setAgent, setStatus } = useRunFilterRouting();
const agents = [
{ value: "", label: "All agents" },
...[...new Set([...runs.flatMap(traceAgentNames), ...(agent ? [agent] : [])])]
.sort()
.map((name) => ({ value: name, label: name })),
];
const statuses = [
{ value: "all", label: "All status" },
{ value: "ok", label: "No errors" },
{ value: "error", label: "With errors" },
];
return (
<div className="@container/traces shrink-0 border-b border-border bg-card">
<div className="grid grid-cols-1 @4xl/traces:grid-cols-[minmax(0,1fr)_auto]">
<RunSearch value={query} onChange={onQueryChange} runs={runs} range={range} />
{children && (
<div className="flex min-h-11 min-w-0 items-stretch justify-end border-t border-border @4xl/traces:border-t-0">
{children}
</div>
)}
</div>
<div className="flex shrink-0 flex-wrap items-center gap-2 border-b bg-card p-2">
<RunSearch value={query} onChange={onQueryChange} runs={runs} range={range} />
<Select items={agents} value={agent} onValueChange={(value) => value !== null && setAgent(value)}>
<SelectTrigger size="sm" className="h-8 max-w-48 min-w-32 text-xs" aria-label="Filter traces by agent">
<SelectValue />
</SelectTrigger>
<SelectContent>
{agents.map((item) => (
<SelectItem key={item.value} value={item.value}>
{item.label}
</SelectItem>
))}
</SelectContent>
</Select>
<Select
items={statuses}
value={status}
onValueChange={(value: "all" | "ok" | "error" | null) => value !== null && setStatus(value)}
>
<SelectTrigger size="sm" className="h-8 min-w-28 text-xs" aria-label="Filter traces by status">
<SelectValue />
</SelectTrigger>
<SelectContent>
{statuses.map((item) => (
<SelectItem key={item.value} value={item.value}>
{item.label}
</SelectItem>
))}
</SelectContent>
</Select>
{children && <div className="ml-auto flex max-w-full flex-wrap items-center justify-end gap-2">{children}</div>}
</div>
);
}

7
ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.test.ts
View file

@ -45,3 +45,10 @@ describe("RUN_INDEX values", () => {
expect(fieldValues(RUN_INDEX, [run({ models: [] })], "model")).toEqual([]);
});
});
it("combines quick filters with search and treats agent names literally", () => {
const selected = filterRuns(runs, "vector", { agent: "researcher", status: "error" });
expect(selected.map((run) => run.trace_id)).toEqual(["bbb222"]);
expect(filterRuns(runs, "vector", { agent: "triage", status: "all" })).toEqual([]);
expect(filterRuns(runs, "", { agent: "research*", status: "all" })).toEqual([]);
});

12
ui/litellm-dashboard/src/components/lens/traces/list/runSearch/runQuery.ts
View file

@ -32,5 +32,13 @@ export const RUN_INDEX: ClientIndex<TraceSummary, RunField> = {
freeText: (run) => [run.trace_id, previewText(run.input_preview), run.name],
};
export const filterRuns = (runs: TraceSummary[], query: string): TraceSummary[] =>
filterItems(RUN_QUERY, RUN_INDEX, runs, query);
export function filterRuns(
runs: TraceSummary[],
query: string,
filters: { agent: string; status: "all" | "ok" | "error" } = { agent: "", status: "all" },
): TraceSummary[] {
const matchesAgent = (run: TraceSummary) => !filters.agent || traceAgentNames(run).includes(filters.agent);
const matchesStatus = (run: TraceSummary) =>
filters.status === "all" || (run.error_count > 0 ? "error" : "ok") === filters.status;
return filterItems(RUN_QUERY, RUN_INDEX, runs, query).filter((run) => matchesAgent(run) && matchesStatus(run));
}

15
ui/litellm-dashboard/src/components/lens/traces/routing.ts
View file

@ -59,6 +59,8 @@ export const OPEN_TRACE_PARSERS = {
export const RUN_FILTER_PARSERS = {
q: parseAsString.withDefault(""),
agent: parseAsString.withDefault(""),
status: parseAsStringLiteral(["all", "ok", "error"]).withDefault("all"),
hours: parseAsNumberLiteral(RANGE_HOURS).withDefault(DEFAULT_RANGE_HOURS),
from: parseAsInteger,
to: parseAsInteger,
@ -139,9 +141,16 @@ export function useLocalRunSelection(initialSpanId: string | null): RunSelection
return { spanId, view, spanTab, stepQuery, errorsOnly, selectSpan, setView, setSpanTab, setStepQuery, setErrorsOnly };
}
export function useRunFilterRouting(): { query: string; setQuery: (query: string) => void } {
const [query, setQuery] = useQueryState("q", RUN_FILTER_PARSERS.q);
return { query, setQuery: useCallback((q: string) => void setQuery(q), [setQuery]) };
export function useRunFilterRouting() {
const [{ q, agent, status }, setParams] = useQueryStates(RUN_FILTER_PARSERS);
return {
query: q,
agent,
status,
setQuery: (query: string) => void setParams({ q: query }),
setAgent: (agent: string) => void setParams({ agent }),
setStatus: (status: "all" | "ok" | "error") => void setParams({ status }),
};
}
export function useRangeHoursRouting(): [number, (hours: number) => void] {

38
ui/litellm-dashboard/src/components/lens/ui/frame.ts
View file

@ -1,38 +0,0 @@
import { cva } from "@/lib/cva.config";
export const frameCard = cva(
"flex min-h-0 flex-1 flex-col overflow-hidden rounded-b-2xl bg-card @min-[24rem]/lens-frame:rounded-t-2xl",
{
variants: {
session: {
live: "border border-t-0 border-foreground/15 @min-[24rem]/lens-frame:border-t",
demo: "border-2 border-t-0 border-info @min-[24rem]/lens-frame:border-t-2",
},
},
defaultVariants: { session: "live" },
},
);
export const frameTab = cva("relative z-raised rounded-t-2xl bg-card", {
variants: {
session: {
live: "border-x border-t border-foreground/15 @min-[24rem]/lens-frame:-mb-px",
demo: "border-x-2 border-t-2 border-info @min-[24rem]/lens-frame:-mb-0.5",
},
},
defaultVariants: { session: "live" },
});
export const frameCorner = cva("block size-full shadow-[0_0_0_12px_var(--card)]", {
variants: {
session: { live: "border-foreground/15", demo: "border-info" },
side: { left: "rounded-br-xl", right: "rounded-bl-xl" },
},
compoundVariants: [
{ session: "live", side: "left", className: "border-r border-b" },
{ session: "live", side: "right", className: "border-l border-b" },
{ session: "demo", side: "left", className: "border-r-2 border-b-2" },
{ session: "demo", side: "right", className: "border-l-2 border-b-2" },
],
defaultVariants: { session: "live" },
});

158
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -9283,6 +9283,23 @@ export interface paths {
patch?: never;
trace?: never;
};
"/lens/{lens_id}/runs/{job_id}/reviews": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/** Read Reviews */
get: operations["read_reviews_lens__lens_id__runs__job_id__reviews_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/liteadmin/slack/connect/{token}": {
parameters: {
query?: never;
@ -33006,6 +33023,20 @@ export interface components {
} & {
[key: string]: unknown;
};
/** InFlight */
InFlight: {
/** Agent */
agent: string;
/** Execution Id */
execution_id: string;
/**
* Started At
* Format: date-time
*/
started_at: string;
/** Trace Id */
trace_id: string;
};
/**
* IncompleteDetails
* @description Details about why the response is incomplete.
@ -33314,6 +33345,21 @@ export interface components {
id: string;
/** Lease Until */
lease_until?: string | null;
/**
* Reading
* @default []
*/
reading: components["schemas"]["InFlight"][];
/**
* Reviewed
* @default 0
*/
reviewed: number;
/**
* Reviews
* @default []
*/
reviews: components["schemas"]["Review"][];
/** Revision */
revision: number;
sample?: components["schemas"]["Sample"] | null;
@ -40908,6 +40954,9 @@ export interface components {
* }
*/
coverage: components["schemas"]["Coverage"];
/** Reading */
reading?: components["schemas"]["InFlight"][] | null;
review?: components["schemas"]["Review"] | null;
/** Stage */
stage: string;
};
@ -43671,6 +43720,81 @@ export interface components {
/** Timeouterrorretries */
TimeoutErrorRetries?: number | null;
};
/** Review */
Review: {
/** Agent */
agent: string;
/**
* At
* Format: date-time
*/
at: string;
/**
* Cannot Assess
* @default false
*/
cannot_assess: boolean;
/** Duration Ms */
duration_ms: number;
/** Execution Id */
execution_id: string;
/** Model */
model: string;
/** Name */
name: string;
/**
* Reasoning
* @default
*/
reasoning: string;
/**
* Spans
* @default []
*/
spans: components["schemas"]["ReviewSpan"][];
/** Trace Id */
trace_id: string;
/**
* Verdicts
* @default []
*/
verdicts: components["schemas"]["ReviewVerdict"][];
};
/** ReviewPage */
ReviewPage: {
/** Reviewed */
reviewed: number;
/** Reviews */
reviews: components["schemas"]["Review"][];
};
/** ReviewSpan */
ReviewSpan: {
/**
* Cited
* @default false
*/
cited: boolean;
/** Kind */
kind: string;
/** Name */
name: string;
/** Preview */
preview: string;
/** Span Id */
span_id: string;
};
/** ReviewVerdict */
ReviewVerdict: {
/** Check Id */
check_id: string;
/**
* Kind
* @enum {string}
*/
kind: "issue" | "pattern";
/** Summary */
summary: string;
};
/**
* RoleMappings
* @description Configuration for mapping SSO groups to LiteLLM roles.
@ -62676,6 +62800,40 @@ export interface operations {
};
};
};
read_reviews_lens__lens_id__runs__job_id__reviews_get: {
parameters: {
query?: {
after?: number;
};
header?: never;
path: {
lens_id: string;
job_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ReviewPage"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
connect_page_liteadmin_slack_connect__token__get: {
parameters: {
query?: never;