mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
feat(lens): flag traces with global System 1 signals (#45094)
* feat(lens): add global trace signals Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * feat(lens): show global trace signals in Settings and Traces Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(proxy): resolve lens signal router at call time Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * feat(lens): signal library, all signal pills and an always-on Signals column Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): tidy signal setup and stop trace ids crowding signal pills Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): address review on signal scans, claims and settings drafts Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): resume signal scan at a partially consumed page Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): keep custom signals separate from library signals Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): give toggled library signals their own row keys Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * test(lens): restore signal toggle-off checks Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> * fix(lens): map signal spans and remove recursive helpers Co-Authored-By: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
d474b433cf
commit
8caf271e6c
35 changed files with 3377 additions and 62 deletions
|
|
@ -0,0 +1,16 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_LensSignalConfig" (
|
||||
"id" TEXT NOT NULL,
|
||||
"data" JSONB NOT NULL,
|
||||
CONSTRAINT "LiteLLM_LensSignalConfig_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_LensTraceSignal" (
|
||||
"trace_id" TEXT NOT NULL,
|
||||
"trace_ref" TEXT NOT NULL DEFAULT '',
|
||||
"config_key" TEXT NOT NULL,
|
||||
"span_count" INTEGER NOT NULL,
|
||||
"claimed_until" TIMESTAMP(3),
|
||||
"classified_at" TIMESTAMP(3),
|
||||
"data" JSONB NOT NULL,
|
||||
CONSTRAINT "LiteLLM_LensTraceSignal_pkey" PRIMARY KEY ("trace_id", "trace_ref")
|
||||
);
|
||||
|
|
@ -1977,3 +1977,20 @@ model LiteLLM_LensDataset {
|
|||
|
||||
@@id([id, revision])
|
||||
}
|
||||
|
||||
model LiteLLM_LensSignalConfig {
|
||||
id String @id
|
||||
data Json
|
||||
}
|
||||
|
||||
model LiteLLM_LensTraceSignal {
|
||||
trace_id String
|
||||
trace_ref String @default("")
|
||||
config_key String
|
||||
span_count Int
|
||||
claimed_until DateTime?
|
||||
classified_at DateTime?
|
||||
data Json
|
||||
|
||||
@@id([trace_id, trace_ref])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -353,6 +353,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_LensRun",
|
||||
"LiteLLM_LensReview",
|
||||
"LiteLLM_LensWorker",
|
||||
"LiteLLM_LensSignalConfig",
|
||||
"LiteLLM_LensTraceSignal",
|
||||
)
|
||||
)
|
||||
PRISMA_RELATIONS: Final[frozenset[str]] = _PRISMA_MODELS | _PRISMA_VIEWS
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ from litellm.proxy.lens.models import (
|
|||
from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image
|
||||
from litellm.proxy.lens.repository import DueLens, LensRepository, WriterDatabase
|
||||
from litellm.proxy.lens.reviews import criteria_key
|
||||
from litellm.proxy.lens.signal_repository import SignalRepository
|
||||
from litellm.proxy.lens.signals import SignalConfig, TraceSignals, trace_signals
|
||||
from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
|
||||
from litellm.proxy.lens.state import (
|
||||
can_access,
|
||||
|
|
@ -69,6 +71,7 @@ from litellm.proxy.lens.state import (
|
|||
summarized,
|
||||
)
|
||||
from litellm.proxy.tracing_runtime import provide_storage
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router: Final = APIRouter(prefix="/lens", tags=["Lens"])
|
||||
|
|
@ -101,6 +104,14 @@ def repository() -> LensRepository:
|
|||
return LensRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
|
||||
|
||||
|
||||
def signals_repository() -> SignalRepository:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(503, "Lens needs a connected Postgres database")
|
||||
return SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db)))
|
||||
|
||||
|
||||
def source_reader(storage: Storage | None) -> SourceReader:
|
||||
if storage is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -118,6 +129,20 @@ def user_scope(auth: UserAPIKeyAuth, write: bool = False) -> Scope:
|
|||
raise HTTPException(403, "Lens requires proxy administrator access")
|
||||
|
||||
|
||||
def validate_signal_model(config: SignalConfig, llm_router: Router | None) -> None:
|
||||
if not config.model:
|
||||
return
|
||||
message: Final = "Choose a System 1 model (evaluation mode) configured on this proxy"
|
||||
if llm_router is None:
|
||||
raise HTTPException(400, message)
|
||||
try:
|
||||
model_group: Final = llm_router.get_model_group_info(model_group=config.model)
|
||||
except Exception as error:
|
||||
raise HTTPException(400, message) from error
|
||||
if model_group is None or model_group.mode != "evaluation":
|
||||
raise HTTPException(400, message)
|
||||
|
||||
|
||||
async def get_lens(lens_id: str, scope: Scope) -> Lens:
|
||||
lens: Final = await repository().get(lens_id)
|
||||
if lens is None or not can_access(scope, lens.scope):
|
||||
|
|
@ -261,6 +286,39 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
|
|||
return await source_reader(storage).agents(scope) if storage is not None else ()
|
||||
|
||||
|
||||
@router.get("/signals", response_model=SignalConfig)
|
||||
async def get_signals(auth: Auth) -> SignalConfig:
|
||||
user_scope(auth)
|
||||
return await signals_repository().get_config()
|
||||
|
||||
|
||||
@router.put("/signals", response_model=SignalConfig)
|
||||
async def put_signals(body: SignalConfig, auth: Auth) -> SignalConfig:
|
||||
user_scope(auth, write=True)
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
validate_signal_model(body, llm_router)
|
||||
await signals_repository().save_config(body)
|
||||
return body
|
||||
|
||||
|
||||
@router.post("/traces/signals", response_model=tuple[TraceSignals, ...])
|
||||
async def trace_signal_statuses(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceSignals, ...]:
|
||||
user_scope(auth)
|
||||
repo: Final = signals_repository()
|
||||
config: Final = await repo.get_config()
|
||||
existing: Final = await repo.traces(body.traces)
|
||||
rows: Final = MappingProxyType({(row.trace_id, row.trace_ref): row for row in existing})
|
||||
return tuple(
|
||||
trace_signals(
|
||||
trace,
|
||||
rows.get((trace.trace_id, trace.trace_ref)),
|
||||
config,
|
||||
)
|
||||
for trace in body.traces
|
||||
)
|
||||
|
||||
|
||||
@router.post("/traces/findings", response_model=tuple[TraceFindingCount, ...])
|
||||
async def trace_findings(body: TraceFindingsRequest, auth: Auth) -> tuple[TraceFindingCount, ...]:
|
||||
user_scope(auth)
|
||||
|
|
|
|||
142
litellm/proxy/lens/signal_repository.py
Normal file
142
litellm/proxy/lens/signal_repository.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy.lens.models import Execution, TraceIdentity
|
||||
from litellm.proxy.lens.repository import Database, Row
|
||||
from litellm.proxy.lens.signals import (
|
||||
SIGNAL_RECLASSIFY_AFTER,
|
||||
SIGNAL_RETRY_FAILED_AFTER,
|
||||
SignalAttempt,
|
||||
SignalConfig,
|
||||
StoredTraceSignal,
|
||||
)
|
||||
|
||||
_ROWS: Final[TypeAdapter[tuple[Row, ...]]] = TypeAdapter(tuple[Row, ...])
|
||||
|
||||
|
||||
class SignalRepository:
|
||||
def __init__(self, db: Database) -> None:
|
||||
self.db: Final = db
|
||||
|
||||
async def get_config(self) -> SignalConfig:
|
||||
rows: Final = _ROWS.validate_python(
|
||||
await self.db.query_raw('SELECT data FROM "LiteLLM_LensSignalConfig" WHERE id=$1', "global")
|
||||
)
|
||||
return SignalConfig() if not rows else SignalConfig.model_validate(rows[0].data)
|
||||
|
||||
async def save_config(self, config: SignalConfig) -> None:
|
||||
await self.db.execute_raw(
|
||||
"""INSERT INTO "LiteLLM_LensSignalConfig" (id, data)
|
||||
VALUES ($1, $2::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET data=EXCLUDED.data""",
|
||||
"global",
|
||||
json.dumps(config.model_dump(mode="json")),
|
||||
)
|
||||
|
||||
async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]:
|
||||
if not identities:
|
||||
return ()
|
||||
payload: Final = json.dumps(
|
||||
tuple({"trace_id": trace.trace_id, "trace_ref": trace.trace_ref} for trace in identities)
|
||||
)
|
||||
rows: Final = _ROWS.validate_python(
|
||||
await self.db.query_raw(
|
||||
"""SELECT jsonb_build_object(
|
||||
'trace_id', trace_id,
|
||||
'trace_ref', trace_ref,
|
||||
'config_key', config_key,
|
||||
'span_count', span_count,
|
||||
'claimed_until', claimed_until,
|
||||
'classified_at', classified_at,
|
||||
'data', data
|
||||
) AS data
|
||||
FROM "LiteLLM_LensTraceSignal"
|
||||
WHERE (trace_id, trace_ref) IN (
|
||||
SELECT trace_id, trace_ref FROM jsonb_to_recordset($1::jsonb) AS requested(
|
||||
trace_id text, trace_ref text
|
||||
)
|
||||
)""",
|
||||
payload,
|
||||
)
|
||||
)
|
||||
return tuple(StoredTraceSignal.model_validate(row.data) for row in rows)
|
||||
|
||||
async def claim(
|
||||
self,
|
||||
execution: Execution,
|
||||
config: SignalConfig,
|
||||
claimed_until: datetime,
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
data: Final = json.dumps({"status": "pending", "scores": {}, "model": config.model, "error": ""})
|
||||
rows: Final = _ROWS.validate_python(
|
||||
await self.db.query_raw(
|
||||
"""INSERT INTO "LiteLLM_LensTraceSignal" AS stored
|
||||
(trace_id, trace_ref, config_key, span_count, claimed_until, classified_at, data)
|
||||
VALUES ($1, $2, $3, $4, $5::timestamp, NULL, $6::jsonb)
|
||||
ON CONFLICT (trace_id, trace_ref) DO UPDATE SET
|
||||
config_key=EXCLUDED.config_key,
|
||||
span_count=EXCLUDED.span_count,
|
||||
claimed_until=EXCLUDED.claimed_until,
|
||||
classified_at=NULL,
|
||||
data=EXCLUDED.data
|
||||
WHERE (stored.claimed_until IS NULL OR stored.claimed_until < $7::timestamp)
|
||||
AND (
|
||||
stored.config_key IS DISTINCT FROM EXCLUDED.config_key
|
||||
OR (
|
||||
stored.data->>'status'='pending'
|
||||
AND stored.claimed_until < $7::timestamp
|
||||
)
|
||||
OR (
|
||||
EXCLUDED.span_count > stored.span_count
|
||||
AND stored.classified_at < $8::timestamp
|
||||
)
|
||||
OR (
|
||||
stored.data->>'status'='failed'
|
||||
AND stored.classified_at < $9::timestamp
|
||||
)
|
||||
)
|
||||
RETURNING jsonb_build_object('trace_id', trace_id) AS data""",
|
||||
execution.trace_id,
|
||||
execution.trace_ref,
|
||||
config.key(),
|
||||
execution.span_count,
|
||||
claimed_until,
|
||||
data,
|
||||
now,
|
||||
now - SIGNAL_RECLASSIFY_AFTER,
|
||||
now - SIGNAL_RETRY_FAILED_AFTER,
|
||||
)
|
||||
)
|
||||
return bool(rows)
|
||||
|
||||
async def store(
|
||||
self,
|
||||
execution: Execution,
|
||||
config: SignalConfig,
|
||||
claimed_until: datetime,
|
||||
classified_at: datetime,
|
||||
attempt: SignalAttempt,
|
||||
) -> None:
|
||||
payload: Final = json.dumps(
|
||||
{
|
||||
"status": attempt.status,
|
||||
"scores": dict(attempt.scores),
|
||||
"model": attempt.model,
|
||||
"error": attempt.error,
|
||||
}
|
||||
)
|
||||
await self.db.execute_raw(
|
||||
"""UPDATE "LiteLLM_LensTraceSignal"
|
||||
SET classified_at=$1::timestamp, claimed_until=NULL, data=$2::jsonb
|
||||
WHERE trace_id=$3 AND trace_ref=$4 AND config_key=$5 AND claimed_until=$6::timestamp""",
|
||||
classified_at,
|
||||
payload,
|
||||
execution.trace_id,
|
||||
execution.trace_ref,
|
||||
config.key(),
|
||||
claimed_until,
|
||||
)
|
||||
586
litellm/proxy/lens/signals.py
Normal file
586
litellm/proxy/lens/signals.py
Normal file
|
|
@ -0,0 +1,586 @@
|
|||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import accumulate
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, Field, JsonValue, ValidationError, field_validator, model_validator
|
||||
|
||||
from litellm.integrations.clickhouse.context import lens_analysis
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_internal_details
|
||||
from litellm.proxy.lens.models import ActivitySelection, Execution, Record, Scope, TraceIdentity
|
||||
from litellm.proxy.lens.sources import SourceReader, Storage
|
||||
|
||||
SIGNAL_INTERVAL_SECONDS: Final = 60
|
||||
SIGNAL_PAGE_SIZE: Final = 100
|
||||
SIGNAL_MAX_PER_TICK: Final = 50
|
||||
SIGNAL_CONCURRENCY: Final = 8
|
||||
SIGNAL_CLAIM_LEASE: Final = timedelta(minutes=5)
|
||||
SIGNAL_RECLASSIFY_AFTER: Final = timedelta(minutes=5)
|
||||
SIGNAL_RETRY_FAILED_AFTER: Final = timedelta(minutes=30)
|
||||
SIGNAL_MAX_CONTENT_PAGES: Final = 3
|
||||
SIGNAL_PART_MAX_CHARS: Final = 2000
|
||||
SIGNAL_PART_HEAD_CHARS: Final = 800
|
||||
SIGNAL_PART_TAIL_CHARS: Final = 1200
|
||||
SIGNAL_TRANSCRIPT_MAX_CHARS: Final = 40000
|
||||
SIGNAL_TRANSCRIPT_HEAD_CHARS: Final = 15000
|
||||
SIGNAL_TRANSCRIPT_TAIL_CHARS: Final = 25000
|
||||
SIGNAL_MAX_SCAN_PAGES: Final = 10
|
||||
SIGNAL_TASK: Final = (
|
||||
"An AI agent run recorded as a trace. Judge only what the user and the agent said and did in these steps."
|
||||
)
|
||||
|
||||
|
||||
class Signal(Record):
|
||||
id: str = Field(pattern=r"^[a-z][a-z0-9_]{0,63}$")
|
||||
name: str = Field(min_length=1, max_length=60)
|
||||
question: str = Field(min_length=3, max_length=500)
|
||||
|
||||
|
||||
DEFAULT_SIGNALS: Final[tuple[Signal, ...]] = (
|
||||
Signal(
|
||||
id="user_frustration",
|
||||
name="User frustration",
|
||||
question=(
|
||||
"Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example "
|
||||
"complaints, irritated corrections, all caps, profanity, or giving up on the task?"
|
||||
),
|
||||
),
|
||||
Signal(
|
||||
id="missing_capability",
|
||||
name="Missing capability",
|
||||
question=(
|
||||
"Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it "
|
||||
"lacks a tool, permission, integration or data source, or fails because the capability does not exist?"
|
||||
),
|
||||
),
|
||||
Signal(
|
||||
id="repeated_request",
|
||||
name="Repeated request",
|
||||
question=(
|
||||
"Does the user ask for the same thing more than once in this run, usually because the agent did not "
|
||||
"deliver it the first time?"
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SignalConfig(Record):
|
||||
model: str = ""
|
||||
threshold: float = Field(default=0.5, ge=0.05, le=0.95, allow_inf_nan=False)
|
||||
signals: tuple[Signal, ...] = DEFAULT_SIGNALS
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_signals(self) -> "SignalConfig":
|
||||
if len(self.signals) > 20:
|
||||
raise ValueError("A maximum of 20 signals is allowed")
|
||||
if len(frozenset(signal.id for signal in self.signals)) != len(self.signals):
|
||||
raise ValueError("Signal IDs must be unique")
|
||||
return self
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.model) and bool(self.signals)
|
||||
|
||||
def key(self) -> str:
|
||||
payload: Final = json.dumps(
|
||||
{
|
||||
"model": self.model,
|
||||
"signals": tuple({"id": signal.id, "question": signal.question} for signal in self.signals),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(payload.encode()).hexdigest()
|
||||
|
||||
|
||||
Score: TypeAlias = Annotated[float, Field(ge=0, le=1, allow_inf_nan=False)]
|
||||
|
||||
|
||||
class SignalFlag(Record):
|
||||
signal_id: str
|
||||
name: str
|
||||
score: Score
|
||||
|
||||
|
||||
class TraceSignals(TraceIdentity):
|
||||
status: Literal["unclassified", "pending", "classified", "failed"]
|
||||
flags: tuple[SignalFlag, ...] = ()
|
||||
model: str = ""
|
||||
classified_at: datetime | None = None
|
||||
|
||||
|
||||
class SignalStep(Record):
|
||||
kind: str
|
||||
name: str
|
||||
content: str
|
||||
|
||||
|
||||
class SignalData(Record):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
status: Literal["pending", "classified", "failed"] = "pending"
|
||||
scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
|
||||
model: str = ""
|
||||
error: str = ""
|
||||
|
||||
|
||||
class SignalAttempt(Record):
|
||||
status: Literal["classified", "failed"]
|
||||
scores: Mapping[str, Score] = Field(default_factory=lambda: MappingProxyType({}))
|
||||
model: str
|
||||
error: str = ""
|
||||
|
||||
|
||||
class StoredTraceSignal(Record):
|
||||
trace_id: str
|
||||
trace_ref: str = ""
|
||||
config_key: str
|
||||
span_count: int
|
||||
claimed_until: datetime | None = None
|
||||
classified_at: datetime | None = None
|
||||
data: JsonValue
|
||||
|
||||
@field_validator("claimed_until", "classified_at")
|
||||
@classmethod
|
||||
def normalize_database_timestamp(cls, value: datetime | None) -> datetime | None:
|
||||
if value is not None and value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
class NoulAnswer(Record):
|
||||
model_config = ConfigDict(extra="ignore", allow_inf_nan=False, from_attributes=True)
|
||||
|
||||
type: Literal["noul"]
|
||||
noul: float = Field(ge=0, le=1, allow_inf_nan=False)
|
||||
|
||||
|
||||
class DecisionsOutput(Record):
|
||||
model_config = ConfigDict(extra="ignore", from_attributes=True)
|
||||
|
||||
answers: Mapping[str, object]
|
||||
|
||||
|
||||
DecisionState: TypeAlias = Mapping[str, object]
|
||||
DecisionQuestions: TypeAlias = Mapping[str, Mapping[str, str]]
|
||||
Clock: TypeAlias = Callable[[], datetime]
|
||||
RouterReady: TypeAlias = Callable[[], bool]
|
||||
|
||||
|
||||
class DecisionsCall(Protocol):
|
||||
async def __call__(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
state: DecisionState,
|
||||
questions: DecisionQuestions,
|
||||
timeout: float,
|
||||
metadata: Mapping[str, object],
|
||||
) -> object: ...
|
||||
|
||||
|
||||
class SignalRepositoryProtocol(Protocol):
|
||||
async def get_config(self) -> SignalConfig: ...
|
||||
async def traces(self, identities: tuple[TraceIdentity, ...]) -> tuple[StoredTraceSignal, ...]: ...
|
||||
async def claim(
|
||||
self,
|
||||
execution: Execution,
|
||||
config: SignalConfig,
|
||||
claimed_until: datetime,
|
||||
now: datetime,
|
||||
) -> bool: ...
|
||||
async def store(
|
||||
self,
|
||||
execution: Execution,
|
||||
config: SignalConfig,
|
||||
claimed_until: datetime,
|
||||
classified_at: datetime,
|
||||
attempt: SignalAttempt,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def signal_identity(trace: TraceIdentity | StoredTraceSignal | Execution) -> tuple[str, str]:
|
||||
return trace.trace_id, trace.trace_ref
|
||||
|
||||
|
||||
def candidate(
|
||||
trace: Execution,
|
||||
existing: StoredTraceSignal | None,
|
||||
config_key: str,
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
if existing is None:
|
||||
return True
|
||||
if existing.claimed_until is not None and existing.claimed_until > now:
|
||||
return False
|
||||
if existing.config_key != config_key:
|
||||
return True
|
||||
status: Final = existing.data.get("status") if isinstance(existing.data, dict) else ""
|
||||
if status == "pending":
|
||||
return existing.claimed_until is not None and existing.claimed_until <= now
|
||||
if existing.span_count > trace.span_count:
|
||||
return False
|
||||
if existing.span_count < trace.span_count:
|
||||
return existing.classified_at is not None and existing.classified_at < now - SIGNAL_RECLASSIFY_AFTER
|
||||
return (
|
||||
status == "failed"
|
||||
and existing.classified_at is not None
|
||||
and existing.classified_at < now - SIGNAL_RETRY_FAILED_AFTER
|
||||
)
|
||||
|
||||
|
||||
def _take_head(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
|
||||
if remaining <= 0:
|
||||
return ()
|
||||
cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in steps))
|
||||
boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
|
||||
if boundary is None:
|
||||
return steps
|
||||
preceding: Final = steps[:boundary]
|
||||
last: Final = steps[boundary]
|
||||
used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
|
||||
last_length: Final = remaining - used
|
||||
return (
|
||||
*preceding,
|
||||
last
|
||||
if last_length == len(last.content)
|
||||
else last.model_copy(update=MappingProxyType({"content": last.content[:last_length]})),
|
||||
)
|
||||
|
||||
|
||||
def _take_tail(steps: tuple[SignalStep, ...], remaining: int) -> tuple[SignalStep, ...]:
|
||||
if remaining <= 0:
|
||||
return ()
|
||||
reversed_steps: Final = tuple(reversed(steps))
|
||||
cumulative_lengths: Final = tuple(accumulate(len(step.content) for step in reversed_steps))
|
||||
boundary: Final = next((index for index, total in enumerate(cumulative_lengths) if total >= remaining), None)
|
||||
if boundary is None:
|
||||
return steps
|
||||
preceding: Final = reversed_steps[:boundary]
|
||||
last: Final = reversed_steps[boundary]
|
||||
used: Final = cumulative_lengths[boundary - 1] if boundary > 0 else 0
|
||||
last_length: Final = remaining - used
|
||||
selected: Final = (
|
||||
*preceding,
|
||||
last
|
||||
if last_length == len(last.content)
|
||||
else last.model_copy(update=MappingProxyType({"content": last.content[-last_length:]})),
|
||||
)
|
||||
return tuple(reversed(selected))
|
||||
|
||||
|
||||
def _bounded_steps(steps: tuple[SignalStep, ...]) -> tuple[SignalStep, ...]:
|
||||
if sum(len(step.content) for step in steps) <= SIGNAL_TRANSCRIPT_MAX_CHARS:
|
||||
return steps
|
||||
head: Final = _take_head(steps, SIGNAL_TRANSCRIPT_HEAD_CHARS)
|
||||
tail: Final = _take_tail(steps, SIGNAL_TRANSCRIPT_TAIL_CHARS)
|
||||
omitted_count: Final = len(steps) - len(head) - len(tail)
|
||||
marker: Final = SignalStep(kind="omitted", name="", content=f"{omitted_count} steps omitted")
|
||||
return (*head, marker, *tail)
|
||||
|
||||
|
||||
def _part_excerpt(content: str) -> str:
|
||||
if len(content) <= SIGNAL_PART_MAX_CHARS:
|
||||
return content
|
||||
omitted: Final = len(content) - SIGNAL_PART_MAX_CHARS
|
||||
marker: Final = f"\n[... {omitted} characters omitted ...]\n"
|
||||
return f"{content[:SIGNAL_PART_HEAD_CHARS]}{marker}{content[-SIGNAL_PART_TAIL_CHARS:]}"
|
||||
|
||||
|
||||
async def _content_pages(
|
||||
reader: SourceReader,
|
||||
scope: Scope,
|
||||
execution: Execution,
|
||||
cursor: str,
|
||||
pages_left: int,
|
||||
) -> tuple[SignalStep, ...]:
|
||||
if pages_left == 0:
|
||||
return ()
|
||||
content: Final = await reader.content(scope, execution, cursor)
|
||||
current: Final = tuple(
|
||||
SignalStep(kind=part.kind, name=part.name, content=_part_excerpt(part.content)) for part in content.parts
|
||||
)
|
||||
rest: Final = (
|
||||
await _content_pages(reader, scope, execution, content.next_cursor, pages_left - 1)
|
||||
if content.next_cursor is not None
|
||||
else ()
|
||||
)
|
||||
return (*current, *rest)
|
||||
|
||||
|
||||
async def signal_state(reader: SourceReader, scope: Scope, execution: Execution) -> DecisionState:
|
||||
steps: Final = _bounded_steps(await _content_pages(reader, scope, execution, "", SIGNAL_MAX_CONTENT_PAGES))
|
||||
return {
|
||||
"task": SIGNAL_TASK,
|
||||
"steps": tuple(step.model_dump(mode="json") for step in steps),
|
||||
}
|
||||
|
||||
|
||||
def _noul_score(value: object) -> float | None:
|
||||
try:
|
||||
return NoulAnswer.model_validate(value).noul
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
class SignalClassifier:
|
||||
def __init__(self, reader: SourceReader, completion: DecisionsCall, clock: Clock) -> None:
|
||||
self.reader: Final = reader
|
||||
self.completion: Final = completion
|
||||
self.clock: Final = clock
|
||||
|
||||
async def classify(self, scope: Scope, execution: Execution, config: SignalConfig) -> SignalAttempt:
|
||||
try:
|
||||
state: Final = await signal_state(self.reader, scope, execution)
|
||||
questions: Final = {
|
||||
signal.id: {"type": "noul", "instructions": signal.question} for signal in config.signals
|
||||
}
|
||||
with lens_analysis(), inherit_message_logging_privacy(True):
|
||||
response: Final = await self.completion(
|
||||
model=config.model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
timeout=60,
|
||||
metadata={"tags": ["litellm-lens-signals"]},
|
||||
)
|
||||
output: Final = DecisionsOutput.model_validate(response)
|
||||
scores: Final = MappingProxyType(
|
||||
{
|
||||
signal.id: score
|
||||
for signal in config.signals
|
||||
if (score := _noul_score(output.answers.get(signal.id))) is not None
|
||||
}
|
||||
)
|
||||
if len(scores) != len(config.signals):
|
||||
return SignalAttempt(
|
||||
status="failed",
|
||||
scores=scores,
|
||||
model=config.model,
|
||||
error="Decisions response omitted a configured noul answer",
|
||||
)
|
||||
return SignalAttempt(status="classified", scores=scores, model=config.model)
|
||||
except Exception as error:
|
||||
detail: Final = redact_internal_details(str(error))[:300]
|
||||
return SignalAttempt(status="failed", model=config.model, error=detail)
|
||||
|
||||
|
||||
def trace_signals(
|
||||
trace: TraceIdentity,
|
||||
existing: StoredTraceSignal | None,
|
||||
config: SignalConfig,
|
||||
) -> TraceSignals:
|
||||
if existing is None or existing.config_key != config.key():
|
||||
return TraceSignals(trace_id=trace.trace_id, trace_ref=trace.trace_ref, status="unclassified")
|
||||
data: Final = SignalData.model_validate(existing.data)
|
||||
if data.status == "pending":
|
||||
return TraceSignals(
|
||||
trace_id=trace.trace_id,
|
||||
trace_ref=trace.trace_ref,
|
||||
status="pending",
|
||||
model=data.model,
|
||||
)
|
||||
if data.status == "failed" or data.error:
|
||||
return TraceSignals(
|
||||
trace_id=trace.trace_id,
|
||||
trace_ref=trace.trace_ref,
|
||||
status="failed",
|
||||
model=data.model,
|
||||
classified_at=existing.classified_at,
|
||||
)
|
||||
flags: Final = tuple(
|
||||
sorted(
|
||||
(
|
||||
SignalFlag(signal_id=signal.id, name=signal.name, score=data.scores[signal.id])
|
||||
for signal in config.signals
|
||||
if signal.id in data.scores and data.scores[signal.id] >= config.threshold
|
||||
),
|
||||
key=lambda flag: flag.score,
|
||||
reverse=True,
|
||||
)
|
||||
)
|
||||
return TraceSignals(
|
||||
trace_id=trace.trace_id,
|
||||
trace_ref=trace.trace_ref,
|
||||
status="classified",
|
||||
flags=flags,
|
||||
model=data.model,
|
||||
classified_at=existing.classified_at,
|
||||
)
|
||||
|
||||
|
||||
async def _process_claimed(
|
||||
classifier: SignalClassifier,
|
||||
repository: SignalRepositoryProtocol,
|
||||
scope: Scope,
|
||||
execution: Execution,
|
||||
config: SignalConfig,
|
||||
claimed_until: datetime,
|
||||
) -> None:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
attempt: Final = await classifier.classify(scope, execution, config)
|
||||
try:
|
||||
await repository.store(execution, config, claimed_until, classifier.clock(), attempt)
|
||||
except Exception as error:
|
||||
verbose_proxy_logger.error("Lens signal result could not be stored: %s", redact_internal_details(str(error)))
|
||||
|
||||
|
||||
class _SignalScan:
|
||||
def __init__(
|
||||
self,
|
||||
reader: SourceReader,
|
||||
repository: SignalRepositoryProtocol,
|
||||
scope: Scope,
|
||||
config: SignalConfig,
|
||||
now: datetime,
|
||||
cursor: str,
|
||||
limit: int,
|
||||
) -> None:
|
||||
self.reader: Final = reader
|
||||
self.repository: Final = repository
|
||||
self.scope: Final = scope
|
||||
self.config: Final = config
|
||||
self.now: Final = now
|
||||
self.cursor: str = cursor
|
||||
self.limit: Final = limit
|
||||
self.executions: tuple[Execution, ...] = ()
|
||||
self.finished: bool = False
|
||||
|
||||
async def _read_page(self, start: int, end: int) -> tuple[tuple[Execution, ...], str | None]:
|
||||
page_cursor: Final = self.cursor
|
||||
sample: Final = await self.reader.sample(
|
||||
self.scope,
|
||||
ActivitySelection(source="traces"),
|
||||
start,
|
||||
end,
|
||||
page_size=SIGNAL_PAGE_SIZE,
|
||||
cursor=page_cursor,
|
||||
)
|
||||
identities: Final = tuple(
|
||||
TraceIdentity(trace_id=trace.trace_id, trace_ref=trace.trace_ref) for trace in sample.executions
|
||||
)
|
||||
existing_rows: Final = await self.repository.traces(identities)
|
||||
existing: Final = MappingProxyType({signal_identity(row): row for row in existing_rows})
|
||||
remaining: Final = self.limit - len(self.executions)
|
||||
all_eligible: Final = tuple(
|
||||
execution
|
||||
for execution in sample.executions
|
||||
if candidate(execution, existing.get(signal_identity(execution)), self.config.key(), self.now)
|
||||
)
|
||||
eligible: Final = all_eligible[:remaining]
|
||||
next_cursor: Final = page_cursor if len(all_eligible) > remaining else sample.next_cursor
|
||||
return eligible, next_cursor
|
||||
|
||||
async def run(self) -> tuple[tuple[Execution, ...], str]:
|
||||
start: Final = int((self.now - timedelta(hours=24)).timestamp() * 1000)
|
||||
end: Final = int((self.now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
for _ in range(SIGNAL_MAX_SCAN_PAGES):
|
||||
if self.finished or len(self.executions) >= self.limit:
|
||||
break
|
||||
eligible, next_cursor = await self._read_page(start, end)
|
||||
self.executions = (*self.executions, *eligible)
|
||||
if next_cursor is None:
|
||||
self.cursor = ""
|
||||
self.finished = True
|
||||
else:
|
||||
self.cursor = next_cursor
|
||||
return self.executions, self.cursor
|
||||
|
||||
|
||||
async def _scan_pages(
|
||||
reader: SourceReader,
|
||||
repository: SignalRepositoryProtocol,
|
||||
scope: Scope,
|
||||
config: SignalConfig,
|
||||
now: datetime,
|
||||
cursor: str,
|
||||
remaining: int,
|
||||
) -> tuple[tuple[Execution, ...], str]:
|
||||
if remaining <= 0:
|
||||
return (), cursor
|
||||
scan: Final = _SignalScan(reader, repository, scope, config, now, cursor, remaining)
|
||||
return await scan.run()
|
||||
|
||||
|
||||
async def run_signal_tick(
|
||||
storage: Storage,
|
||||
repository: SignalRepositoryProtocol | None,
|
||||
completion: DecisionsCall | None,
|
||||
clock: Clock,
|
||||
router_ready: RouterReady = lambda: True,
|
||||
cursor: str = "",
|
||||
) -> str:
|
||||
if repository is None or completion is None or not router_ready():
|
||||
return cursor
|
||||
now: Final = clock()
|
||||
config: Final = await repository.get_config()
|
||||
if not config.enabled:
|
||||
return cursor
|
||||
reader: Final = SourceReader(storage)
|
||||
scope: Final = Scope(all_teams=True)
|
||||
candidates: Final = await _scan_pages(
|
||||
reader,
|
||||
repository,
|
||||
scope,
|
||||
config,
|
||||
now,
|
||||
cursor,
|
||||
SIGNAL_MAX_PER_TICK,
|
||||
)
|
||||
executions, next_cursor = candidates
|
||||
classifier: Final = SignalClassifier(reader, completion, clock)
|
||||
semaphore: Final = asyncio.Semaphore(SIGNAL_CONCURRENCY)
|
||||
|
||||
async def process(execution: Execution) -> None:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
async with semaphore:
|
||||
claimed_at: Final = classifier.clock()
|
||||
claimed_until: Final = claimed_at + SIGNAL_CLAIM_LEASE
|
||||
try:
|
||||
claimed: Final = await repository.claim(execution, config, claimed_until, claimed_at)
|
||||
except Exception as error:
|
||||
verbose_proxy_logger.error("Lens signal claim failed: %s", redact_internal_details(str(error)))
|
||||
return
|
||||
if not claimed:
|
||||
return
|
||||
await _process_claimed(classifier, repository, scope, execution, config, claimed_until)
|
||||
|
||||
await asyncio.gather(*(process(execution) for execution in executions))
|
||||
return next_cursor
|
||||
|
||||
|
||||
class _SignalLoopState:
|
||||
def __init__(self) -> None:
|
||||
self.cursor: str = ""
|
||||
|
||||
|
||||
async def run_signal_loop(
|
||||
storage: Storage,
|
||||
repository: SignalRepositoryProtocol | None,
|
||||
completion: DecisionsCall | None,
|
||||
clock: Clock = lambda: datetime.now(timezone.utc),
|
||||
router_ready: RouterReady = lambda: True,
|
||||
) -> None:
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
state: Final = _SignalLoopState()
|
||||
while True:
|
||||
try:
|
||||
state.cursor = await run_signal_tick(
|
||||
storage,
|
||||
repository,
|
||||
completion,
|
||||
clock,
|
||||
router_ready,
|
||||
cursor=state.cursor,
|
||||
)
|
||||
except Exception as error:
|
||||
verbose_proxy_logger.error("Lens signal tick failed: %s", redact_internal_details(str(error)))
|
||||
await asyncio.sleep(SIGNAL_INTERVAL_SECONDS)
|
||||
|
|
@ -581,6 +581,14 @@ from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_sp
|
|||
from litellm.proxy.image_endpoints.endpoints import router as image_router
|
||||
from litellm.proxy.lens.dataset_endpoints import router as lens_dataset_router
|
||||
from litellm.proxy.lens.endpoints import router as lens_router
|
||||
from litellm.proxy.lens.repository import WriterDatabase
|
||||
from litellm.proxy.lens.signal_repository import SignalRepository
|
||||
from litellm.proxy.lens.signals import (
|
||||
DecisionQuestions,
|
||||
DecisionsCall,
|
||||
DecisionState,
|
||||
run_signal_loop,
|
||||
)
|
||||
from litellm.proxy.list_api.common import (
|
||||
ManagementProblem,
|
||||
problem_response,
|
||||
|
|
@ -1275,6 +1283,27 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries:
|
|||
return client.writer_db
|
||||
|
||||
|
||||
async def _call_current_lens_signal_router(
|
||||
*,
|
||||
model: str,
|
||||
state: DecisionState,
|
||||
questions: DecisionQuestions,
|
||||
timeout: float,
|
||||
metadata: Mapping[str, object],
|
||||
) -> object:
|
||||
current_router: Final = llm_router
|
||||
if current_router is None:
|
||||
raise RuntimeError("The proxy router is not initialized")
|
||||
decisions: Final[DecisionsCall] = cast(DecisionsCall, current_router.adecisions)
|
||||
return await decisions(
|
||||
model=model,
|
||||
state=state,
|
||||
questions=questions,
|
||||
timeout=timeout,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
|
||||
global \
|
||||
|
|
@ -1645,12 +1674,30 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
|
|||
state: Final[ProxyLifespanState] = {"tracing_receiver": receiver}
|
||||
from litellm.proxy.admin_mcp import admin_mcp_lifespan
|
||||
|
||||
signal_completion: Final[DecisionsCall] = _call_current_lens_signal_router
|
||||
signal_task: Final = (
|
||||
asyncio.create_task(
|
||||
run_signal_loop(
|
||||
receiver.storage,
|
||||
SignalRepository(WriterDatabase(writer_wrapper(prisma_client.db))),
|
||||
signal_completion,
|
||||
router_ready=lambda: llm_router is not None,
|
||||
)
|
||||
)
|
||||
if receiver is not None and prisma_client is not None
|
||||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncExitStack() as admin_mcp_stack:
|
||||
try:
|
||||
await admin_mcp_stack.enter_async_context(admin_mcp_lifespan(app))
|
||||
yield state
|
||||
finally:
|
||||
if signal_task is not None:
|
||||
signal_task.cancel()
|
||||
await asyncio.gather(signal_task, return_exceptions=True)
|
||||
|
||||
if model_info_scheduler is not None and model_info_scheduler.running:
|
||||
model_info_scheduler.remove_job("refresh_model_info")
|
||||
if model_info_scheduler is not scheduler:
|
||||
|
|
@ -13951,7 +13998,7 @@ async def run_thread(
|
|||
# )
|
||||
# async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.model_repository import ModelRepository
|
||||
from litellm.repositories.table_repositories import (
|
||||
|
|
|
|||
|
|
@ -1977,3 +1977,20 @@ model LiteLLM_LensDataset {
|
|||
|
||||
@@id([id, revision])
|
||||
}
|
||||
|
||||
model LiteLLM_LensSignalConfig {
|
||||
id String @id
|
||||
data Json
|
||||
}
|
||||
|
||||
model LiteLLM_LensTraceSignal {
|
||||
trace_id String
|
||||
trace_ref String @default("")
|
||||
config_key String
|
||||
span_count Int
|
||||
claimed_until DateTime?
|
||||
classified_at DateTime?
|
||||
data Json
|
||||
|
||||
@@id([trace_id, trace_ref])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1977,3 +1977,20 @@ model LiteLLM_LensDataset {
|
|||
|
||||
@@id([id, revision])
|
||||
}
|
||||
|
||||
model LiteLLM_LensSignalConfig {
|
||||
id String @id
|
||||
data Json
|
||||
}
|
||||
|
||||
model LiteLLM_LensTraceSignal {
|
||||
trace_id String
|
||||
trace_ref String @default("")
|
||||
config_key String
|
||||
span_count Int
|
||||
claimed_until DateTime?
|
||||
classified_at DateTime?
|
||||
data Json
|
||||
|
||||
@@id([trace_id, trace_ref])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,30 @@
|
|||
from collections.abc import Callable
|
||||
import asyncio
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ValidationError
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.lens.endpoints import (
|
||||
claim_due,
|
||||
get_signals,
|
||||
list_agents,
|
||||
put_signals,
|
||||
read_reviews,
|
||||
result,
|
||||
run_settings,
|
||||
run_window,
|
||||
trace_findings,
|
||||
trace_signal_statuses,
|
||||
user_scope,
|
||||
validate_model,
|
||||
validate_signal_model,
|
||||
watchable,
|
||||
watching,
|
||||
worker_supports_model,
|
||||
|
|
@ -43,6 +48,7 @@ from litellm.proxy.lens.models import (
|
|||
Worker,
|
||||
)
|
||||
from litellm.proxy.lens.repository import DueLens, Row
|
||||
from litellm.proxy.lens.signals import SignalConfig, StoredTraceSignal
|
||||
from litellm.proxy.lens.state import claim_job, queue_job, replace_job
|
||||
from litellm.rust_bridge.trace.generated.models import ExecutionRow, LensSampleParams
|
||||
from tests.unit.proxy.lens.test_agent_workspace import execution
|
||||
|
|
@ -71,6 +77,46 @@ class ResultDatabase:
|
|||
return len(self.completed)
|
||||
|
||||
|
||||
class SignalStatusDatabase:
|
||||
def __init__(self, config: SignalConfig, rows: Mapping[str, StoredTraceSignal]) -> None:
|
||||
self.config: Final = config
|
||||
self.rows: Final = rows
|
||||
self.saved: Final[asyncio.Queue[tuple[object, ...]]] = asyncio.Queue()
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> object:
|
||||
if '"LiteLLM_LensSignalConfig"' in query:
|
||||
return ({"data": self.config.model_dump(mode="json")},)
|
||||
payload: Final = args[0]
|
||||
assert isinstance(payload, str)
|
||||
requested: Final = TypeAdapter(tuple[TraceIdentity, ...]).validate_json(payload)
|
||||
return tuple(
|
||||
{"data": row.model_dump(mode="json")}
|
||||
for identity in requested
|
||||
if (row := self.rows.get(identity.trace_id)) is not None
|
||||
)
|
||||
|
||||
async def execute_raw(self, query: str, *args: object) -> int:
|
||||
await self.saved.put(args)
|
||||
return 1
|
||||
|
||||
|
||||
def signal_router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "decision",
|
||||
"litellm_params": {"model": "openai/test-decision", "api_key": "test-key"},
|
||||
"model_info": {"mode": "evaluation"},
|
||||
},
|
||||
{
|
||||
"model_name": "chat",
|
||||
"litellm_params": {"model": "openai/test-chat", "api_key": "test-key"},
|
||||
"model_info": {"mode": "chat"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_sample_retries_oversized_pages_and_keeps_all_executions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -469,6 +515,144 @@ async def test_trace_finding_counts_require_investigation_read_access() -> None:
|
|||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signal_endpoints_return_statuses_in_request_order_for_admin_viewers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
config: Final = SignalConfig(model="decision")
|
||||
rows: Final = {
|
||||
"pending": StoredTraceSignal(
|
||||
trace_id="pending",
|
||||
config_key=config.key(),
|
||||
span_count=1,
|
||||
claimed_until=NOW + timedelta(minutes=1),
|
||||
data={"status": "pending", "scores": {}, "model": "decision", "error": ""},
|
||||
),
|
||||
"classified": StoredTraceSignal(
|
||||
trace_id="classified",
|
||||
config_key=config.key(),
|
||||
span_count=1,
|
||||
classified_at=NOW,
|
||||
data={
|
||||
"status": "classified",
|
||||
"scores": {"user_frustration": 0.7, "missing_capability": 0.8},
|
||||
"model": "decision",
|
||||
"error": "",
|
||||
},
|
||||
),
|
||||
"failed": StoredTraceSignal(
|
||||
trace_id="failed",
|
||||
config_key=config.key(),
|
||||
span_count=1,
|
||||
classified_at=NOW,
|
||||
data={"status": "failed", "scores": {}, "model": "decision", "error": "classification failed"},
|
||||
),
|
||||
"stale": StoredTraceSignal(
|
||||
trace_id="stale",
|
||||
config_key="old-config",
|
||||
span_count=1,
|
||||
classified_at=NOW,
|
||||
data={
|
||||
"status": "classified",
|
||||
"scores": {"user_frustration": 1.0},
|
||||
"model": "old",
|
||||
"error": "",
|
||||
},
|
||||
),
|
||||
}
|
||||
database: Final = SignalStatusDatabase(config, rows)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=database))
|
||||
viewer: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
request: Final = TraceFindingsRequest(
|
||||
traces=tuple(
|
||||
TraceIdentity(trace_id=trace_id) for trace_id in ("failed", "classified", "missing", "pending", "stale")
|
||||
)
|
||||
)
|
||||
|
||||
assert await get_signals(viewer) == config
|
||||
results: Final = await trace_signal_statuses(request, viewer)
|
||||
|
||||
assert tuple((result.trace_id, result.status) for result in results) == (
|
||||
("failed", "failed"),
|
||||
("classified", "classified"),
|
||||
("missing", "unclassified"),
|
||||
("pending", "pending"),
|
||||
("stale", "unclassified"),
|
||||
)
|
||||
assert tuple((flag.signal_id, flag.name, flag.score) for flag in results[1].flags) == (
|
||||
("missing_capability", "Missing capability", 0.8),
|
||||
("user_frustration", "User frustration", 0.7),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_signal_endpoints_require_connected_postgres(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await get_signals(auth)
|
||||
|
||||
assert error.value.status_code == 503
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_put_signals_saves_config_for_admin(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
database: Final = SignalStatusDatabase(SignalConfig(), {})
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=database))
|
||||
monkeypatch.setattr(proxy_server, "llm_router", signal_router())
|
||||
auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
body: Final = SignalConfig(model="decision", threshold=0.7)
|
||||
|
||||
assert await put_signals(body, auth) == body
|
||||
|
||||
saved: Final = await database.saved.get()
|
||||
assert saved[0] == "global"
|
||||
assert isinstance(saved[1], str)
|
||||
assert SignalConfig.model_validate_json(saved[1]) == body
|
||||
|
||||
|
||||
def test_signal_model_requires_a_ready_router() -> None:
|
||||
with pytest.raises(HTTPException) as error:
|
||||
validate_signal_model(SignalConfig(model="decision"), None)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("role", (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY))
|
||||
async def test_put_signals_rejects_non_admin_roles(role: LitellmUserRoles) -> None:
|
||||
auth: Final = UserAPIKeyAuth(user_role=role)
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await put_signals(SignalConfig(model="decision"), auth)
|
||||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ("chat", "unconfigured"))
|
||||
async def test_put_signals_rejects_chat_and_unknown_model_groups(monkeypatch: pytest.MonkeyPatch, model: str) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", signal_router())
|
||||
auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await put_signals(SignalConfig(model=model), auth)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert error.value.detail == "Choose a System 1 model (evaluation mode) configured on this proxy"
|
||||
|
||||
|
||||
def test_signal_model_accepts_only_evaluation_mode_groups() -> None:
|
||||
assert validate_signal_model(SignalConfig(model="decision"), signal_router()) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
(LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.TEAM),
|
||||
|
|
|
|||
1002
tests/unit/proxy/lens/test_signals.py
Normal file
1002
tests/unit/proxy/lens/test_signals.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -291,7 +291,7 @@ describe("Lens interactive demo", () => {
|
|||
await expectUrl(onUrlUpdate, (url) => expect(url.get("tab")).toBe("settings"));
|
||||
expect(screen.queryByRole("dialog")).not.toBeInTheDocument();
|
||||
const panel = within(screen.getByRole("region", { name: "Settings" }));
|
||||
expect(panel.getByRole("status")).toHaveTextContent("Tracing enabled");
|
||||
expect(panel.getByText("Tracing enabled", { exact: true })).toBeVisible();
|
||||
expect(panel.getByRole("heading", { name: "Analysis worker" })).toBeVisible();
|
||||
expect(panel.getByRole("heading", { name: worker.name })).toBeVisible();
|
||||
expect(panel.getByText("Connected")).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -196,6 +196,7 @@ function LensContent({ userRole, readOnly }: Omit<WorkspaceProps, "accessToken">
|
|||
readOnly={readOnly}
|
||||
canMintTracingKey={isAdmin}
|
||||
canViewFindings={canViewInvestigations}
|
||||
onSetUpSignals={canConfigure ? showSettings : undefined}
|
||||
/>
|
||||
</TabsContent>
|
||||
<TabsContent value="findings" className={PANEL}>
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ function demoLensApi(data: LensDemoData): LensApi {
|
|||
saveLens: readOnly,
|
||||
startRun: readOnly,
|
||||
watchAll: async () => ({ watching: [], skipped: [] }),
|
||||
signalConfig: async () => ({ model: "", threshold: 0.5, signals: [] }),
|
||||
saveSignalConfig: readOnly,
|
||||
cancelRun: readOnly,
|
||||
reviewFinding: readOnly,
|
||||
registerWorker: readOnly,
|
||||
|
|
@ -90,6 +92,8 @@ function demoTracesApi(data: LensDemoData): TracesApi {
|
|||
);
|
||||
return { ...trace, finding_count: assessed.length ? findings.size : null };
|
||||
}),
|
||||
signals: async (traces) =>
|
||||
traces.map((trace) => ({ ...trace, status: "unclassified" as const, flags: [], model: "", classified_at: null })),
|
||||
anyRecorded: async () => data.runs.length > 0,
|
||||
trace: (traceId) => found(run(traceId)?.trace),
|
||||
span: (traceId, spanId) => found(run(traceId)?.details.find((span) => span.span_id === spanId)),
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ export const lensKeys = {
|
|||
models: (scope: string) => [...lensKeys.all, "models", { scope }] as const,
|
||||
modelDetails: (scope: string) => [...lensKeys.all, "model-details", { scope }] as const,
|
||||
activity: (scope: string) => [...lensKeys.all, "activity-available", { scope }] as const,
|
||||
signalConfig: (scope: string) => [...lensKeys.all, "signal-config", { scope }] as const,
|
||||
discoveries: () => [...lensKeys.all, "discovery"] as const,
|
||||
discovery: (scope: string, source: Settings["source"], hours: number | undefined) =>
|
||||
[...lensKeys.discoveries(), { scope, source, hours }] as const,
|
||||
|
|
@ -46,6 +47,13 @@ export const lensQueries = {
|
|||
modelDetails(api: LensApi) {
|
||||
return queryOptions({ queryKey: lensKeys.modelDetails(api.scope), queryFn: () => api.modelDetails() });
|
||||
},
|
||||
signalConfig(api: LensApi) {
|
||||
return queryOptions({
|
||||
queryKey: lensKeys.signalConfig(api.scope),
|
||||
queryFn: () => api.signalConfig(),
|
||||
staleTime: 5000,
|
||||
});
|
||||
},
|
||||
activity(api: LensApi, loaded: boolean) {
|
||||
const options = {
|
||||
queryKey: lensKeys.activity(api.scope),
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type {
|
|||
RunWindow,
|
||||
Sample,
|
||||
Settings,
|
||||
SignalConfig,
|
||||
WorkerCreated,
|
||||
} from "../model/types";
|
||||
|
||||
|
|
@ -62,6 +63,8 @@ export interface LensApi {
|
|||
saveLens(id: string | undefined, settings: Settings): Promise<Lens>;
|
||||
startRun(lensId: string, request?: RunWindow): Promise<void>;
|
||||
watchAll(): Promise<components["schemas"]["WatchAllResult"]>;
|
||||
signalConfig(): Promise<SignalConfig>;
|
||||
saveSignalConfig(config: SignalConfig): Promise<SignalConfig>;
|
||||
cancelRun(lensId: string): Promise<void>;
|
||||
reviewFinding(lensId: string, findingId: string, status: FindingStatus, reason: string): Promise<void>;
|
||||
registerWorker(analysisKeyId: string): Promise<WorkerCreated>;
|
||||
|
|
@ -168,6 +171,8 @@ export function liveLensApi(client: LensClient, apiClient: ApiClient, accessToke
|
|||
),
|
||||
startRun: (lensId, request = {}) => sent(client.POST("/lens/{lens_id}/runs", { ...lens(lensId), body: request })),
|
||||
watchAll: () => required(client.POST("/lens/watch-all", { headers })),
|
||||
signalConfig: () => required(client.GET("/lens/signals", { headers })),
|
||||
saveSignalConfig: (config) => required(client.PUT("/lens/signals", { headers, body: config })),
|
||||
cancelRun: (lensId) => sent(client.POST("/lens/{lens_id}/cancel", lens(lensId))),
|
||||
reviewFinding: (lensId, findingId, status, reason) =>
|
||||
sent(
|
||||
|
|
|
|||
74
ui/litellm-dashboard/src/components/lens/model/signals.ts
Normal file
74
ui/litellm-dashboard/src/components/lens/model/signals.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import type { AnalysisModelInfo, SignalConfig } from "./types";
|
||||
|
||||
export const SYSTEM_ONE_MODE = "evaluation";
|
||||
|
||||
export const signalsConfigured = (config: SignalConfig): boolean =>
|
||||
Boolean(config.model) && (config.signals?.length ?? 0) > 0;
|
||||
|
||||
export const systemOneModels = (details: readonly AnalysisModelInfo[]): AnalysisModelInfo[] =>
|
||||
details
|
||||
.filter((info) => info.mode === SYSTEM_ONE_MODE)
|
||||
.toSorted((a, b) => a.model_group.localeCompare(b.model_group));
|
||||
|
||||
export interface LibrarySignal {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly summary: string;
|
||||
readonly question: string;
|
||||
}
|
||||
|
||||
export const SIGNAL_LIBRARY: readonly LibrarySignal[] = [
|
||||
{
|
||||
id: "user_frustration",
|
||||
name: "User frustration",
|
||||
summary: "annoyed, complaining or giving up",
|
||||
question:
|
||||
"Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example complaints, irritated corrections, all caps, profanity, or giving up on the task?",
|
||||
},
|
||||
{
|
||||
id: "missing_capability",
|
||||
name: "Missing capability",
|
||||
summary: "asked for something the agent can't do",
|
||||
question:
|
||||
"Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it lacks a tool, permission, integration or data source, or fails because the capability does not exist?",
|
||||
},
|
||||
{
|
||||
id: "repeated_request",
|
||||
name: "Repeated request",
|
||||
summary: "had to ask for the same thing again",
|
||||
question:
|
||||
"Does the user ask for the same thing more than once in this run, usually because the agent did not deliver it the first time?",
|
||||
},
|
||||
{
|
||||
id: "asked_for_human",
|
||||
name: "Asked for a human",
|
||||
summary: "wants a person, not the agent",
|
||||
question: "Does the user ask to talk to a human, a support person or a manager instead of the agent in this run?",
|
||||
},
|
||||
{
|
||||
id: "tool_failure",
|
||||
name: "Tool failure",
|
||||
summary: "a tool or step failed and stayed broken",
|
||||
question:
|
||||
"Does a tool call or step fail in this run with an error, exception or timeout that the agent does not recover from?",
|
||||
},
|
||||
{
|
||||
id: "refused_request",
|
||||
name: "Refused request",
|
||||
summary: "declined a reasonable ask",
|
||||
question: "Does the agent refuse or decline a reasonable, allowed user request in this run?",
|
||||
},
|
||||
{
|
||||
id: "made_up_answer",
|
||||
name: "Made-up answer",
|
||||
summary: "facts or links with no source",
|
||||
question:
|
||||
"Does the agent state facts, numbers, links or tool results in this run that are not supported by the conversation or by any tool output?",
|
||||
},
|
||||
{
|
||||
id: "task_abandoned",
|
||||
name: "Task abandoned",
|
||||
summary: "run ended without what was asked",
|
||||
question: "Does the run end without the user getting what they asked for?",
|
||||
},
|
||||
];
|
||||
|
|
@ -5,6 +5,8 @@ export type Lens = components["schemas"]["Lens"];
|
|||
export type Settings = components["schemas"]["LensSettings"];
|
||||
|
||||
export type LensList = components["schemas"]["LensList"];
|
||||
export type SignalConfig = components["schemas"]["SignalConfig"];
|
||||
export type Signal = NonNullable<SignalConfig["signals"]>[number];
|
||||
|
||||
export type Finding = components["schemas"]["Finding"];
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { Activity, ArrowUpRight } from "lucide-react";
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { StatusDot } from "@/components/shared/StatusDot";
|
||||
import { WorkerSettings } from "./worker/WorkerSettings";
|
||||
import { SignalSettings } from "./signals/SignalSettings";
|
||||
import { SettingsCard, SettingsSection } from "./SettingsSection";
|
||||
import type { LensList } from "../model/types";
|
||||
|
||||
|
|
@ -53,6 +54,7 @@ export function LensSettings({
|
|||
return (
|
||||
<div aria-label="Settings" role="region" className="flex w-full flex-col divide-y divide-border">
|
||||
<TracingSection enabled={list.tracing_enabled} onOpenTraces={onOpenTraces} />
|
||||
<SignalSettings />
|
||||
<SettingsSection
|
||||
heading="Analysis worker"
|
||||
description="Runs investigations on your server and bills model usage to an analysis key."
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
import { fireEvent, screen } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { renderWithLens } from "@/../tests/lens-test-utils";
|
||||
import { testQueryClient } from "@/../tests/test-utils";
|
||||
|
||||
import type { SignalConfig } from "../../model/types";
|
||||
import { SignalForm } from "./SignalSettings";
|
||||
|
||||
const saved: SignalConfig = {
|
||||
model: "jev",
|
||||
threshold: 0.5,
|
||||
signals: [{ id: "user_frustration", name: "User frustration", question: "Is the user frustrated?" }],
|
||||
};
|
||||
|
||||
const updated: SignalConfig = {
|
||||
...saved,
|
||||
model: "new-jev",
|
||||
threshold: 0.8,
|
||||
};
|
||||
|
||||
const network = vi.fn<typeof fetch>();
|
||||
|
||||
describe("signal settings", () => {
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
network.mockReset();
|
||||
network.mockImplementation(async (input) => {
|
||||
const path = new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname;
|
||||
return Response.json(path === "/model_group/info" ? { data: [] } : {});
|
||||
});
|
||||
vi.stubGlobal("fetch", network);
|
||||
});
|
||||
|
||||
it("updates a clean draft when saved settings change", async () => {
|
||||
const { rerender } = renderWithLens(<SignalForm saved={saved} />);
|
||||
const model = await screen.findByRole("combobox", { name: "System 1 model" });
|
||||
|
||||
expect(model).toHaveValue("jev");
|
||||
rerender(<SignalForm saved={updated} />);
|
||||
|
||||
expect(model).toHaveValue("new-jev");
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps dirty edits until the user loads the latest settings", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = renderWithLens(<SignalForm saved={saved} />);
|
||||
const threshold = await screen.findByRole("spinbutton", { name: "Flag at score" });
|
||||
|
||||
fireEvent.change(threshold, { target: { value: "70" } });
|
||||
rerender(<SignalForm saved={updated} />);
|
||||
|
||||
expect(threshold).toHaveValue(70);
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("Signals were changed elsewhere");
|
||||
await user.click(screen.getByRole("button", { name: "Load latest" }));
|
||||
|
||||
expect(threshold).toHaveValue(80);
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps a custom Tool failure signal separate while toggling the library signal", async () => {
|
||||
const user = userEvent.setup();
|
||||
const customQuestion = "Does this custom failure condition apply?";
|
||||
const customSignal = { id: "tool_failure", name: "Tool failure", question: customQuestion };
|
||||
const customConfig: SignalConfig = {
|
||||
...saved,
|
||||
signals: [customSignal],
|
||||
};
|
||||
renderWithLens(<SignalForm saved={customConfig} />);
|
||||
|
||||
const question = await screen.findByRole("textbox", { name: "Question for Tool failure" });
|
||||
const model = await screen.findByRole("combobox", { name: "System 1 model" });
|
||||
const threshold = screen.getByRole("spinbutton", { name: "Flag at score" });
|
||||
expect(question).toHaveValue(customQuestion);
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
|
||||
expect(model).toHaveValue("jev");
|
||||
expect(threshold).toHaveValue(50);
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /^Tool failure/ }));
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
|
||||
expect(question).toHaveValue(customQuestion);
|
||||
|
||||
const editedQuestion = "Does this edited custom failure condition apply?";
|
||||
fireEvent.change(question, { target: { value: editedQuestion } });
|
||||
expect(question).toHaveValue(editedQuestion);
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Remove Tool failure" }));
|
||||
expect(screen.queryByRole("textbox", { name: "Question for Tool failure" })).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "true");
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /^Tool failure/ }));
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
|
||||
expect(screen.getByText("Pick at least one signal to flag traces")).toBeInTheDocument();
|
||||
expect(model).toHaveValue("jev");
|
||||
expect(threshold).toHaveValue(50);
|
||||
});
|
||||
|
||||
it("removes only the library signal when toggling it off beside a custom signal", async () => {
|
||||
const user = userEvent.setup();
|
||||
const customQuestion = "Does this custom failure condition apply?";
|
||||
const customConfig: SignalConfig = {
|
||||
...saved,
|
||||
signals: [{ id: "tool_failure", name: "Tool failure", question: customQuestion }],
|
||||
};
|
||||
renderWithLens(<SignalForm saved={customConfig} />);
|
||||
|
||||
const question = await screen.findByRole("textbox", { name: "Question for Tool failure" });
|
||||
const tile = screen.getByRole("button", { name: /^Tool failure/ });
|
||||
|
||||
await user.click(tile);
|
||||
expect(tile).toHaveAttribute("aria-pressed", "true");
|
||||
await user.click(tile);
|
||||
|
||||
expect(screen.getByRole("button", { name: /^Tool failure/ })).toHaveAttribute("aria-pressed", "false");
|
||||
expect(question).toHaveValue(customQuestion);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
"use client";
|
||||
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { Flag, Trash2 } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { useId, useState } from "react";
|
||||
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { StatusDot } from "@/components/shared/StatusDot";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
|
||||
import { useLensApi } from "../../data/LensServices";
|
||||
import { lensKeys, lensQueries } from "../../data/queries";
|
||||
import { SIGNAL_LIBRARY, signalsConfigured, systemOneModels } from "../../model/signals";
|
||||
import { WatchPicker } from "../../setup/WatchPicker";
|
||||
import type { SignalConfig } from "../../model/types";
|
||||
import { SettingsCard, SettingsSection } from "../SettingsSection";
|
||||
import {
|
||||
MAX_SIGNALS,
|
||||
configFrom,
|
||||
draftFrom,
|
||||
draftProblems,
|
||||
newRow,
|
||||
type SignalDraft,
|
||||
type SignalRow,
|
||||
} from "./signalDraft";
|
||||
|
||||
const LIBRARY_QUESTIONS: ReadonlySet<string> = new Set(SIGNAL_LIBRARY.map((signal) => signal.question));
|
||||
|
||||
export function SignalSettings() {
|
||||
return (
|
||||
<SettingsSection
|
||||
heading="Signals"
|
||||
description="Set once for every trace. A System 1 model checks each run and Traces flags matches in red."
|
||||
>
|
||||
<SignalConfigLoader />
|
||||
</SettingsSection>
|
||||
);
|
||||
}
|
||||
|
||||
function SignalConfigLoader() {
|
||||
const api = useLensApi();
|
||||
const config = useQuery(lensQueries.signalConfig(api));
|
||||
if (config.error)
|
||||
return (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
Could not load signals: {config.error.message}
|
||||
</p>
|
||||
);
|
||||
if (!config.data) return <Skeleton aria-label="Loading signals" className="h-40 w-full" />;
|
||||
return <SignalForm saved={config.data} />;
|
||||
}
|
||||
|
||||
const FieldError = ({ children }: { children?: string }) =>
|
||||
children ? <p className="mt-1 text-xs text-destructive">{children}</p> : null;
|
||||
|
||||
function SetupCallout({ hasModels }: { hasModels: boolean }) {
|
||||
return (
|
||||
<div className="flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm">
|
||||
<Flag aria-hidden="true" className="mt-0.5 size-4 shrink-0 text-destructive" />
|
||||
<div className="space-y-1">
|
||||
<p className="font-medium">Choose a System 1 model to start flagging traces</p>
|
||||
{hasModels ? (
|
||||
<p className="text-xs text-muted-foreground">Pick one of the evaluation models on this proxy below.</p>
|
||||
) : (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
This proxy has no System 1 models yet. Add one with mode evaluation, for example typesafe/jev-latest, on{" "}
|
||||
<Link
|
||||
href={uiHref("models-and-endpoints")}
|
||||
className="font-medium text-foreground underline underline-offset-2"
|
||||
>
|
||||
Models + Endpoints
|
||||
</Link>
|
||||
.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SignalFields({
|
||||
row,
|
||||
problems,
|
||||
onChange,
|
||||
onRemove,
|
||||
}: {
|
||||
row: SignalRow;
|
||||
problems?: { name?: string; question?: string };
|
||||
onChange: (row: SignalRow) => void;
|
||||
onRemove: () => void;
|
||||
}) {
|
||||
const label = row.name.trim() || "new signal";
|
||||
return (
|
||||
<li className="grid gap-2 rounded-md border border-border p-3 sm:grid-cols-[200px_minmax(0,1fr)_auto]">
|
||||
<div>
|
||||
<Input
|
||||
aria-label="Signal name"
|
||||
placeholder="User frustration"
|
||||
value={row.name}
|
||||
aria-invalid={Boolean(problems?.name)}
|
||||
onChange={(event) => onChange({ ...row, name: event.target.value })}
|
||||
/>
|
||||
<FieldError>{problems?.name}</FieldError>
|
||||
</div>
|
||||
<div>
|
||||
<Textarea
|
||||
aria-label={`Question for ${label}`}
|
||||
placeholder="Does the user show frustration with the agent in this run?"
|
||||
rows={2}
|
||||
value={row.question}
|
||||
aria-invalid={Boolean(problems?.question)}
|
||||
onChange={(event) => onChange({ ...row, question: event.target.value })}
|
||||
/>
|
||||
<FieldError>{problems?.question}</FieldError>
|
||||
</div>
|
||||
<Button variant="ghost" size="icon-sm" aria-label={`Remove ${label}`} onClick={onRemove}>
|
||||
<Trash2 aria-hidden="true" />
|
||||
</Button>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
export function SignalForm({ saved }: { saved: SignalConfig }) {
|
||||
const api = useLensApi();
|
||||
const queryClient = useQueryClient();
|
||||
const modelId = useId();
|
||||
const thresholdId = useId();
|
||||
const [draft, setDraft] = useState<SignalDraft>(() => draftFrom(saved));
|
||||
const [draftBase, setDraftBase] = useState<SignalConfig>(() => saved);
|
||||
const [savedElsewhere, setSavedElsewhere] = useState(false);
|
||||
const savedKey = JSON.stringify(saved);
|
||||
const draftBaseKey = JSON.stringify(draftBase);
|
||||
if (savedKey !== draftBaseKey) {
|
||||
const draftIsDirty = JSON.stringify(configFrom(draft)) !== JSON.stringify(configFrom(draftFrom(draftBase)));
|
||||
setDraftBase(saved);
|
||||
if (draftIsDirty) {
|
||||
setSavedElsewhere(true);
|
||||
} else {
|
||||
setDraft(draftFrom(saved));
|
||||
setSavedElsewhere(false);
|
||||
}
|
||||
}
|
||||
const details = useQuery(lensQueries.modelDetails(api));
|
||||
const models = systemOneModels(details.data?.data ?? []);
|
||||
const options = models.map((info) => ({
|
||||
value: info.model_group,
|
||||
label: info.model_group,
|
||||
sublabel: info.providers.join(", "),
|
||||
}));
|
||||
const problems = draftProblems(draft);
|
||||
const next = configFrom(draft);
|
||||
const dirty = JSON.stringify(next) !== JSON.stringify(configFrom(draftFrom(draftBase)));
|
||||
const save = useMutation({
|
||||
mutationFn: (config: SignalConfig) => api.saveSignalConfig(config),
|
||||
onSuccess: (config) => {
|
||||
queryClient.setQueryData(lensKeys.signalConfig(api.scope), config);
|
||||
setDraft(draftFrom(config));
|
||||
setDraftBase(config);
|
||||
setSavedElsewhere(false);
|
||||
void queryClient.invalidateQueries({ queryKey: ["traceSignals"] });
|
||||
},
|
||||
});
|
||||
const setRows = (rows: readonly SignalRow[]) => setDraft((current) => ({ ...current, rows }));
|
||||
const addRow = () => setRows([...draft.rows, newRow(crypto.randomUUID())]);
|
||||
const active = signalsConfigured(saved);
|
||||
const custom = draft.rows.filter((row) => !LIBRARY_QUESTIONS.has(row.question));
|
||||
const picked = new Set(
|
||||
SIGNAL_LIBRARY.filter((signal) => draft.rows.some((row) => row.question === signal.question)).map(
|
||||
(signal) => signal.id,
|
||||
),
|
||||
);
|
||||
const pick = (next: ReadonlySet<string>) =>
|
||||
setRows([
|
||||
...SIGNAL_LIBRARY.filter((signal) => next.has(signal.id)).map(
|
||||
(signal) =>
|
||||
draft.rows.find((row) => row.question === signal.question) ?? {
|
||||
key: crypto.randomUUID(),
|
||||
id: draft.rows.some((row) => row.id === signal.id) ? "" : signal.id,
|
||||
name: signal.name,
|
||||
question: signal.question,
|
||||
},
|
||||
),
|
||||
...custom,
|
||||
]);
|
||||
const loadLatest = () => {
|
||||
setDraft(draftFrom(saved));
|
||||
setDraftBase(saved);
|
||||
setSavedElsewhere(false);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<SettingsCard className="space-y-4">
|
||||
<p role="status" className="inline-flex items-center gap-2 text-sm">
|
||||
<StatusDot state={active ? "ok" : "off"} />
|
||||
{active ? `Flagging traces with ${saved.model}` : "Signals are off"}
|
||||
</p>
|
||||
{!saved.model && !details.isPending && <SetupCallout hasModels={models.length > 0} />}
|
||||
{savedElsewhere && (
|
||||
<div role="alert" className="flex items-center justify-between gap-3 rounded-md border p-3 text-sm">
|
||||
<span>Signals were changed elsewhere</span>
|
||||
<Button type="button" variant="outline" size="sm" onClick={loadLatest}>
|
||||
Load latest
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<div className="grid gap-4 sm:grid-cols-[minmax(0,1fr)_160px]">
|
||||
<div className="space-y-1.5">
|
||||
<label htmlFor={modelId} className="text-sm font-medium">
|
||||
System 1 model
|
||||
</label>
|
||||
<SearchSelect
|
||||
inputId={modelId}
|
||||
aria-label="System 1 model"
|
||||
options={options}
|
||||
value={draft.model}
|
||||
onValueChange={(value) => setDraft((current) => ({ ...current, model: value ?? "" }))}
|
||||
placeholder={details.isPending ? "Loading models…" : "Choose a System 1 model"}
|
||||
disabled={details.isPending}
|
||||
emptyText="No System 1 models on this proxy"
|
||||
allowClear
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Decisions API models onboarded with mode evaluation, such as TypeSafe JEV
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<label htmlFor={thresholdId} className="text-sm font-medium">
|
||||
Flag at score
|
||||
</label>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<Input
|
||||
id={thresholdId}
|
||||
type="number"
|
||||
inputMode="numeric"
|
||||
min={5}
|
||||
max={95}
|
||||
step={5}
|
||||
value={Number.isNaN(draft.thresholdPercent) ? "" : draft.thresholdPercent}
|
||||
aria-invalid={Boolean(problems.threshold)}
|
||||
onChange={(event) =>
|
||||
setDraft((current) => ({ ...current, thresholdPercent: event.target.valueAsNumber }))
|
||||
}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">%</span>
|
||||
</div>
|
||||
<FieldError>{problems.threshold}</FieldError>
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-4 border-t border-border pt-4">
|
||||
<WatchPicker
|
||||
label="Flag runs where"
|
||||
options={SIGNAL_LIBRARY}
|
||||
selected={picked}
|
||||
onChange={pick}
|
||||
onAddCustom={addRow}
|
||||
addDisabled={draft.rows.length >= MAX_SIGNALS}
|
||||
/>
|
||||
{custom.length > 0 && (
|
||||
<ul aria-label="Custom signals" className="space-y-2">
|
||||
{custom.map((row) => (
|
||||
<SignalFields
|
||||
key={row.key}
|
||||
row={row}
|
||||
problems={problems.rows.get(row.key)}
|
||||
onChange={(changed) => setRows(draft.rows.map((other) => (other.key === row.key ? changed : other)))}
|
||||
onRemove={() => setRows(draft.rows.filter((other) => other.key !== row.key))}
|
||||
/>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{draft.rows.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">Pick at least one signal to flag traces</p>
|
||||
)}
|
||||
<FieldError>{problems.signals}</FieldError>
|
||||
</div>
|
||||
<div className="flex items-center justify-end gap-3 border-t border-border pt-4">
|
||||
{save.error && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
Could not save signals: {save.error.message}
|
||||
</p>
|
||||
)}
|
||||
{save.isSuccess && !dirty && <p className="text-xs text-muted-foreground">Saved</p>}
|
||||
<Button disabled={!dirty || problems.any || save.isPending} onClick={() => save.mutate(next)}>
|
||||
{save.isPending ? "Saving…" : "Save signals"}
|
||||
</Button>
|
||||
</div>
|
||||
</SettingsCard>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { SIGNAL_LIBRARY } from "../../model/signals";
|
||||
import { configFrom, draftFrom, draftProblems, newRow, signalIds } from "./signalDraft";
|
||||
|
||||
const saved = {
|
||||
model: "jev",
|
||||
threshold: 0.5,
|
||||
signals: [{ id: "user_frustration", name: "User frustration", question: "Is the user frustrated?" }],
|
||||
};
|
||||
|
||||
describe("signal drafts", () => {
|
||||
it("round-trips a saved config unchanged", () => {
|
||||
expect(configFrom(draftFrom(saved))).toEqual(saved);
|
||||
});
|
||||
|
||||
it("keeps saved IDs on rename and derives unique IDs for new signals", () => {
|
||||
const rows = [
|
||||
{ ...draftFrom(saved).rows[0], name: "Annoyed user" },
|
||||
{ ...newRow("a"), name: "Repeat request!" },
|
||||
{ ...newRow("b"), name: "Repeat request?" },
|
||||
{ ...newRow("c"), name: "2nd try" },
|
||||
];
|
||||
expect(signalIds(rows)).toEqual(["user_frustration", "repeat_request", "repeat_request_2", "signal_2nd_try"]);
|
||||
});
|
||||
|
||||
it("does not assign a library ID to a new custom question", () => {
|
||||
const rows = [{ ...newRow("custom"), name: "Tool failure", question: "Does this custom signal apply?" }];
|
||||
|
||||
expect(signalIds(rows)[0]).not.toBe("tool_failure");
|
||||
});
|
||||
|
||||
it("keeps custom and library signals distinct when their names match", () => {
|
||||
const toolFailure = SIGNAL_LIBRARY.find((signal) => signal.id === "tool_failure");
|
||||
if (!toolFailure) throw new Error("Tool failure is missing from the signal library");
|
||||
const customQuestion = "Does this custom signal apply?";
|
||||
const draft = {
|
||||
model: "jev",
|
||||
thresholdPercent: 50,
|
||||
rows: [
|
||||
{ ...newRow("library"), name: toolFailure.name, question: toolFailure.question },
|
||||
{ ...newRow("custom"), id: toolFailure.id, name: toolFailure.name, question: customQuestion },
|
||||
],
|
||||
};
|
||||
|
||||
expect(configFrom(draft).signals).toEqual([
|
||||
{ id: "tool_failure_2", name: "Tool failure", question: toolFailure.question },
|
||||
{ id: "tool_failure", name: "Tool failure", question: customQuestion },
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports blank, duplicate and out of range fields", () => {
|
||||
const draft = {
|
||||
model: "",
|
||||
thresholdPercent: 99,
|
||||
rows: [
|
||||
{ ...newRow("a"), name: "Loop", question: "Does the agent loop?" },
|
||||
{ ...newRow("b"), name: " loop ", question: "?" },
|
||||
newRow("c"),
|
||||
],
|
||||
};
|
||||
const problems = draftProblems(draft);
|
||||
expect(problems.any).toBe(true);
|
||||
expect(problems.threshold).toBe("Use a whole number from 5 to 95");
|
||||
expect(problems.rows.get("a")).toBeUndefined();
|
||||
expect(problems.rows.get("b")).toEqual({
|
||||
name: "Another signal has this name",
|
||||
question: "Ask a yes or no question about the run",
|
||||
});
|
||||
expect(problems.rows.get("c")?.name).toBe("Name the signal");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
import type { SignalConfig } from "../../model/types";
|
||||
import { SIGNAL_LIBRARY } from "../../model/signals";
|
||||
|
||||
export const MAX_SIGNALS = 20;
|
||||
const MAX_NAME = 60;
|
||||
const MAX_QUESTION = 500;
|
||||
const MAX_ID = 64;
|
||||
|
||||
export interface SignalRow {
|
||||
readonly key: string;
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly question: string;
|
||||
}
|
||||
|
||||
export interface SignalDraft {
|
||||
readonly model: string;
|
||||
readonly thresholdPercent: number;
|
||||
readonly rows: readonly SignalRow[];
|
||||
}
|
||||
|
||||
export interface RowProblems {
|
||||
readonly name?: string;
|
||||
readonly question?: string;
|
||||
}
|
||||
|
||||
export interface DraftProblems {
|
||||
readonly threshold?: string;
|
||||
readonly signals?: string;
|
||||
readonly rows: ReadonlyMap<string, RowProblems>;
|
||||
readonly any: boolean;
|
||||
}
|
||||
|
||||
export const draftFrom = (config: SignalConfig): SignalDraft => ({
|
||||
model: config.model ?? "",
|
||||
thresholdPercent: Math.round((config.threshold ?? 0.5) * 100),
|
||||
rows: (config.signals ?? []).map((signal) => ({
|
||||
key: signal.id,
|
||||
id: signal.id,
|
||||
name: signal.name,
|
||||
question: signal.question,
|
||||
})),
|
||||
});
|
||||
|
||||
export const newRow = (key: string): SignalRow => ({ key, id: "", name: "", question: "" });
|
||||
|
||||
const slug = (name: string): string => {
|
||||
const words = name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "_")
|
||||
.replace(/^_+|_+$/g, "");
|
||||
const lettered = /^[a-z]/.test(words) ? words : `signal_${words}`.replace(/_+$/, "");
|
||||
return lettered.slice(0, MAX_ID);
|
||||
};
|
||||
|
||||
const uniqueId = (base: string, taken: ReadonlySet<string>, attempt = 1): string => {
|
||||
const suffix = attempt === 1 ? "" : `_${attempt}`;
|
||||
const id = `${base.slice(0, MAX_ID - suffix.length)}${suffix}`;
|
||||
return taken.has(id) ? uniqueId(base, taken, attempt + 1) : id;
|
||||
};
|
||||
|
||||
export const signalIds = (rows: readonly SignalRow[]): string[] => {
|
||||
const savedIds: ReadonlySet<string> = new Set(rows.flatMap((row) => (row.id ? [row.id] : [])));
|
||||
return rows.reduce<string[]>((ids, row) => {
|
||||
if (row.id) return [...ids, row.id];
|
||||
const librarySignal = SIGNAL_LIBRARY.find((signal) => signal.question === row.question);
|
||||
const libraryIds = new Set(
|
||||
SIGNAL_LIBRARY.filter((signal) => signal.id !== librarySignal?.id).map((signal) => signal.id),
|
||||
);
|
||||
const taken = new Set([...savedIds, ...ids, ...libraryIds]);
|
||||
return [...ids, uniqueId(librarySignal?.id ?? slug(row.name), taken)];
|
||||
}, []);
|
||||
};
|
||||
|
||||
const rowProblems = (row: SignalRow, duplicateName: boolean): RowProblems => {
|
||||
const name = row.name.trim();
|
||||
const question = row.question.trim();
|
||||
const nameProblem = [
|
||||
!name ? "Name the signal" : undefined,
|
||||
name.length > MAX_NAME ? `Keep the name under ${MAX_NAME} characters` : undefined,
|
||||
duplicateName ? "Another signal has this name" : undefined,
|
||||
].find((problem) => problem !== undefined);
|
||||
const questionProblem = [
|
||||
question.length < 3 ? "Ask a yes or no question about the run" : undefined,
|
||||
question.length > MAX_QUESTION ? `Keep the question under ${MAX_QUESTION} characters` : undefined,
|
||||
].find((problem) => problem !== undefined);
|
||||
return { name: nameProblem, question: questionProblem };
|
||||
};
|
||||
|
||||
export function draftProblems(draft: SignalDraft): DraftProblems {
|
||||
const names = draft.rows.map((row) => row.name.trim().toLowerCase());
|
||||
const rows = new Map(
|
||||
draft.rows
|
||||
.map((row, index): [string, RowProblems] => [
|
||||
row.key,
|
||||
rowProblems(row, names[index] !== "" && names.indexOf(names[index]) !== index),
|
||||
])
|
||||
.filter(([, problems]) => problems.name || problems.question),
|
||||
);
|
||||
const { thresholdPercent } = draft;
|
||||
const threshold =
|
||||
Number.isInteger(thresholdPercent) && thresholdPercent >= 5 && thresholdPercent <= 95
|
||||
? undefined
|
||||
: "Use a whole number from 5 to 95";
|
||||
const signals = draft.rows.length > MAX_SIGNALS ? `Use at most ${MAX_SIGNALS} signals` : undefined;
|
||||
return { threshold, signals, rows, any: Boolean(threshold || signals || rows.size) };
|
||||
}
|
||||
|
||||
export function configFrom(draft: SignalDraft): SignalConfig {
|
||||
const ids = signalIds(draft.rows);
|
||||
return {
|
||||
model: draft.model,
|
||||
threshold: draft.thresholdPercent / 100,
|
||||
signals: draft.rows.map((row, index) => ({ id: ids[index], name: row.name.trim(), question: row.question.trim() })),
|
||||
};
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState, type KeyboardEvent } from "react";
|
||||
import { watches } from "../model/watches";
|
||||
import { watches as defaultWatches } from "../model/watches";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
const dotColors = ["#8b5cf6", "#22b3e8", "#e3a32b", "#eb6b93", "#22b3e8", "#8b5cf6", "#e3a32b", "#eb6b93"];
|
||||
|
|
@ -75,14 +75,26 @@ function DotFlow({ active }: { active: readonly string[] }) {
|
|||
);
|
||||
}
|
||||
|
||||
export interface PickerOption {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly summary: string;
|
||||
}
|
||||
|
||||
export function WatchPicker({
|
||||
selected,
|
||||
onChange,
|
||||
onAddCustom,
|
||||
options: watches = defaultWatches,
|
||||
label = "Watch for",
|
||||
addDisabled = false,
|
||||
}: {
|
||||
selected: ReadonlySet<string>;
|
||||
onChange: (next: ReadonlySet<string>) => void;
|
||||
onAddCustom: () => void;
|
||||
options?: readonly PickerOption[];
|
||||
label?: string;
|
||||
addDisabled?: boolean;
|
||||
}) {
|
||||
const [cursor, setCursor] = useState(0);
|
||||
const items = useRef<(HTMLButtonElement | null)[]>([]);
|
||||
|
|
@ -105,18 +117,20 @@ export function WatchPicker({
|
|||
} else return;
|
||||
event.preventDefault();
|
||||
};
|
||||
const activeColors = watches.flatMap((watch, index) => (selected.has(watch.id) ? [dotColors[index]] : []));
|
||||
const activeColors = watches.flatMap((watch, index) =>
|
||||
selected.has(watch.id) ? [dotColors[index % dotColors.length]] : [],
|
||||
);
|
||||
|
||||
return (
|
||||
<fieldset className="space-y-2.5">
|
||||
<div className="flex items-end justify-between gap-3">
|
||||
<legend className="text-sm font-medium">Watch for</legend>
|
||||
<legend className="text-sm font-medium">{label}</legend>
|
||||
<span className="text-xs tabular-nums text-muted-foreground">
|
||||
{selected.size} of {watches.length} selected
|
||||
{watches.filter((watch) => selected.has(watch.id)).length} of {watches.length} selected
|
||||
</span>
|
||||
</div>
|
||||
<DotFlow active={activeColors} />
|
||||
<div role="group" aria-label="Watch for" onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
|
||||
<div role="group" aria-label={label} onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
|
||||
{watches.map((watch, index) => {
|
||||
const on = selected.has(watch.id);
|
||||
return (
|
||||
|
|
@ -164,7 +178,8 @@ export function WatchPicker({
|
|||
<button
|
||||
type="button"
|
||||
onClick={onAddCustom}
|
||||
className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
|
||||
disabled={addDisabled}
|
||||
className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted disabled:cursor-not-allowed disabled:opacity-50 focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
|
||||
>
|
||||
<span
|
||||
aria-hidden="true"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import type {
|
|||
TracePage,
|
||||
TraceFindingCount,
|
||||
TraceFindingsRequest,
|
||||
TraceSignals,
|
||||
} from "./types";
|
||||
|
||||
export interface TraceWindow {
|
||||
|
|
@ -38,6 +39,7 @@ export interface TracesApi {
|
|||
handoff(traceId: string, spanId?: string | null, traceRef?: string): TraceHandoff;
|
||||
list(window: TraceWindow): Promise<TracePage>;
|
||||
findings(traces: TraceFindingsRequest["traces"]): Promise<TraceFindingCount[]>;
|
||||
signals(traces: TraceFindingsRequest["traces"]): Promise<TraceSignals[]>;
|
||||
anyRecorded(): Promise<boolean>;
|
||||
trace(traceId: string, traceRef?: string, cursor?: string | null): Promise<Trace>;
|
||||
span(traceId: string, spanId: string, traceRef?: string): Promise<SpanDetail>;
|
||||
|
|
@ -82,6 +84,11 @@ export function liveTracesApi(accessToken: string): TracesApi {
|
|||
accessToken,
|
||||
body: { traces } satisfies TraceFindingsRequest,
|
||||
}),
|
||||
signals: (traces) =>
|
||||
apiClient.post<TraceSignals[]>("/lens/traces/signals", {
|
||||
accessToken,
|
||||
body: { traces } satisfies TraceFindingsRequest,
|
||||
}),
|
||||
anyRecorded: async () => {
|
||||
const page = await apiClient.get<TracePage>("/v1/traces", {
|
||||
accessToken,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ import { traceRefOf, traceShareUrl } from "../../routing";
|
|||
import { IdChip } from "../../ui/IdChip";
|
||||
import { SpanIcon } from "../../ui/SpanIcon";
|
||||
import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework";
|
||||
import type { Trace } from "../../types";
|
||||
import type { SignalFlag, Trace } from "../../types";
|
||||
import { SignalPills } from "../../ui/SignalPills";
|
||||
import { fmtMs, fmtTok, traceAgentNames, traceDisplayName } from "../../utils";
|
||||
|
||||
interface CopyButtonProps {
|
||||
|
|
@ -90,6 +91,7 @@ interface RunHeaderProps {
|
|||
live: boolean;
|
||||
canLive: boolean;
|
||||
onLiveChange: () => void;
|
||||
signals?: readonly SignalFlag[];
|
||||
}
|
||||
|
||||
/** Run identity, view switch and totals in two tight rows. */
|
||||
|
|
@ -103,6 +105,7 @@ export function RunHeader({
|
|||
live,
|
||||
canLive,
|
||||
onLiveChange,
|
||||
signals = [],
|
||||
}: RunHeaderProps) {
|
||||
const { summary } = trace;
|
||||
const failed = summary.status === "error";
|
||||
|
|
@ -162,6 +165,7 @@ export function RunHeader({
|
|||
</div>
|
||||
<div className="flex flex-wrap items-center gap-x-4 gap-y-1.5 text-xs text-muted-foreground">
|
||||
<StatusPill failed={failed} />
|
||||
{signals.length > 0 && <SignalPills flags={signals} showScore className="flex-wrap" />}
|
||||
<Stat label="Duration" value={fmtMs(summary.duration_ms)} />
|
||||
<Stat label="Steps" value={summary.span_count.toLocaleString()} />
|
||||
<Stat label="Tokens" value={fmtTok(summary.input_tokens + summary.output_tokens)} />
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import type { Trace } from "../../types";
|
|||
import { PagingBanner } from "./PagingBanner";
|
||||
import { RunBody } from "./RunBody";
|
||||
import { RunHeader } from "./RunHeader";
|
||||
import { useTraceSignalFlags } from "../../list/useTraceSignals";
|
||||
|
||||
interface RunViewProps {
|
||||
traceId: string;
|
||||
|
|
@ -26,6 +27,7 @@ interface RunViewProps {
|
|||
onBack: () => void;
|
||||
/** Rendered inside the side drawer: the drawer owns closing and sizing. */
|
||||
embedded?: boolean;
|
||||
showSignals?: boolean;
|
||||
}
|
||||
|
||||
function selectedSpanMissing(trace: Trace, spanId: string | null): boolean {
|
||||
|
|
@ -87,6 +89,7 @@ function LoadedRun({
|
|||
accessToken,
|
||||
onBack,
|
||||
embedded = false,
|
||||
showSignals = false,
|
||||
switching,
|
||||
}: RunViewProps & { switching: boolean }) {
|
||||
const traces = useTracesApi(accessToken);
|
||||
|
|
@ -108,6 +111,7 @@ function LoadedRun({
|
|||
retryDelay: traceReadRetryDelay,
|
||||
};
|
||||
const traceQuery = useSuspenseInfiniteQuery(traceQueryOptions);
|
||||
const signals = useTraceSignalFlags(accessToken, { trace_id: traceId, trace_ref: traceRef }, showSignals);
|
||||
const refreshTrace = () => queryClient.resetQueries({ queryKey, exact: true });
|
||||
const failure = traceQuery.isFetchNextPageError ? classifyTraceReadFailure(traceQuery.error) : null;
|
||||
const readManually = (read: () => Promise<unknown>) => {
|
||||
|
|
@ -169,6 +173,7 @@ function LoadedRun({
|
|||
live={live}
|
||||
canLive={traces.live}
|
||||
onLiveChange={toggleLive}
|
||||
signals={signals}
|
||||
/>
|
||||
{traceQuery.isRefetchError && (
|
||||
<div role="alert" className="flex items-center gap-3 border-b p-3 text-xs text-muted-foreground">
|
||||
|
|
|
|||
|
|
@ -11,12 +11,14 @@ export default function AgentTracesPage({
|
|||
readOnly = false,
|
||||
canMintTracingKey = false,
|
||||
canViewFindings = true,
|
||||
onSetUpSignals,
|
||||
}: {
|
||||
accessToken: string;
|
||||
isActive?: boolean;
|
||||
readOnly?: boolean;
|
||||
canMintTracingKey?: boolean;
|
||||
canViewFindings?: boolean;
|
||||
onSetUpSignals?: () => void;
|
||||
}) {
|
||||
const time = useRelativeRange(useTracesLive());
|
||||
return (
|
||||
|
|
@ -28,6 +30,7 @@ export default function AgentTracesPage({
|
|||
readOnly={readOnly}
|
||||
canMintTracingKey={canMintTracingKey}
|
||||
canViewFindings={canViewFindings}
|
||||
onSetUpSignals={onSetUpSignals}
|
||||
timeControls={time}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import moment from "moment";
|
||||
import { skipToken, useQuery } from "@tanstack/react-query";
|
||||
import { RefreshCw } from "lucide-react";
|
||||
import { traceAgentNames } from "../utils";
|
||||
import { useMemo, useState } from "react";
|
||||
|
|
@ -12,6 +13,10 @@ import { Button } from "@/components/ui/button";
|
|||
|
||||
import { AgentTracesTable } from "./AgentTracesTable";
|
||||
import { useTraceFindings } from "./useTraceFindings";
|
||||
import { useTraceSignals } from "./useTraceSignals";
|
||||
import { useOptionalLensApi } from "../../data/LensServices";
|
||||
import { lensKeys } from "../../data/queries";
|
||||
import { signalsConfigured } from "../../model/signals";
|
||||
import { type TraceRef, traceKey, traceRefOf, useOpenTraceRouting, useRunFilterRouting } from "../routing";
|
||||
import type { TraceSummary } from "../types";
|
||||
import { RunView } from "../detail/run/RunView";
|
||||
|
|
@ -41,6 +46,18 @@ interface AgentTracesSectionProps {
|
|||
readOnly?: boolean;
|
||||
canMintTracingKey?: boolean;
|
||||
canViewFindings?: boolean;
|
||||
onSetUpSignals?: () => void;
|
||||
}
|
||||
|
||||
function useSignalSetup(enabled: boolean) {
|
||||
const api = useOptionalLensApi();
|
||||
const config = useQuery({
|
||||
queryKey: lensKeys.signalConfig(api?.scope ?? ""),
|
||||
queryFn: api && enabled ? () => api.signalConfig() : skipToken,
|
||||
staleTime: 5000,
|
||||
});
|
||||
const loaded = enabled && config.data !== undefined;
|
||||
return { on: loaded && signalsConfigured(config.data), missing: loaded && !signalsConfigured(config.data) };
|
||||
}
|
||||
|
||||
function useTracingSetup(traces: AgentTracesResult, isActive: boolean, rangeChanged: boolean) {
|
||||
|
|
@ -79,6 +96,7 @@ export function AgentTracesSection({
|
|||
readOnly = false,
|
||||
canMintTracingKey = false,
|
||||
canViewFindings,
|
||||
onSetUpSignals,
|
||||
}: AgentTracesSectionProps) {
|
||||
const live = useTracesLive();
|
||||
const { trace: openTrace, openTrace: openRun, selection, fullScreen, setFullScreen } = useOpenTraceRouting();
|
||||
|
|
@ -109,6 +127,8 @@ export function AgentTracesSection({
|
|||
const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
|
||||
const runRefs = useMemo(() => runs.map(traceRefOf), [runs]);
|
||||
const findings = useTraceFindings(accessToken, runs, isActive, canViewFindings);
|
||||
const signalSetup = useSignalSetup(isActive && canViewFindings !== false);
|
||||
const signals = useTraceSignals(accessToken, runs, isActive && signalSetup.on);
|
||||
|
||||
const changeRange = (hours: number, apply: (hours: number) => void) => {
|
||||
setZoom(null);
|
||||
|
|
@ -173,6 +193,7 @@ export function AgentTracesSection({
|
|||
accessToken={accessToken}
|
||||
onBack={() => openRun(null)}
|
||||
embedded
|
||||
showSignals={signalSetup.on}
|
||||
/>
|
||||
)}
|
||||
</Inspector.Panel>
|
||||
|
|
@ -211,6 +232,10 @@ export function AgentTracesSection({
|
|||
traces={runs}
|
||||
findings={findings}
|
||||
canViewFindings={canViewFindings}
|
||||
signals={signals}
|
||||
showSignals={signalSetup.on}
|
||||
signalsColumn={signalSetup.on || signalSetup.missing}
|
||||
onSetUpSignals={onSetUpSignals}
|
||||
isLoading={traces.isLoading || (checkHistory && history.isLoading)}
|
||||
error={traces.error}
|
||||
hasMore={traces.hasMore}
|
||||
|
|
|
|||
|
|
@ -254,3 +254,79 @@ describe("AgentTracesTable column picker", () => {
|
|||
expect(within(placeholder).getAllByRole("cell", { hidden: true })).toHaveLength(columnCount);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AgentTracesTable signals", () => {
|
||||
const [flaggedRun, cleanRun, queuedRun] = ((traceList as TracePage).data as TraceSummary[]).slice(0, 3);
|
||||
const key = (run: TraceSummary) => run.trace_ref || run.trace_id;
|
||||
const result = (
|
||||
run: TraceSummary,
|
||||
status: "classified" | "unclassified",
|
||||
flags: { signal_id: string; name: string; score: number }[] = [],
|
||||
) => ({
|
||||
status: "ready" as const,
|
||||
signals: {
|
||||
trace_id: run.trace_id,
|
||||
trace_ref: run.trace_ref ?? "",
|
||||
status,
|
||||
flags,
|
||||
model: "jev",
|
||||
classified_at: null,
|
||||
},
|
||||
});
|
||||
const renderTable = (showSignals: boolean, onSetUpSignals?: () => void) =>
|
||||
renderWithProviders(
|
||||
inList(
|
||||
<AgentTracesTable
|
||||
traces={[flaggedRun, cleanRun, queuedRun]}
|
||||
findings={new Map()}
|
||||
signals={
|
||||
new Map([
|
||||
[
|
||||
key(flaggedRun),
|
||||
result(flaggedRun, "classified", [
|
||||
{ signal_id: "user_frustration", name: "User frustration", score: 0.92 },
|
||||
{ signal_id: "repeated_request", name: "Repeated request", score: 0.71 },
|
||||
]),
|
||||
],
|
||||
[key(cleanRun), result(cleanRun, "classified")],
|
||||
[key(queuedRun), result(queuedRun, "unclassified")],
|
||||
])
|
||||
}
|
||||
showSignals={showSignals}
|
||||
signalsColumn
|
||||
onSetUpSignals={onSetUpSignals}
|
||||
isLoading={false}
|
||||
error={null}
|
||||
hasMore={false}
|
||||
onLoadMore={vi.fn()}
|
||||
rangeEmpty={false}
|
||||
onSetUpTracing={vi.fn()}
|
||||
/>,
|
||||
),
|
||||
);
|
||||
|
||||
it("flags matching runs in red and names every detected signal", () => {
|
||||
renderTable(true);
|
||||
const rows = screen.getAllByTestId("agent-trace-row");
|
||||
expect(rows.map((row) => row.hasAttribute("data-flagged"))).toEqual([true, false, false]);
|
||||
const flagged = within(rows[0]).getByRole("list", { name: "Signals" });
|
||||
expect(
|
||||
within(flagged)
|
||||
.getAllByRole("listitem")
|
||||
.map((item) => item.textContent),
|
||||
).toEqual(["User frustration", "Repeated request"]);
|
||||
expect(flagged).toHaveAttribute("title", "Signals: User frustration (92%), Repeated request (71%)");
|
||||
expect(within(rows[1]).getByTitle("No signals detected")).toBeInTheDocument();
|
||||
expect(within(rows[2]).getByText("Queued")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the signals column with a setup link until signals are configured", async () => {
|
||||
const onSetUpSignals = vi.fn();
|
||||
renderTable(false, onSetUpSignals);
|
||||
const header = screen.getByRole("columnheader", { name: /Signals/ });
|
||||
await userEvent.click(within(header).getByRole("button", { name: "Set up signals" }));
|
||||
expect(onSetUpSignals).toHaveBeenCalledOnce();
|
||||
expect(screen.getAllByTitle("Signals are not set up")).toHaveLength(3);
|
||||
expect(screen.getAllByTestId("agent-trace-row").some((row) => row.hasAttribute("data-flagged"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { getCoreRowModel, useReactTable, type ColumnDef, type TableOptions } from "@tanstack/react-table";
|
||||
import { ArrowDown, ChevronRight } from "lucide-react";
|
||||
import { ArrowDown, ChevronRight, Plus } from "lucide-react";
|
||||
import { createContext, useContext, useEffect } from "react";
|
||||
import { useInView } from "react-intersection-observer";
|
||||
|
||||
|
|
@ -15,6 +15,8 @@ import { formatActivityTimestamp, formatRunTimestamp, localTimeZoneAbbreviation
|
|||
|
||||
import { SpanIcon } from "../ui/SpanIcon";
|
||||
import type { TraceFindingState } from "./useTraceFindings";
|
||||
import { flaggedSignals, isFlagged, type TraceSignalState } from "./useTraceSignals";
|
||||
import { SignalPills } from "../ui/SignalPills";
|
||||
import { FrameworkLogo, traceFramework } from "../ui/TraceFramework";
|
||||
import type { TraceSummary } from "../types";
|
||||
import { traceRefOf } from "../routing";
|
||||
|
|
@ -24,6 +26,10 @@ interface AgentTracesTableProps {
|
|||
traces: TraceSummary[];
|
||||
findings: ReadonlyMap<string, TraceFindingState>;
|
||||
canViewFindings?: boolean;
|
||||
signals?: ReadonlyMap<string, TraceSignalState>;
|
||||
showSignals?: boolean;
|
||||
signalsColumn?: boolean;
|
||||
onSetUpSignals?: () => void;
|
||||
isLoading: boolean;
|
||||
error: Error | null;
|
||||
hasMore: boolean;
|
||||
|
|
@ -79,6 +85,10 @@ const ROW_HEIGHT = 36;
|
|||
const MUTED_NUM = "font-mono text-muted-foreground";
|
||||
const NUM = "font-mono text-foreground";
|
||||
const FindingsContext = createContext<ReadonlyMap<string, TraceFindingState>>(new Map());
|
||||
const SignalsContext = createContext<ReadonlyMap<string, TraceSignalState>>(new Map());
|
||||
const NO_SIGNALS: ReadonlyMap<string, TraceSignalState> = new Map();
|
||||
const SignalSetupContext = createContext<{ configured: boolean; onSetUp?: () => void }>({ configured: false });
|
||||
const FLAGGED_ROW = "bg-destructive/[0.04] shadow-[inset_2px_0_0_var(--color-destructive)] hover:bg-destructive/[0.07]";
|
||||
|
||||
function AgentCell({ run }: { run: TraceSummary }) {
|
||||
const framework = traceFramework(run);
|
||||
|
|
@ -106,7 +116,9 @@ function InputCell({ run }: { run: TraceSummary }) {
|
|||
Partial totals
|
||||
</span>
|
||||
)}
|
||||
<span className="hidden shrink-0 font-mono text-xs text-muted-foreground 2xl:inline">{run.trace_id}</span>
|
||||
<span className="hidden max-w-32 min-w-0 shrink-[100] truncate font-mono text-xs text-muted-foreground 2xl:inline">
|
||||
{run.trace_id}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -120,6 +132,52 @@ function FindingCount({ run }: { run: TraceSummary }) {
|
|||
return <span title={`${state.count} findings from completed investigations`}>{state.count.toLocaleString()}</span>;
|
||||
}
|
||||
|
||||
function SignalsHeader() {
|
||||
const { configured, onSetUp } = useContext(SignalSetupContext);
|
||||
if (configured || !onSetUp) return <>Signals</>;
|
||||
return (
|
||||
<span className="inline-flex items-center gap-2 whitespace-nowrap">
|
||||
Signals
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="xs"
|
||||
aria-label="Set up signals"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
onSetUp();
|
||||
}}
|
||||
className="h-5 rounded-full px-2 font-medium tracking-normal normal-case"
|
||||
>
|
||||
<Plus />
|
||||
Set up
|
||||
</Button>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const mutedCell = (label: string, title: string) => (
|
||||
<span className="text-muted-foreground" title={title}>
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
|
||||
function SignalsCell({ run }: { run: TraceSummary }) {
|
||||
const { configured } = useContext(SignalSetupContext);
|
||||
const state = useContext(SignalsContext).get(runKey(run));
|
||||
const muted = mutedCell;
|
||||
if (!configured) return muted("-", "Signals are not set up");
|
||||
if (!state || state.status === "pending") return <Skeleton aria-label="Loading signals" className="h-3 w-16" />;
|
||||
if (state.status === "error") return muted("Unavailable", "Could not load signals");
|
||||
const { status } = state.signals;
|
||||
if (status === "unclassified") return muted("Queued", "Waiting for the System 1 model to check this run");
|
||||
if (status === "pending") return muted("Checking", "The System 1 model is checking this run");
|
||||
if (status === "failed") return muted("Not checked", "The System 1 model could not check this run");
|
||||
const flags = flaggedSignals(state.signals);
|
||||
if (!flags.length) return muted("-", "No signals detected");
|
||||
return <SignalPills flags={flags} className="overflow-hidden" />;
|
||||
}
|
||||
|
||||
const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
|
||||
{
|
||||
id: "time",
|
||||
|
|
@ -160,6 +218,13 @@ const RUN_COLUMNS: ColumnDef<TraceSummary>[] = [
|
|||
},
|
||||
},
|
||||
{ id: "input", header: "Input", cell: ({ row }) => <InputCell run={row.original} /> },
|
||||
{
|
||||
id: "signals",
|
||||
size: 300,
|
||||
header: () => <SignalsHeader />,
|
||||
cell: ({ row }) => <SignalsCell run={row.original} />,
|
||||
meta: { title: "Signals", renderSkeleton: () => <Skeleton className="h-3 w-16" /> },
|
||||
},
|
||||
{
|
||||
id: "agents",
|
||||
size: 72,
|
||||
|
|
@ -248,6 +313,10 @@ export function AgentTracesTable({
|
|||
traces,
|
||||
findings,
|
||||
canViewFindings = true,
|
||||
signals = NO_SIGNALS,
|
||||
showSignals = false,
|
||||
signalsColumn = showSignals,
|
||||
onSetUpSignals,
|
||||
isLoading,
|
||||
error,
|
||||
hasMore,
|
||||
|
|
@ -265,7 +334,9 @@ export function AgentTracesTable({
|
|||
const { columnVisibility, onColumnVisibilityChange } = usePersistedColumnVisibility("lens-traces");
|
||||
const tableOptions: TableOptions<TraceSummary> = {
|
||||
data: traces,
|
||||
columns: RUN_COLUMNS.filter((column) => canViewFindings || column.id !== "findings"),
|
||||
columns: RUN_COLUMNS.filter(
|
||||
(column) => (canViewFindings || column.id !== "findings") && (signalsColumn || column.id !== "signals"),
|
||||
),
|
||||
defaultColumn: { size: undefined },
|
||||
getRowId: runKey,
|
||||
autoResetAll: false,
|
||||
|
|
@ -276,56 +347,64 @@ export function AgentTracesTable({
|
|||
const table = useReactTable(tableOptions);
|
||||
return (
|
||||
<FindingsContext.Provider value={findings}>
|
||||
<InspectorTable.Root table={table} data-testid="runs-table">
|
||||
<InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
|
||||
<InspectorTable.Header />
|
||||
<InspectorTable.Body<TraceSummary>
|
||||
className={bodyClassName(isPlaceholder)}
|
||||
rowHeight={() => ROW_HEIGHT}
|
||||
after={
|
||||
<>
|
||||
{isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
|
||||
{autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
|
||||
</>
|
||||
}
|
||||
>
|
||||
{(row) => (
|
||||
<InspectorTable.Row
|
||||
row={row}
|
||||
item={traceRefOf(row.original)}
|
||||
data-testid="agent-trace-row"
|
||||
className="h-9"
|
||||
/>
|
||||
<SignalsContext.Provider value={signals}>
|
||||
<SignalSetupContext.Provider value={{ configured: showSignals, onSetUp: onSetUpSignals }}>
|
||||
<InspectorTable.Root table={table} data-testid="runs-table">
|
||||
<InspectorTable.Grid aria-label="Agent runs" aria-busy={isFetching} className="min-w-[900px] text-xs">
|
||||
<InspectorTable.Header />
|
||||
<InspectorTable.Body<TraceSummary>
|
||||
className={bodyClassName(isPlaceholder)}
|
||||
rowHeight={() => ROW_HEIGHT}
|
||||
after={
|
||||
<>
|
||||
{isLoading && SKELETON_ROWS.map((row) => <PlaceholderRow key={row} index={row} />)}
|
||||
{autoContinue && <LoadMoreRows isFetching={isFetching} onLoadMore={onLoadMore} />}
|
||||
</>
|
||||
}
|
||||
>
|
||||
{(row) => {
|
||||
const flagged = showSignals && isFlagged(signals.get(runKey(row.original)));
|
||||
return (
|
||||
<InspectorTable.Row
|
||||
row={row}
|
||||
item={traceRefOf(row.original)}
|
||||
data-testid="agent-trace-row"
|
||||
data-flagged={flagged || undefined}
|
||||
className={cn("h-9", flagged && FLAGGED_ROW)}
|
||||
/>
|
||||
);
|
||||
}}
|
||||
</InspectorTable.Body>
|
||||
</InspectorTable.Grid>
|
||||
{isLoading && (
|
||||
<p role="status" className="sr-only">
|
||||
Loading runs…
|
||||
</p>
|
||||
)}
|
||||
</InspectorTable.Body>
|
||||
</InspectorTable.Grid>
|
||||
{isLoading && (
|
||||
<p role="status" className="sr-only">
|
||||
Loading runs…
|
||||
</p>
|
||||
)}
|
||||
{error && (
|
||||
<div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
|
||||
</span>
|
||||
{onRetry && (
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
|
||||
Retry
|
||||
</Button>
|
||||
{error && (
|
||||
<div role="alert" className="flex items-center justify-center gap-3 py-6 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message}
|
||||
</span>
|
||||
{onRetry && (
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onRetry}>
|
||||
Retry
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{canContinue && traces.length === 0 && (
|
||||
<div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
|
||||
<span>No loaded runs match these filters.</span>
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
|
||||
Load older runs
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
|
||||
</InspectorTable.Root>
|
||||
{canContinue && traces.length === 0 && (
|
||||
<div className="flex items-center justify-center gap-3 py-16 text-xs text-muted-foreground">
|
||||
<span>No loaded runs match these filters.</span>
|
||||
<Button size="xs" variant="outline" disabled={isFetching} onClick={onLoadMore}>
|
||||
Load older runs
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
{isEmpty && <EmptyRuns rangeEmpty={rangeEmpty} onSetUpTracing={onSetUpTracing} />}
|
||||
</InspectorTable.Root>
|
||||
</SignalSetupContext.Provider>
|
||||
</SignalsContext.Provider>
|
||||
</FindingsContext.Provider>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import { useQueries, useQuery } from "@tanstack/react-query";
|
||||
import { chunk } from "es-toolkit";
|
||||
|
||||
import { useTracesApi } from "../api";
|
||||
import type { SignalFlag, TraceSignals, TraceSummary } from "../types";
|
||||
|
||||
export type TraceSignalState = { status: "ready"; signals: TraceSignals } | { status: "pending" } | { status: "error" };
|
||||
|
||||
const POLL_MS = 15000;
|
||||
|
||||
const identity = ({ trace_id, trace_ref }: { trace_id: string; trace_ref?: string | null }) => ({
|
||||
trace_id,
|
||||
trace_ref: trace_ref ?? "",
|
||||
});
|
||||
|
||||
export const flaggedSignals = (signals?: TraceSignals): SignalFlag[] =>
|
||||
signals?.status === "classified" ? signals.flags ?? [] : [];
|
||||
|
||||
export const isFlagged = (state?: TraceSignalState): boolean =>
|
||||
state?.status === "ready" && flaggedSignals(state.signals).length > 0;
|
||||
|
||||
export function useTraceSignals(accessToken: string, runs: TraceSummary[], enabled: boolean) {
|
||||
const api = useTracesApi(accessToken);
|
||||
const batches = chunk(runs.map(identity), 500);
|
||||
const queries = useQueries({
|
||||
queries: batches.map((traces) => ({
|
||||
queryKey: ["traceSignals", accessToken, traces],
|
||||
queryFn: () => api.signals(traces),
|
||||
enabled,
|
||||
staleTime: POLL_MS,
|
||||
refetchInterval: enabled && api.live ? POLL_MS : false,
|
||||
retry: false,
|
||||
})),
|
||||
});
|
||||
return new Map<string, TraceSignalState>(
|
||||
batches.flatMap((traces, index) => {
|
||||
const query = queries[index];
|
||||
const results = new Map(query.data?.map((result) => [result.trace_ref || result.trace_id, result]));
|
||||
return traces.map((trace): [string, TraceSignalState] => {
|
||||
const key = trace.trace_ref || trace.trace_id;
|
||||
const found = results.get(key);
|
||||
if (query.isError) return [key, { status: "error" }];
|
||||
if (query.isPending) return [key, { status: "pending" }];
|
||||
if (!found) return [key, { status: "error" }];
|
||||
return [key, { status: "ready", signals: found }];
|
||||
});
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function useTraceSignalFlags(
|
||||
accessToken: string,
|
||||
trace: { trace_id: string; trace_ref?: string | null },
|
||||
enabled: boolean,
|
||||
): SignalFlag[] {
|
||||
const api = useTracesApi(accessToken);
|
||||
const traces = [identity(trace)];
|
||||
const queryOptions = {
|
||||
queryKey: ["traceSignals", accessToken, traces],
|
||||
queryFn: () => api.signals(traces),
|
||||
enabled,
|
||||
staleTime: POLL_MS,
|
||||
refetchInterval: enabled && api.live ? POLL_MS : (false as const),
|
||||
retry: false,
|
||||
};
|
||||
const query = useQuery<TraceSignals[]>(queryOptions);
|
||||
return enabled ? flaggedSignals(query.data?.[0]) : [];
|
||||
}
|
||||
|
|
@ -13,6 +13,8 @@ export type SpanErrorQuery = NonNullable<
|
|||
export type TraceQueryBody = components["schemas"]["TraceQueryRequest"];
|
||||
export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"];
|
||||
export type TraceFindingCount = components["schemas"]["TraceFindingCount"];
|
||||
export type TraceSignals = components["schemas"]["TraceSignals"];
|
||||
export type SignalFlag = NonNullable<TraceSignals["flags"]>[number];
|
||||
type ApiSpanDetail =
|
||||
paths["/v1/traces/{trace_id}/spans/{span_id}"]["get"]["responses"][200]["content"]["application/json"];
|
||||
export type Span = Trace["spans"][number];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import type { SignalFlag } from "../types";
|
||||
|
||||
const percent = (score: number): string => `${Math.round(score * 100)}%`;
|
||||
|
||||
export const signalSummary = (flags: readonly SignalFlag[]): string =>
|
||||
flags.map((flag) => `${flag.name} (${percent(flag.score)})`).join(", ");
|
||||
|
||||
function SignalPill({ flag, showScore }: { flag: SignalFlag; showScore: boolean }) {
|
||||
return (
|
||||
<span className="inline-flex max-w-full min-w-0 items-center gap-1 rounded-full bg-destructive/10 px-1.5 py-0.5 text-xs leading-none font-medium text-destructive">
|
||||
<span aria-hidden="true" className="size-1.5 shrink-0 rounded-full bg-destructive" />
|
||||
<span className="truncate">{flag.name}</span>
|
||||
{showScore && <span className="shrink-0 font-normal tabular-nums opacity-80">{percent(flag.score)}</span>}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
export function SignalPills({
|
||||
flags,
|
||||
showScore = false,
|
||||
className,
|
||||
}: {
|
||||
flags: readonly SignalFlag[];
|
||||
showScore?: boolean;
|
||||
className?: string;
|
||||
}) {
|
||||
return (
|
||||
<span
|
||||
role="list"
|
||||
aria-label="Signals"
|
||||
title={`Signals: ${signalSummary(flags)}`}
|
||||
className={cn("inline-flex max-w-full min-w-0 items-center gap-1", className)}
|
||||
>
|
||||
{flags.map((flag) => (
|
||||
<span role="listitem" key={flag.signal_id} className="min-w-0">
|
||||
<SignalPill flag={flag} showScore={showScore} />
|
||||
</span>
|
||||
))}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
200
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
200
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -9103,6 +9103,24 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/signals": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Get Signals */
|
||||
get: operations["get_signals_lens_signals_get"];
|
||||
/** Put Signals */
|
||||
put: operations["put_signals_lens_signals_put"];
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/traces/findings": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -9120,6 +9138,23 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/traces/signals": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Trace Signal Statuses */
|
||||
post: operations["trace_signal_statuses_lens_traces_signals_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/lens/watch-all": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -45477,6 +45512,58 @@ export interface components {
|
|||
/** Type */
|
||||
type: "shell" | string;
|
||||
};
|
||||
/** Signal */
|
||||
Signal: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Name */
|
||||
name: string;
|
||||
/** Question */
|
||||
question: string;
|
||||
};
|
||||
/** SignalConfig */
|
||||
SignalConfig: {
|
||||
/**
|
||||
* Model
|
||||
* @default
|
||||
*/
|
||||
model: string;
|
||||
/**
|
||||
* Signals
|
||||
* @default [
|
||||
* {
|
||||
* "id": "user_frustration",
|
||||
* "name": "User frustration",
|
||||
* "question": "Does the user show frustration, annoyance or dissatisfaction with the agent in this run, for example complaints, irritated corrections, all caps, profanity, or giving up on the task?"
|
||||
* },
|
||||
* {
|
||||
* "id": "missing_capability",
|
||||
* "name": "Missing capability",
|
||||
* "question": "Does the user ask for something the agent cannot do in this run, so that the agent refuses, says it lacks a tool, permission, integration or data source, or fails because the capability does not exist?"
|
||||
* },
|
||||
* {
|
||||
* "id": "repeated_request",
|
||||
* "name": "Repeated request",
|
||||
* "question": "Does the user ask for the same thing more than once in this run, usually because the agent did not deliver it the first time?"
|
||||
* }
|
||||
* ]
|
||||
*/
|
||||
signals: components["schemas"]["Signal"][];
|
||||
/**
|
||||
* Threshold
|
||||
* @default 0.5
|
||||
*/
|
||||
threshold: number;
|
||||
};
|
||||
/** SignalFlag */
|
||||
SignalFlag: {
|
||||
/** Name */
|
||||
name: string;
|
||||
/** Score */
|
||||
score: number;
|
||||
/** Signal Id */
|
||||
signal_id: string;
|
||||
};
|
||||
/**
|
||||
* Skill
|
||||
* @description Represents a skill from the Anthropic Skills API
|
||||
|
|
@ -47719,6 +47806,33 @@ export interface components {
|
|||
[key: string]: components["schemas"]["JsonValue"];
|
||||
}[];
|
||||
};
|
||||
/** TraceSignals */
|
||||
TraceSignals: {
|
||||
/** Classified At */
|
||||
classified_at?: string | null;
|
||||
/**
|
||||
* Flags
|
||||
* @default []
|
||||
*/
|
||||
flags: components["schemas"]["SignalFlag"][];
|
||||
/**
|
||||
* Model
|
||||
* @default
|
||||
*/
|
||||
model: string;
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "unclassified" | "pending" | "classified" | "failed";
|
||||
/** Trace Id */
|
||||
trace_id: string;
|
||||
/**
|
||||
* Trace Ref
|
||||
* @default
|
||||
*/
|
||||
trace_ref: string;
|
||||
};
|
||||
/** TraceSource */
|
||||
TraceSource: {
|
||||
/**
|
||||
|
|
@ -63081,6 +63195,59 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_signals_lens_signals_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["SignalConfig"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
put_signals_lens_signals_put: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["SignalConfig"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["SignalConfig"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
trace_findings_lens_traces_findings_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -63114,6 +63281,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
trace_signal_statuses_lens_traces_signals_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["TraceFindingsRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["TraceSignals"][];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
watch_all_lens_watch_all_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue