mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
feat(ui): improve trace inspection and ROI estimation (#44351)
* feat(lens): simplify trace inspection in the gateway drawer * feat(lens): add a conversation view for full traces * test(lens): keep normalized message fixtures type safe * refactor(lens): make conversation view read like a chat * refactor(lens): use a quiet trace view menu * refactor(lens): make trace view tabs explicit * refactor(lens): restore compact trace view switch * fix(lens): preserve complete conversation history and tool types * feat(lens): add trace full-screen and close controls * fix(lens): show forwarded answers and agent errors once * fix(lens): reset full screen when closing a trace * fix(roi): estimate linked authors and clarify model selection * fix: preserve trace errors and ROI results across partial failures * fix(roi): correct pagination variable typing * fix(ui): place loaded root failures in conversation order * fix(roi): read estimator recommendations from model catalog * revert: remove catalog-driven ROI recommendations
This commit is contained in:
parent
5724117116
commit
cb00dbecd7
38 changed files with 2411 additions and 443 deletions
|
|
@ -49148,6 +49148,27 @@
|
|||
"title": "ROIEstimateResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIEstimatorModel": {
|
||||
"properties": {
|
||||
"model_name": {
|
||||
"title": "Model Name",
|
||||
"type": "string"
|
||||
},
|
||||
"provider_models": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": "Provider Models",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"model_name",
|
||||
"provider_models"
|
||||
],
|
||||
"title": "ROIEstimatorModel",
|
||||
"type": "object"
|
||||
},
|
||||
"ROIIdentityMapResponse": {
|
||||
"properties": {
|
||||
"identity_map": {
|
||||
|
|
@ -49583,6 +49604,14 @@
|
|||
"title": "Estimator Model",
|
||||
"type": "string"
|
||||
},
|
||||
"estimator_models": {
|
||||
"default": [],
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ROIEstimatorModel"
|
||||
},
|
||||
"title": "Estimator Models",
|
||||
"type": "array"
|
||||
},
|
||||
"estimator_prompt": {
|
||||
"title": "Estimator Prompt",
|
||||
"type": "string"
|
||||
|
|
|
|||
|
|
@ -32,8 +32,10 @@ from litellm.proxy.roi_calculator.github import SourceError
|
|||
from litellm.proxy.roi_calculator.source import create_source
|
||||
from litellm.proxy.roi_calculator.sync import (
|
||||
BranchSpendReader,
|
||||
GatewayUserReader,
|
||||
SpendReader,
|
||||
SyncManager,
|
||||
read_gateway_user_emails,
|
||||
read_spend,
|
||||
spend_prisma_client,
|
||||
)
|
||||
|
|
@ -43,6 +45,7 @@ from litellm.types.roi_calculator import (
|
|||
DEFAULT_PROMPT,
|
||||
ROIBranchSpend,
|
||||
ROICompletionRequest,
|
||||
ROIEstimatorModel,
|
||||
ROIIdentityMapResponse,
|
||||
ROIIdentityMapUpdate,
|
||||
ROIReport,
|
||||
|
|
@ -94,11 +97,13 @@ class _RouterEstimatorModelInfo(BaseModel):
|
|||
model_config = ConfigDict(extra="ignore", from_attributes=True)
|
||||
|
||||
base_model: str | None = None
|
||||
mode: str | None = None
|
||||
|
||||
|
||||
class _RouterEstimatorDeployment(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", from_attributes=True)
|
||||
|
||||
model_name: str = ""
|
||||
litellm_params: _RouterEstimatorParams
|
||||
model_info: _RouterEstimatorModelInfo | None = None
|
||||
|
||||
|
|
@ -144,7 +149,6 @@ def get_github_transport() -> httpx.AsyncBaseTransport | None:
|
|||
|
||||
|
||||
_ROUTER_ESTIMATOR_DEPLOYMENTS: Final = TypeAdapter(tuple[_RouterEstimatorDeployment, ...])
|
||||
_MODEL_NAMES: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def _estimator_models_from_deployments(deployments: Sequence[object]) -> tuple[EstimatorModel, ...]:
|
||||
|
|
@ -177,12 +181,45 @@ def _router_estimator_models(model_group: str) -> tuple[EstimatorModel, ...]:
|
|||
return _estimator_models_from_deployments(deployments)
|
||||
|
||||
|
||||
def _router_models() -> tuple[str, ...]:
|
||||
def _is_estimator_deployment(deployment: _RouterEstimatorDeployment) -> bool:
|
||||
from litellm import model_cost
|
||||
|
||||
underlying: Final = _estimator_model(deployment)
|
||||
if underlying is None:
|
||||
return False
|
||||
model, provider = underlying
|
||||
candidates: Final = (f"{provider}/{model}", model, model.split("/", 1)[-1])
|
||||
known_modes: Final = tuple(
|
||||
_RouterEstimatorModelInfo.model_validate(model_cost[name]).mode for name in candidates if name in model_cost
|
||||
)
|
||||
mode: Final = (deployment.model_info.mode if deployment.model_info else None) or next(iter(known_modes), None)
|
||||
return mode in (None, "chat")
|
||||
|
||||
|
||||
def _estimator_choices_from_deployments(deployments: Sequence[object]) -> tuple[ROIEstimatorModel, ...]:
|
||||
parsed: Final = _ROUTER_ESTIMATOR_DEPLOYMENTS.validate_python(deployments)
|
||||
names: Final = sorted(
|
||||
frozenset(item.model_name for item in parsed if item.model_name and "*" not in item.model_name)
|
||||
)
|
||||
groups: Final = tuple(tuple(item for item in parsed if item.model_name == name) for name in names)
|
||||
return tuple(
|
||||
ROIEstimatorModel(
|
||||
model_name=group[0].model_name,
|
||||
provider_models=tuple(sorted(frozenset(model[0] for item in group if (model := _estimator_model(item))))),
|
||||
)
|
||||
for group in groups
|
||||
if all(_is_estimator_deployment(item) for item in group)
|
||||
)
|
||||
|
||||
|
||||
def _router_estimator_choices() -> tuple[ROIEstimatorModel, ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
if llm_router is None:
|
||||
return ()
|
||||
return tuple(sorted(frozenset(_MODEL_NAMES.validate_python(llm_router.get_model_names()))))
|
||||
names: Final = frozenset(llm_router.get_model_names())
|
||||
choices: Final = _estimator_choices_from_deployments(llm_router.get_model_list() or ())
|
||||
return tuple(choice for choice in choices if choice.model_name in names)
|
||||
|
||||
|
||||
async def _load_stored_settings(repository: ConfigRepository) -> _StoredSettings:
|
||||
|
|
@ -262,7 +299,8 @@ async def _load_report(repository: ConfigRepository, settings: ROISettings) -> R
|
|||
|
||||
|
||||
def _public_settings(settings: ROISettings) -> ROISettingsResponse:
|
||||
models: Final = _router_models()
|
||||
choices: Final = _router_estimator_choices()
|
||||
models: Final = tuple(choice.model_name for choice in choices)
|
||||
return ROISettingsResponse(
|
||||
source_provider=settings.source_provider,
|
||||
gitlab_api_url=settings.gitlab_api_url,
|
||||
|
|
@ -278,6 +316,7 @@ def _public_settings(settings: ROISettings) -> ROISettingsResponse:
|
|||
update_interval_minutes=settings.update_interval_minutes,
|
||||
default_prompt=DEFAULT_PROMPT,
|
||||
available_models=models,
|
||||
estimator_models=choices,
|
||||
ready=bool(settings.repos and settings.estimator_model and settings.estimator_model in models),
|
||||
)
|
||||
|
||||
|
|
@ -353,6 +392,13 @@ async def _test_estimator_access(settings: ROISettings) -> None:
|
|||
raise HTTPException(status_code=409, detail="The estimator key could not connect to the gateway.") from None
|
||||
|
||||
|
||||
def _gateway_user_reader(repository: ConfigRepository) -> GatewayUserReader:
|
||||
async def get_emails() -> frozenset[str]:
|
||||
return await read_gateway_user_emails(spend_prisma_client(repository.prisma_client))
|
||||
|
||||
return get_emails
|
||||
|
||||
|
||||
def _spend_reader(repository: ConfigRepository) -> SpendReader:
|
||||
async def get_spend(start: date, end: date) -> tuple[ROISpendRecord, ...]:
|
||||
prisma_client: Final = spend_prisma_client(repository.prisma_client)
|
||||
|
|
@ -540,6 +586,7 @@ async def start_roi_calculator_sync(
|
|||
_router_estimator_models(settings.estimator_model),
|
||||
SyncStore(repository.prisma_client),
|
||||
branch_spend_reader=_branch_spend_reader(repository, settings),
|
||||
gateway_user_reader=_gateway_user_reader(repository),
|
||||
):
|
||||
raise HTTPException(status_code=409, detail="A sync is already running.")
|
||||
return manager.status
|
||||
|
|
@ -686,6 +733,7 @@ async def run_scheduled_sync() -> None:
|
|||
coordinator=store,
|
||||
scheduled_interval=settings.update_interval_minutes,
|
||||
branch_spend_reader=_branch_spend_reader(repository, settings),
|
||||
gateway_user_reader=_gateway_user_reader(repository),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from litellm.types.roi_calculator import (
|
|||
ROIBranchAttribution,
|
||||
ROIBranchMetrics,
|
||||
ROIPersonSummary,
|
||||
ROIPullEvidence,
|
||||
ROIPullRecord,
|
||||
ROIPullSummary,
|
||||
ROIReport,
|
||||
|
|
@ -27,7 +28,7 @@ def normalize_email(value: str | None) -> str:
|
|||
|
||||
|
||||
def match_identity(
|
||||
pull: ROIPullRecord,
|
||||
pull: ROIPullRecord | ROIPullEvidence,
|
||||
observed_emails: frozenset[str],
|
||||
mappings: Mapping[str, str],
|
||||
) -> tuple[str, str]:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import asyncio
|
||||
from collections.abc import Awaitable, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
|
|
@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
|||
from typing_extensions import ReadOnly, TypedDict, Unpack
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.roi_calculator.analytics import match_identity, normalize_email
|
||||
from litellm.proxy.roi_calculator.estimator import CompletionCaller, Estimator, EstimatorModel, cache_context
|
||||
from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError
|
||||
from litellm.proxy.roi_calculator.pull_cache import cache_key, settings_fingerprint
|
||||
|
|
@ -29,6 +30,7 @@ from litellm.types.roi_calculator import (
|
|||
)
|
||||
|
||||
PR_CONCURRENCY: Final = 3
|
||||
_GATEWAY_USER_PAGE_SIZE: Final = 1000
|
||||
_ESTIMATE_ADAPTER: Final = TypeAdapter(ROIEstimate)
|
||||
_REPORT_ADAPTER: Final = TypeAdapter(ROIReport)
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -76,6 +78,8 @@ class _UserTable(Protocol):
|
|||
|
||||
|
||||
class _PrismaDatabase(Protocol):
|
||||
async def query_raw(self, query: str, *args: object) -> object: ...
|
||||
|
||||
@property
|
||||
def litellm_dailyuserspend(self) -> _DailySpendTable: ...
|
||||
|
||||
|
|
@ -124,8 +128,6 @@ async def read_spend(
|
|||
start: date,
|
||||
end: date,
|
||||
) -> tuple[ROISpendRecord, ...]:
|
||||
from litellm.proxy.roi_calculator.analytics import normalize_email
|
||||
|
||||
database: Final = prisma_client.db
|
||||
daily_table: Final = database.litellm_dailyuserspend
|
||||
group_by: Final = TypeAdapter(list[Literal["user_id", "date"]]).validate_python(("user_id", "date"))
|
||||
|
|
@ -166,6 +168,35 @@ async def read_spend(
|
|||
)
|
||||
|
||||
|
||||
async def _gateway_users(database: _PrismaDatabase) -> AsyncIterator[_UserEmail]:
|
||||
cursor: str | None = None # rebind-ok: keyset pagination advances after each bounded page
|
||||
while True:
|
||||
users: tuple[_UserEmail, ...] = _USER_EMAILS.validate_python(
|
||||
await database.query_raw(
|
||||
'SELECT "user_id", "user_email" FROM "LiteLLM_UserTable" '
|
||||
'WHERE "user_email" IS NOT NULL AND ($1::text IS NULL OR "user_id" > $1) '
|
||||
'ORDER BY "user_id" LIMIT $2',
|
||||
cursor,
|
||||
_GATEWAY_USER_PAGE_SIZE,
|
||||
)
|
||||
)
|
||||
for user in users:
|
||||
yield user
|
||||
if len(users) < _GATEWAY_USER_PAGE_SIZE:
|
||||
return
|
||||
cursor = users[-1].user_id
|
||||
|
||||
|
||||
async def read_gateway_user_emails(prisma_client: _SpendPrismaClient) -> frozenset[str]:
|
||||
return frozenset(
|
||||
[email async for user in _gateway_users(prisma_client.db) if (email := normalize_email(user.user_email))]
|
||||
)
|
||||
|
||||
|
||||
class GatewayUserReader(Protocol):
|
||||
def __call__(self) -> Awaitable[frozenset[str]]: ...
|
||||
|
||||
|
||||
class GitHubFactory(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
|
|
@ -206,6 +237,26 @@ def _utc_now() -> datetime:
|
|||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _unlinked_estimate(
|
||||
pull: ROIPullEvidence | ROIPullRecord,
|
||||
gateway_emails: frozenset[str],
|
||||
mappings: Mapping[str, str],
|
||||
) -> ROIEstimate | None:
|
||||
email, method = match_identity(pull, gateway_emails, mappings)
|
||||
if email and email in gateway_emails:
|
||||
return None
|
||||
reason: Final = (
|
||||
"Multiple gateway users match this author."
|
||||
if method == "ambiguous emails"
|
||||
else "This author is not linked to a registered gateway user."
|
||||
)
|
||||
return {
|
||||
"status": "needs_review",
|
||||
"hours": None,
|
||||
"reasoning": f"Not estimated: {reason} Link the author to a gateway user and run analysis again.",
|
||||
}
|
||||
|
||||
|
||||
async def _estimate_with_fallback(
|
||||
estimator: Estimator,
|
||||
evidence: ROIPullEvidence,
|
||||
|
|
@ -322,8 +373,8 @@ def _processed_records(processed: tuple[_ProcessedPull, ...]) -> Mapping[int, RO
|
|||
raise SourceError(
|
||||
"The repository source could not provide PR metadata. No new report was published; try analysis again later."
|
||||
)
|
||||
if any(item.record["estimate"]["status"] == "error" for item in processed) and not any(
|
||||
item.record["estimate"]["status"] == "estimated" for item in processed
|
||||
if any(item.record["estimate"]["status"] == "error" for item in processed) and all(
|
||||
item.record["estimate"]["status"] == "error" or item.metadata_unavailable for item in processed
|
||||
):
|
||||
raise SourceError(
|
||||
"The estimator could not score any merged changes. No new report was published; "
|
||||
|
|
@ -399,6 +450,8 @@ class SyncManager:
|
|||
coordinator: SyncCoordinator | None = None,
|
||||
scheduled_interval: float = 0,
|
||||
branch_spend_reader: BranchSpendReader | None = None,
|
||||
*,
|
||||
gateway_user_reader: GatewayUserReader,
|
||||
) -> bool:
|
||||
async with self._start_lock:
|
||||
if not settings.repos or not settings.estimator_model:
|
||||
|
|
@ -439,6 +492,7 @@ class SyncManager:
|
|||
coordinator,
|
||||
owner,
|
||||
branch_spend_reader,
|
||||
gateway_user_reader,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
|
@ -481,12 +535,14 @@ class SyncManager:
|
|||
coordinator: SyncCoordinator | None,
|
||||
owner: str,
|
||||
branch_spend_reader: BranchSpendReader | None,
|
||||
gateway_user_reader: GatewayUserReader,
|
||||
) -> None:
|
||||
monitor: Final = asyncio.create_task(self._heartbeat(asyncio.current_task(), coordinator, owner))
|
||||
github: Final = self._github_factory(settings, github_transport)
|
||||
try:
|
||||
end: Final = self._clock().date()
|
||||
start: Final = end - timedelta(days=settings.backfill_days - 1)
|
||||
gateway_emails: Final = await gateway_user_reader()
|
||||
spend: Final = await spend_reader(start, end)
|
||||
self._update_status(phase="repositories", stage="Reading configured repositories")
|
||||
repositories: Final = await _read_repositories(github, settings.repos, start, end)
|
||||
|
|
@ -545,15 +601,21 @@ class SyncManager:
|
|||
await _cache_estimated_pull(
|
||||
repository, key, cached_record, cached_pull if saved is not None else None
|
||||
)
|
||||
self._update_estimate_progress(cached_record["estimate"])
|
||||
return _ProcessedPull(index, cached_record)
|
||||
cached_estimate: Final = (
|
||||
_unlinked_estimate(cached_record, gateway_emails, settings.identity_map)
|
||||
or cached_record["estimate"]
|
||||
)
|
||||
self._update_estimate_progress(cached_estimate)
|
||||
return _ProcessedPull(index, {**cached_record, "estimate": cached_estimate})
|
||||
try:
|
||||
evidence: Final = await github.evidence(repo, pull)
|
||||
except SourceError as exc:
|
||||
unavailable: Final = await _unavailable_record(github, settings, repo, pull, exc)
|
||||
self._update_estimate_progress(unavailable["estimate"])
|
||||
return _ProcessedPull(index, unavailable, metadata_unavailable=True)
|
||||
estimate: Final = await _estimate_with_fallback(estimator, evidence)
|
||||
estimate: Final = _unlinked_estimate(
|
||||
evidence, gateway_emails, settings.identity_map
|
||||
) or await _estimate_with_fallback(estimator, evidence)
|
||||
evidence_item: Final = GitHubPullListItem.model_validate(
|
||||
MappingProxyType(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -134,6 +134,11 @@ class ROISettingsUpdate(BaseModel):
|
|||
update_interval_minutes: float | None = Field(default=None, ge=0, le=43200, allow_inf_nan=False)
|
||||
|
||||
|
||||
class ROIEstimatorModel(BaseModel):
|
||||
model_name: str
|
||||
provider_models: tuple[str, ...]
|
||||
|
||||
|
||||
class ROISettingsResponse(BaseModel):
|
||||
source_provider: Literal["github", "gitlab"] = "github"
|
||||
gitlab_api_url: str = "https://gitlab.com/api/v4"
|
||||
|
|
@ -149,6 +154,7 @@ class ROISettingsResponse(BaseModel):
|
|||
has_github_token: bool
|
||||
default_prompt: str
|
||||
available_models: tuple[str, ...]
|
||||
estimator_models: tuple[ROIEstimatorModel, ...] = ()
|
||||
ready: bool
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
|||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
|
||||
from litellm.proxy.management_endpoints.roi_calculator_endpoints import (
|
||||
_estimator_choices_from_deployments,
|
||||
_estimator_models_from_deployments,
|
||||
_gateway_transport,
|
||||
_next_update,
|
||||
|
|
@ -411,3 +412,66 @@ def test_old_source_report_is_not_returned_when_matching_new_source_identity() -
|
|||
assert matched.status_code == 200
|
||||
assert matched.json()["report"] is None
|
||||
assert matched.json()["identity_map"] == {"dev.name": "dev@example.test"}
|
||||
|
||||
|
||||
def test_estimator_choices_show_underlying_models_and_exclude_non_chat_routes() -> None:
|
||||
deployments: Final = (
|
||||
{
|
||||
"model_name": "estimator",
|
||||
"litellm_params": {"model": "deployment-name"},
|
||||
"model_info": {"base_model": "gpt-6-luna", "mode": "chat"},
|
||||
},
|
||||
{
|
||||
"model_name": "estimator",
|
||||
"litellm_params": {"model": "second-deployment"},
|
||||
"model_info": {"base_model": "gpt-6-luna", "mode": "chat"},
|
||||
},
|
||||
{
|
||||
"model_name": "embeddings",
|
||||
"litellm_params": {"model": "custom-embedding"},
|
||||
"model_info": {"mode": "embedding"},
|
||||
},
|
||||
{
|
||||
"model_name": "image",
|
||||
"litellm_params": {"model": "custom-image"},
|
||||
"model_info": {"mode": "image_generation"},
|
||||
},
|
||||
{"model_name": "*", "litellm_params": {"model": "openai/*"}},
|
||||
{"model_name": "missing", "litellm_params": {}},
|
||||
{"model_name": "custom-chat", "litellm_params": {"model": "openai/private-model"}},
|
||||
)
|
||||
choices: Final = _estimator_choices_from_deployments(deployments)
|
||||
assert tuple((choice.model_name, choice.provider_models) for choice in choices) == (
|
||||
("custom-chat", ("openai/private-model",)),
|
||||
("estimator", ("gpt-6-luna",)),
|
||||
)
|
||||
|
||||
|
||||
def test_estimator_picker_keeps_callable_aliases_and_routing_groups(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.router import Router
|
||||
|
||||
configured_router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "concrete",
|
||||
"litellm_params": {"model": "openai/gpt-6-luna", "api_key": "test"},
|
||||
},
|
||||
{
|
||||
"model_name": "team-only",
|
||||
"litellm_params": {"model": "openai/gpt-6-luna", "api_key": "test"},
|
||||
"model_info": {"team_id": "other-team", "team_public_model_name": "private-estimator"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"friendly": "concrete"},
|
||||
routing_groups=[{"group_name": "balanced", "models": ["concrete"], "routing_strategy": "simple-shuffle"}],
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", configured_router)
|
||||
client: Final = _client(LitellmUserRoles.PROXY_ADMIN, _ConfigRepository())
|
||||
for name in ("friendly", "balanced"):
|
||||
response: Final = client.put("/roi-calculator/settings", json={"repos": ["org/repo"], "estimator_model": name})
|
||||
assert response.status_code == 200, response.text
|
||||
settings: Final = response.json()
|
||||
assert settings["ready"] is True
|
||||
assert set(settings["available_models"]) == {"concrete", "friendly", "balanced"}
|
||||
assert {"model_name": name, "provider_models": ["openai/gpt-6-luna"]} in settings["estimator_models"]
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from pydantic import TypeAdapter
|
|||
from litellm.proxy.roi_calculator.analytics import summarize
|
||||
from litellm.proxy.roi_calculator.estimator import CompletionCaller
|
||||
from litellm.proxy.roi_calculator.github import GitHubPullListItem
|
||||
from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_spend
|
||||
from litellm.proxy.roi_calculator.sync import SpendReader, SyncManager, read_gateway_user_emails, read_spend
|
||||
from litellm.types.roi_calculator import (
|
||||
ROIBranchSpend,
|
||||
ROICompletionRequest,
|
||||
|
|
@ -130,19 +130,40 @@ class _UserTable:
|
|||
where: Mapping[str, object],
|
||||
) -> Sequence[Mapping[str, str | None]]:
|
||||
_assert_json_round_trip({"where": where})
|
||||
if where == {"user_email": {"not": None}}:
|
||||
return (
|
||||
{"user_id": "u1", "user_email": " Alice@Example.com "},
|
||||
{"user_id": "inactive", "user_email": "inactive@example.com"},
|
||||
{"user_id": "invalid", "user_email": "not-an-email"},
|
||||
{"user_id": "private", "user_email": "123@users.noreply.github.com"},
|
||||
)
|
||||
assert where == {"user_id": {"in": ["missing", "team@example.com", "u1"]}}
|
||||
return (MappingProxyType({"user_id": "u1", "user_email": " Alice@Example.com "}),)
|
||||
|
||||
|
||||
class _SpendDatabase:
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, directory: tuple[Mapping[str, str], ...] = ()) -> None:
|
||||
self.litellm_dailyuserspend: Final = _DailySpendTable()
|
||||
self.litellm_usertable: Final = _UserTable()
|
||||
self.directory: Final = directory or (
|
||||
{"user_id": "inactive", "user_email": "inactive@example.com"},
|
||||
{"user_id": "invalid", "user_email": "not-an-email"},
|
||||
{"user_id": "private", "user_email": "123@users.noreply.github.com"},
|
||||
{"user_id": "u1", "user_email": " Alice@Example.com "},
|
||||
)
|
||||
self.pages_read = 0
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> object:
|
||||
cursor, size = args
|
||||
assert cursor is None or isinstance(cursor, str)
|
||||
assert isinstance(size, int) and 0 < size <= 1000
|
||||
self.pages_read += 1
|
||||
return tuple(row for row in self.directory if cursor is None or row["user_id"] > cursor)[:size]
|
||||
|
||||
|
||||
class _SpendPrismaClient:
|
||||
def __init__(self) -> None:
|
||||
self.db: Final = _SpendDatabase()
|
||||
def __init__(self, directory: tuple[Mapping[str, str], ...] = ()) -> None:
|
||||
self.db: Final = _SpendDatabase(directory)
|
||||
|
||||
|
||||
def _settings(estimator_prompt: str = "Estimate effort.") -> ROISettings:
|
||||
|
|
@ -196,6 +217,10 @@ def _spend_reader() -> SpendReader:
|
|||
return read
|
||||
|
||||
|
||||
async def _gateway_users() -> frozenset[str]:
|
||||
return frozenset({"alice@example.com"})
|
||||
|
||||
|
||||
def _completion() -> CompletionCaller:
|
||||
async def complete(request: ROICompletionRequest) -> object:
|
||||
assert request.model == "test-estimator"
|
||||
|
|
@ -224,7 +249,9 @@ async def test_unchanged_estimated_pull_refreshes_identity_without_model_call()
|
|||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
complete: Final = _completion()
|
||||
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), complete, _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), complete, _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
async def unexpected_completion(request: ROICompletionRequest) -> object:
|
||||
|
|
@ -236,6 +263,7 @@ async def test_unchanged_estimated_pull_refreshes_identity_without_model_call()
|
|||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
_transport(unexpected_details=True, profile_email="new@example.com"),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
|
|
@ -305,11 +333,24 @@ async def test_gitlab_cache_refreshes_branch_attribution_when_source_access_chan
|
|||
async def branch_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]:
|
||||
return (ROIBranchSpend(repo="gitlab.com/" + (after or "dev/fork"), branch="feature", spend=2.5, requests=3),)
|
||||
|
||||
assert await manager.start(settings, repository, _spend_reader(), _completion(), _gitlab_transport(before))
|
||||
assert await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_gitlab_transport(before),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
assert await manager.start(
|
||||
settings, repository, _spend_reader(), _completion(), _gitlab_transport(after), branch_spend_reader=branch_spend
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_gitlab_transport(after),
|
||||
branch_spend_reader=branch_spend,
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
|
|
@ -321,7 +362,14 @@ async def test_gitlab_cache_refreshes_branch_attribution_when_source_access_chan
|
|||
async def unexpected_completion(request: ROICompletionRequest) -> object:
|
||||
raise AssertionError("Unchanged source metadata must reuse the estimate")
|
||||
|
||||
assert await manager.start(settings, repository, _spend_reader(), unexpected_completion, _gitlab_transport(after))
|
||||
assert await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
_gitlab_transport(after),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
assert manager.status.reused == 1
|
||||
|
|
@ -343,6 +391,7 @@ async def test_unreadable_gitlab_details_keep_known_branch_costs() -> None:
|
|||
_completion(),
|
||||
_gitlab_transport("dev/fork", details_fail=True),
|
||||
branch_spend_reader=branch_spend,
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
|
|
@ -390,7 +439,9 @@ async def test_metadata_outage_keeps_previous_report_and_retries_on_next_run() -
|
|||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
previous: Final = repository.values["roi_calculator_report"]
|
||||
assert await manager.start(
|
||||
|
|
@ -399,6 +450,7 @@ async def test_metadata_outage_keeps_previous_report_and_retries_on_next_run() -
|
|||
_spend_reader(),
|
||||
_completion(),
|
||||
_transport(pull_detail_status=500),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
|
|
@ -412,6 +464,7 @@ async def test_metadata_outage_keeps_previous_report_and_retries_on_next_run() -
|
|||
_spend_reader(),
|
||||
_completion(),
|
||||
_transport(),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
recovered: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
|
|
@ -426,7 +479,9 @@ async def test_cancelling_estimation_leaves_the_previous_report_unchanged() -> N
|
|||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
previous_report: Final = repository.values["roi_calculator_report"]
|
||||
|
||||
|
|
@ -441,6 +496,7 @@ async def test_cancelling_estimation_leaves_the_previous_report_unchanged() -> N
|
|||
_spend_reader(),
|
||||
blocked_completion,
|
||||
_transport(),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await entered_estimator.wait()
|
||||
|
||||
|
|
@ -453,11 +509,15 @@ async def test_cancelling_estimation_leaves_the_previous_report_unchanged() -> N
|
|||
async def test_immediate_cancel_allows_another_run() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
assert await manager.cancel()
|
||||
assert manager.status.phase == "cancelled"
|
||||
assert manager.status.finished_at is not None
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
|
||||
|
|
@ -466,7 +526,9 @@ async def test_immediate_cancel_allows_another_run() -> None:
|
|||
async def test_saved_estimates_survive_report_reset() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
repository.values = MappingProxyType(
|
||||
{key: value for key, value in repository.values.items() if key != "roi_calculator_report"}
|
||||
|
|
@ -477,7 +539,12 @@ async def test_saved_estimates_survive_report_reset() -> None:
|
|||
|
||||
restarted: Final = SyncManager(clock=_fixed_now)
|
||||
assert await restarted.start(
|
||||
_settings(), repository, _spend_reader(), unexpected_completion, _transport(unexpected_details=True)
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
_transport(unexpected_details=True),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(restarted)
|
||||
assert restarted.status.phase == "complete"
|
||||
|
|
@ -525,16 +592,34 @@ async def test_expired_lease_can_restart_without_restarting_the_gateway() -> Non
|
|||
cancelled.set()
|
||||
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), blocked_completion, _transport(), coordinator=coordinator
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
blocked_completion,
|
||||
_transport(),
|
||||
coordinator=coordinator,
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await entered.wait()
|
||||
assert not await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), coordinator=coordinator
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_transport(),
|
||||
coordinator=coordinator,
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
assert coordinator.current is not None
|
||||
coordinator.current = coordinator.current.model_copy(update={"running": False, "phase": "error"})
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), coordinator=coordinator
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_transport(),
|
||||
coordinator=coordinator,
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert cancelled.is_set()
|
||||
|
|
@ -558,7 +643,14 @@ async def test_one_unreadable_pr_preserves_other_estimates_in_report() -> None:
|
|||
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), httpx.MockTransport(respond))
|
||||
assert await manager.start(
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
httpx.MockTransport(respond),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert tuple((pull["number"], pull["estimate"]["status"]) for pull in report["pulls"]) == (
|
||||
|
|
@ -597,7 +689,12 @@ async def test_unavailable_repository_publishes_flagged_partial_report_and_recov
|
|||
settings: Final = _settings().model_copy(update=MappingProxyType({"repos": ("org/repo", "org/unavailable")}))
|
||||
|
||||
assert await manager.start(
|
||||
settings, repository, _spend_reader(), _completion(), _repository_outage_transport(status)
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_repository_outage_transport(status),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
|
|
@ -617,7 +714,12 @@ async def test_unavailable_repository_publishes_flagged_partial_report_and_recov
|
|||
raise AssertionError("The healthy repository's estimate must be reused after recovery")
|
||||
|
||||
assert await manager.start(
|
||||
settings, repository, _spend_reader(), unexpected_completion, _repository_outage_transport(200)
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
_repository_outage_transport(200),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
recovered: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
|
|
@ -633,7 +735,14 @@ async def test_repository_outage_without_usable_pulls_preserves_previous_report(
|
|||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
settings: Final = _settings().model_copy(update=MappingProxyType({"repos": ("org/repo", "org/unavailable")}))
|
||||
assert await manager.start(settings, repository, _spend_reader(), _completion(), _repository_outage_transport(200))
|
||||
assert await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
_repository_outage_transport(200),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
previous: Final = repository.values["roi_calculator_report"]
|
||||
|
||||
|
|
@ -643,6 +752,7 @@ async def test_repository_outage_without_usable_pulls_preserves_previous_report(
|
|||
_spend_reader(),
|
||||
_completion(),
|
||||
_repository_outage_transport(403, all_unavailable=all_unavailable, healthy_empty=not all_unavailable),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "error"
|
||||
|
|
@ -662,7 +772,14 @@ async def test_reused_profile_preserves_email_only_when_lookup_fails(profile_sta
|
|||
return httpx.Response(200, content=_COMMITS_JSON.replace("alice@example.com", ""))
|
||||
return baseline.handle_request(request)
|
||||
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), httpx.MockTransport(respond))
|
||||
assert await manager.start(
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
_completion(),
|
||||
httpx.MockTransport(respond),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
def refreshed(request: httpx.Request) -> httpx.Response:
|
||||
|
|
@ -674,13 +791,18 @@ async def test_reused_profile_preserves_email_only_when_lookup_fails(profile_sta
|
|||
raise AssertionError("A reused estimate must not call the estimator")
|
||||
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), unexpected_completion, httpx.MockTransport(refreshed)
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
httpx.MockTransport(refreshed),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
expected: Final = "" if profile_status == 200 else "alice@example.com"
|
||||
assert manager.status.phase == "complete"
|
||||
assert manager.status.reused == 1
|
||||
assert manager.status.reused == (0 if profile_status == 200 else 1)
|
||||
assert report["pulls"][0]["profile_email"] == expected
|
||||
assert report["pulls"][0]["emails"] == ((expected,) if expected else ())
|
||||
assert summarize(report, MappingProxyType({}))["metrics"]["cost_per_hour"] == (None if profile_status == 200 else 3)
|
||||
|
|
@ -695,7 +817,12 @@ async def test_reused_profile_preserves_email_only_when_lookup_fails(profile_sta
|
|||
|
||||
restarted: Final = SyncManager(clock=_fixed_now)
|
||||
assert await restarted.start(
|
||||
_settings(), repository, _spend_reader(), unexpected_completion, httpx.MockTransport(unavailable_profile)
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
unexpected_completion,
|
||||
httpx.MockTransport(unavailable_profile),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(restarted)
|
||||
subsequent: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
|
|
@ -708,7 +835,9 @@ async def test_reused_profile_preserves_email_only_when_lookup_fails(profile_sta
|
|||
async def test_complete_estimator_outage_preserves_report_and_recovers() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
assert await manager.start(_settings(), repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
previous: Final = repository.values["roi_calculator_report"]
|
||||
changed: Final = _settings(estimator_prompt="Updated estimation instructions")
|
||||
|
|
@ -716,13 +845,213 @@ async def test_complete_estimator_outage_preserves_report_and_recovers() -> None
|
|||
async def failed_completion(request: ROICompletionRequest) -> object:
|
||||
raise httpx.ConnectError("Estimator unavailable")
|
||||
|
||||
assert await manager.start(changed, repository, _spend_reader(), failed_completion, _transport())
|
||||
assert await manager.start(
|
||||
changed, repository, _spend_reader(), failed_completion, _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "error"
|
||||
assert manager.status.error is not None and "No new report was published" in manager.status.error
|
||||
assert repository.values["roi_calculator_report"] == previous
|
||||
assert await manager.start(changed, repository, _spend_reader(), _completion(), _transport())
|
||||
assert await manager.start(
|
||||
changed, repository, _spend_reader(), _completion(), _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
recovered: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert recovered["pulls"][0]["estimate"]["hours"] == 4
|
||||
|
||||
|
||||
class _CompletionRecorder:
|
||||
def __init__(self) -> None:
|
||||
self.requests: tuple[ROICompletionRequest, ...] = ()
|
||||
|
||||
async def __call__(self, request: ROICompletionRequest) -> object:
|
||||
self.requests = (*self.requests, request)
|
||||
return await _completion()(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("registered", "mapping", "expected_calls"),
|
||||
(
|
||||
(frozenset(), MappingProxyType({}), 0),
|
||||
(frozenset({"alice@example.com"}), MappingProxyType({}), 1),
|
||||
(frozenset({"other@example.com"}), MappingProxyType({}), 0),
|
||||
(frozenset({"other@example.com"}), MappingProxyType({"alice": "other@example.com"}), 1),
|
||||
(frozenset({"alice@example.com"}), MappingProxyType({"alice": "outside@example.com"}), 0),
|
||||
(frozenset({"alice@example.com", "profile@example.com"}), MappingProxyType({}), 0),
|
||||
),
|
||||
)
|
||||
async def test_only_authors_linked_to_registered_gateway_users_trigger_estimation(
|
||||
registered: frozenset[str], mapping: Mapping[str, str], expected_calls: int
|
||||
) -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
recorder: Final = _CompletionRecorder()
|
||||
settings: Final = _settings().model_copy(update={"identity_map": mapping})
|
||||
|
||||
async def users() -> frozenset[str]:
|
||||
return registered
|
||||
|
||||
assert await manager.start(
|
||||
settings,
|
||||
repository,
|
||||
_spend_reader(),
|
||||
recorder,
|
||||
_transport(profile_email="profile@example.com"),
|
||||
gateway_user_reader=users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
estimate: Final = report["pulls"][0]["estimate"]
|
||||
assert manager.status.phase == "complete"
|
||||
assert len(recorder.requests) == expected_calls
|
||||
assert repository.pull_writes == expected_calls
|
||||
assert estimate["status"] == ("estimated" if expected_calls else "needs_review")
|
||||
assert estimate["hours"] == (4 if expected_calls else None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registered_author_without_spend_is_estimated() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
recorder: Final = _CompletionRecorder()
|
||||
|
||||
async def no_spend(start: date, end: date) -> tuple[ROISpendRecord, ...]:
|
||||
return ()
|
||||
|
||||
assert await manager.start(
|
||||
_settings(), repository, no_spend, recorder, _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert len(recorder.requests) == 1
|
||||
assert report["pulls"][0]["estimate"]["hours"] == 4
|
||||
assert report["spend"] == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlinked_author_is_estimated_after_linking_and_cached_estimate_is_hidden_after_unlinking() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
recorder: Final = _CompletionRecorder()
|
||||
|
||||
async def users() -> frozenset[str]:
|
||||
return frozenset({"member@example.com"})
|
||||
|
||||
async def run(settings: ROISettings) -> ROIReport:
|
||||
assert await manager.start(
|
||||
settings, repository, _spend_reader(), recorder, _transport(), gateway_user_reader=users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete"
|
||||
return TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
|
||||
unlinked: Final = await run(_settings())
|
||||
assert unlinked["pulls"][0]["estimate"]["hours"] is None
|
||||
assert len(recorder.requests) == 0
|
||||
linked_settings: Final = _settings().model_copy(update={"identity_map": {"alice": "member@example.com"}})
|
||||
linked: Final = await run(linked_settings)
|
||||
assert linked["pulls"][0]["estimate"]["hours"] == 4
|
||||
assert len(recorder.requests) == 1
|
||||
unlinked_again: Final = await run(_settings())
|
||||
assert unlinked_again["pulls"][0]["estimate"]["hours"] is None
|
||||
assert manager.status.reused == 0
|
||||
assert len(recorder.requests) == 1
|
||||
relinked: Final = await run(linked_settings)
|
||||
assert relinked["pulls"][0]["estimate"]["hours"] == 4
|
||||
assert manager.status.reused == 1
|
||||
assert len(recorder.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unavailable_gateway_directory_stops_estimation_and_preserves_report() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
recorder: Final = _CompletionRecorder()
|
||||
assert await manager.start(
|
||||
_settings(), repository, _spend_reader(), recorder, _transport(), gateway_user_reader=_gateway_users
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
previous: Final = repository.values["roi_calculator_report"]
|
||||
|
||||
async def unavailable_users() -> frozenset[str]:
|
||||
raise ConnectionError("Gateway directory unavailable")
|
||||
|
||||
assert await manager.start(
|
||||
_settings("Changed prompt"),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
recorder,
|
||||
_transport(),
|
||||
gateway_user_reader=unavailable_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "error"
|
||||
assert len(recorder.requests) == 1
|
||||
assert repository.values["roi_calculator_report"] == previous
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_directory_includes_users_without_spend_and_normalizes_emails() -> None:
|
||||
assert await read_gateway_user_emails(_SpendPrismaClient()) == frozenset(
|
||||
{"alice@example.com", "inactive@example.com"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("size", (1000, 2501))
|
||||
async def test_gateway_directory_reads_every_page(size: int) -> None:
|
||||
directory: Final = tuple(
|
||||
{"user_id": f"user-{index:04d}", "user_email": f" Member-{index}@Example.com "} for index in range(size)
|
||||
)
|
||||
client: Final = _SpendPrismaClient(directory)
|
||||
assert await read_gateway_user_emails(client) == frozenset(f"member-{index}@example.com" for index in range(size))
|
||||
assert client.db.pages_read == size // 1000 + 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unlinked_results_survive_when_the_only_linked_estimate_fails() -> None:
|
||||
repository: Final = _ReportRepository()
|
||||
manager: Final = SyncManager(clock=_fixed_now)
|
||||
baseline: Final = _transport()
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/repos/org/repo/pulls":
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=_PULL_LIST_JSON[:-1]
|
||||
+ ","
|
||||
+ _PULL_LIST_JSON[1:].replace("42", "43").replace("alice", "outsider"),
|
||||
)
|
||||
if request.url.path.startswith("/repos/org/repo/pulls/43"):
|
||||
original: Final = baseline.handle_request(httpx.Request("GET", str(request.url).replace("/43", "/42")))
|
||||
return httpx.Response(
|
||||
original.status_code, content=original.text.replace("42", "43").replace("alice", "outsider")
|
||||
)
|
||||
if request.url.path == "/users/outsider":
|
||||
return httpx.Response(200, json={"email": "outsider@example.com"})
|
||||
return baseline.handle_request(request)
|
||||
|
||||
async def failed_completion(request: ROICompletionRequest) -> object:
|
||||
raise httpx.ConnectError("Estimator unavailable")
|
||||
|
||||
assert await manager.start(
|
||||
_settings(),
|
||||
repository,
|
||||
_spend_reader(),
|
||||
failed_completion,
|
||||
httpx.MockTransport(respond),
|
||||
gateway_user_reader=_gateway_users,
|
||||
)
|
||||
await _wait_until_finished(manager)
|
||||
assert manager.status.phase == "complete", manager.status.error
|
||||
report: Final = TypeAdapter(ROIReport).validate_python(repository.values["roi_calculator_report"])
|
||||
assert tuple(
|
||||
(pull["login"], pull["estimate"]["status"], pull["estimate"]["hours"]) for pull in report["pulls"]
|
||||
) == (
|
||||
("alice", "error", None),
|
||||
("outsider", "needs_review", None),
|
||||
)
|
||||
assert "not linked" in report["pulls"][1]["estimate"]["reasoning"]
|
||||
|
|
|
|||
|
|
@ -3722,7 +3722,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
|
|||
|
|
@ -25,3 +25,13 @@ Rules beyond the enabled set were measured against the whole suite and left off
|
|||
Never run the full unit suite (`npx vitest run` with no path). It is 380 files and thousands of tests, it saturates the machine for many minutes, and CI runs it anyway. Run only the test files your change touches, plus any file whose failure your change could plausibly explain, by passing explicit paths
|
||||
|
||||
Type tests are `*.test-d.ts` files run by the `types` vitest project (`npm run test:types`). Keep them out of the `src/app/(dashboard)/` route group. Vitest matches a tsc error back to the test file by path, the parentheses break that match, and `ignoreSourceErrors: true` then drops the error as if it came from a source file. The test still collects and still reports as passing, so a `.test-d.ts` under a parenthesized directory is green no matter what it asserts. Confirm any new one has teeth by breaking the type it guards and watching it fail
|
||||
|
||||
<!-- BEGIN:nextjs-agent-rules -->
|
||||
|
||||
# This is NOT the Next.js you know
|
||||
|
||||
This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices.
|
||||
|
||||
This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean.
|
||||
|
||||
<!-- END:nextjs-agent-rules -->
|
||||
|
|
|
|||
|
|
@ -2307,11 +2307,6 @@
|
|||
"count": 1
|
||||
}
|
||||
},
|
||||
"src/components/view_logs/index.tsx": {
|
||||
"local/filename-pascal-case": {
|
||||
"count": 1
|
||||
}
|
||||
},
|
||||
"src/components/view_logs/log_filter_logic.tsx": {
|
||||
"local/filename-pascal-case": {
|
||||
"count": 1
|
||||
|
|
|
|||
|
|
@ -38,6 +38,21 @@ describe("Lens demo data", () => {
|
|||
}
|
||||
});
|
||||
|
||||
it("includes a long release review with unique steps, complete details and three failed checks", () => {
|
||||
const run = createLensDemoData().runs[6];
|
||||
const ids = new Set(run.trace.spans.map((span) => span.span_id));
|
||||
expect(run.trace.spans).toHaveLength(362);
|
||||
expect(ids.size).toBe(362);
|
||||
expect(run.trace.summary.error_count).toBe(3);
|
||||
expect(run.trace.summary.status).toBe("ok");
|
||||
for (const span of run.trace.spans) {
|
||||
if (span.parent_span_id) expect(ids.has(span.parent_span_id)).toBe(true);
|
||||
expect(run.details.find((detail) => detail.span_id === span.span_id)).toBeDefined();
|
||||
}
|
||||
expect(JSON.parse(run.details.at(-1)!.input)).toHaveLength(120);
|
||||
expect(run.details.at(-1)!.output).toContain("Hold the release");
|
||||
});
|
||||
|
||||
it("filters time windows locally and rejects writes or unknown reads without network access", async () => {
|
||||
const network = vi.spyOn(globalThis, "fetch");
|
||||
const now = Date.now();
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { createApiClient } from "@/lib/http/client";
|
|||
import type { LensDemo } from "@/components/lens/LensDemoContext";
|
||||
import type { Trace, Span, SpanDetail } from "@/components/view_logs/TraceView/traceTypes";
|
||||
import type { Lens, Finding, Job, Settings } from "./lensData";
|
||||
import { withReleaseCases } from "./lensDemoLongTrace";
|
||||
|
||||
type Scenario = {
|
||||
agent: string;
|
||||
|
|
@ -61,8 +62,8 @@ const scenarios: Scenario[] = [
|
|||
agent: "release_agent",
|
||||
question: "Review the search release",
|
||||
tool: "read_test_results",
|
||||
result: "Search: 86 passed, 1 failed. Unicode query regression remains open.",
|
||||
answer: "Hold the release. The Unicode query regression is still failing.",
|
||||
result: "Search: 117 passed, 3 failed. Cases 17, 63 and 104 returned empty results.",
|
||||
answer: "Hold the release. Three of 120 cases returned empty results. Review cases 17, 63 and 104 before shipping.",
|
||||
},
|
||||
{
|
||||
agent: "support_agent",
|
||||
|
|
@ -206,7 +207,10 @@ function makeTrace(scene: Scenario, index: number, now: number) {
|
|||
}
|
||||
|
||||
export function createLensDemoData(now = Date.now()) {
|
||||
const runs = scenarios.map((scene, index) => makeTrace(scene, index, now));
|
||||
const runs = scenarios.map((scene, index) => {
|
||||
const run = makeTrace(scene, index, now);
|
||||
return index === 6 ? withReleaseCases(run) : run;
|
||||
});
|
||||
const finding = ({
|
||||
id,
|
||||
check,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
import type { Span, SpanDetail, Trace } from "@/components/view_logs/TraceView/traceTypes";
|
||||
|
||||
export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) {
|
||||
const { trace } = run;
|
||||
const root = trace.spans[0];
|
||||
const final = trace.spans.at(-1)!;
|
||||
const caseCount = 120;
|
||||
const caseSpans: Span[] = [];
|
||||
const caseDetails: SpanDetail[] = [];
|
||||
const checks = ["Unicode queries", "Empty results", "Pagination", "Ranking", "Filters", "Permissions"];
|
||||
const failedCases = new Set([17, 63, 104]);
|
||||
for (let index = 1; index <= caseCount; index++) {
|
||||
const failed = failedCases.has(index);
|
||||
const id = (step: number) => (0x70000 + index * 10 + step).toString(16).padStart(16, "0");
|
||||
const question = `Case ${index}: ${checks[(index - 1) % checks.length]}`;
|
||||
const result = failed ? "Expected matching results; received an empty result set" : "Expected results matched";
|
||||
const start = (index - 1) * 2200;
|
||||
const agent: Span = {
|
||||
...root,
|
||||
span_id: id(0),
|
||||
parent_span_id: root.span_id,
|
||||
agent: "search_case",
|
||||
name: "search_case",
|
||||
input_preview: question,
|
||||
start_offset_ms: start,
|
||||
duration_ms: 2100,
|
||||
};
|
||||
const tool: Span = {
|
||||
...agent,
|
||||
span_id: id(1),
|
||||
parent_span_id: agent.span_id,
|
||||
name: "run_search_check",
|
||||
type: "tool",
|
||||
start_offset_ms: start + 100,
|
||||
duration_ms: 800,
|
||||
status: failed ? "error" : "ok",
|
||||
error: failed ? result : null,
|
||||
};
|
||||
const model: Span = {
|
||||
...final,
|
||||
span_id: id(2),
|
||||
parent_span_id: agent.span_id,
|
||||
agent: agent.agent,
|
||||
name: "Review case result",
|
||||
input_preview: question,
|
||||
start_offset_ms: start + 950,
|
||||
duration_ms: 1100,
|
||||
};
|
||||
caseSpans.push(agent, tool, model);
|
||||
for (const span of [agent, tool, model]) {
|
||||
const detail: SpanDetail = {
|
||||
span_id: span.span_id,
|
||||
input:
|
||||
span.type === "tool"
|
||||
? JSON.stringify({ case: index, check: question })
|
||||
: JSON.stringify([{ role: "user", content: question }]),
|
||||
output:
|
||||
span.type === "tool"
|
||||
? result
|
||||
: JSON.stringify([
|
||||
{
|
||||
role: "assistant",
|
||||
content: failed ? `Hold this case for review. ${result}.` : `Case ${index} passed. ${result}.`,
|
||||
},
|
||||
]),
|
||||
attributes: { "gen_ai.agent.name": "search_case", "test.case": String(index), demo: "true" },
|
||||
};
|
||||
caseDetails.push(detail);
|
||||
}
|
||||
}
|
||||
const finalSpan = { ...final, start_offset_ms: caseCount * 2200 };
|
||||
const duration = finalSpan.start_offset_ms + finalSpan.duration_ms;
|
||||
const spans = [{ ...root, duration_ms: duration }, ...caseSpans, finalSpan];
|
||||
const models = spans.filter((span) => span.type === "llm");
|
||||
const summary = {
|
||||
...trace.summary,
|
||||
duration_ms: duration,
|
||||
span_count: spans.length,
|
||||
agent_count: 2,
|
||||
agent_invocations: caseCount + 1,
|
||||
agent_names: [root.name, "search_case"],
|
||||
llm_calls: models.length,
|
||||
tool_calls: caseCount,
|
||||
error_count: failedCases.size,
|
||||
input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0),
|
||||
output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0),
|
||||
spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0),
|
||||
};
|
||||
const finalDetail = run.details.at(-1)!;
|
||||
const history = caseDetails.filter((_, index) => index % 3 === 2);
|
||||
return {
|
||||
trace: {
|
||||
summary,
|
||||
spans,
|
||||
agents: [
|
||||
{ ...trace.agents[0], duration_ms: duration, tool_calls: 0 },
|
||||
{
|
||||
name: "search_case",
|
||||
parent_agent: root.name,
|
||||
duration_ms: caseCount * 2100,
|
||||
invocations: caseCount,
|
||||
llm_calls: caseCount,
|
||||
tool_calls: caseCount,
|
||||
spend: summary.spend - (final.spend ?? 0),
|
||||
},
|
||||
],
|
||||
},
|
||||
details: [
|
||||
run.details[0],
|
||||
...caseDetails,
|
||||
{
|
||||
...finalDetail,
|
||||
input: JSON.stringify(
|
||||
history.map((detail) => ({ role: "user", content: JSON.parse(detail.output)[0].content })),
|
||||
),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
|
|
@ -283,6 +284,39 @@ describe("ROICalculatorView", () => {
|
|||
expect(screen.getByRole("textbox", { name: "Repository name" })).toBeEnabled();
|
||||
});
|
||||
|
||||
it("searches by the real model name and saves the selected gateway alias", async () => {
|
||||
const user = userEvent.setup();
|
||||
const modelSettings = {
|
||||
...settings,
|
||||
available_models: ["estimator", "fast-estimator"],
|
||||
estimator_models: [
|
||||
{ model_name: "estimator", provider_models: ["custom-model"] },
|
||||
{ model_name: "fast-estimator", provider_models: ["openai/gpt-6-luna"] },
|
||||
],
|
||||
};
|
||||
vi.mocked(apiClient.get).mockImplementation((path: string) => {
|
||||
if (path === "/roi-calculator/settings") return Promise.resolve(modelSettings);
|
||||
if (path === "/roi-calculator/report") return Promise.resolve({ report: summary });
|
||||
return Promise.resolve(idleStatus);
|
||||
});
|
||||
vi.mocked(apiClient.put).mockResolvedValue({ ...modelSettings, estimator_model: "fast-estimator" });
|
||||
render(<ROICalculatorView accessToken="token" />);
|
||||
await user.click(await screen.findByRole("button", { name: "Settings" }));
|
||||
const search = screen.getByRole("combobox", { name: "Estimator model" });
|
||||
await user.clear(search);
|
||||
await user.type(search, "Luna");
|
||||
expect(screen.queryByRole("option", { name: /custom-model/ })).not.toBeInTheDocument();
|
||||
await user.click(await screen.findByRole("option", { name: /GPT-6 Luna.*Recommended/ }));
|
||||
expect(search).toHaveValue("GPT-6 Luna");
|
||||
await user.click(screen.getByRole("button", { name: "Save settings" }));
|
||||
expect(apiClient.put).toHaveBeenCalledWith(
|
||||
"/roi-calculator/settings",
|
||||
expect.objectContaining({
|
||||
body: expect.objectContaining({ estimator_model: "fast-estimator" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("closes the old settings dialog when saving a different source", async () => {
|
||||
const gitlabSettings = {
|
||||
...settings,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
import React from "react";
|
||||
|
||||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { estimatorModelOptions } from "./roiCalculatorData";
|
||||
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { extractErrorMessage } from "@/utils/errorUtils";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -408,23 +411,19 @@ export default function ROISettingsPanel({
|
|||
{!onboarding && <h3 className="font-semibold">Estimation and updates</h3>}
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="roi-estimator-model">Estimator model</Label>
|
||||
<select
|
||||
id="roi-estimator-model"
|
||||
className="h-9 min-w-0 rounded-md border bg-background pl-3 pr-9 text-sm"
|
||||
disabled={readOnly}
|
||||
<SearchSelect
|
||||
inputId="roi-estimator-model"
|
||||
options={estimatorModelOptions(initialSettings)}
|
||||
value={model}
|
||||
onChange={(event) => setModel(event.target.value)}
|
||||
>
|
||||
<option value="">Select a router model</option>
|
||||
{model && !initialSettings.available_models.includes(model) && (
|
||||
<option value={model}>{model}</option>
|
||||
)}
|
||||
{initialSettings.available_models.map((availableModel) => (
|
||||
<option key={availableModel} value={availableModel}>
|
||||
{availableModel}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
onValueChange={(value) => setModel(value ?? "")}
|
||||
placeholder="Search estimator models"
|
||||
emptyText="No matching models configured on this gateway"
|
||||
disabled={readOnly}
|
||||
className="h-9"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
We recommend GPT-6 Luna for estimating PR effort. Choose a model configured on your gateway.
|
||||
</p>
|
||||
</div>
|
||||
<details>
|
||||
<summary className="cursor-pointer text-sm text-muted-foreground">Advanced estimator options</summary>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
coverageLabel,
|
||||
effortNote,
|
||||
estimateLabel,
|
||||
estimatorModelOptions,
|
||||
filterPulls,
|
||||
formatMoney,
|
||||
formatNumber,
|
||||
|
|
@ -44,9 +45,17 @@ const summary = {
|
|||
describe("ROI calculator display helpers", () => {
|
||||
it("ranks only attributed PR costs, limits the overview to five and preserves report order", () => {
|
||||
const pulls = [2, 6, 1, 4, 3, 5].map((spend) =>
|
||||
pull({ number: spend, branch_cost: { status: "matched", spend, requests: 1, cost_per_hour: spend } }),
|
||||
pull({
|
||||
number: spend,
|
||||
branch_cost: { status: "matched", spend, requests: 1, repo: "github.com/org/repo", branch: "feature" },
|
||||
}),
|
||||
);
|
||||
pulls.push(
|
||||
pull({
|
||||
number: 99,
|
||||
branch_cost: { status: "ambiguous", spend: 99, requests: 1, repo: "github.com/org/repo", branch: "feature" },
|
||||
}),
|
||||
);
|
||||
pulls.push(pull({ number: 99, branch_cost: { status: "ambiguous", spend: 99, requests: 1, cost_per_hour: null } }));
|
||||
pulls.push(pull({ number: 100 }));
|
||||
expect(highestCostPulls(pulls).map((item) => item.number)).toEqual([6, 5, 4, 3, 2]);
|
||||
expect(pulls.map((item) => item.number)).toEqual([2, 6, 1, 4, 3, 5, 99, 100]);
|
||||
|
|
@ -109,3 +118,33 @@ it("exports precise spend, cohort eligibility and safely quoted CSV values", ()
|
|||
expect(csv).toContain('"\'=HYPERLINK(""bad"")","alice;bob","0.0001","4","1","0","true","0.000025"');
|
||||
expect(csv).toContain('"2026-09-01","2026-09-30","without_ai"');
|
||||
});
|
||||
|
||||
it("recommends the real Luna model and preserves its gateway name for requests", () => {
|
||||
expect(
|
||||
estimatorModelOptions({
|
||||
available_models: ["general", "fast-estimator"],
|
||||
estimator_models: [
|
||||
{ model_name: "general", provider_models: ["anthropic/claude-haiku"] },
|
||||
{ model_name: "fast-estimator", provider_models: ["openai/gpt-6-luna"] },
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
value: "fast-estimator",
|
||||
label: "GPT-6 Luna",
|
||||
sublabel: "Recommended · Gateway name: fast-estimator",
|
||||
recommended: true,
|
||||
},
|
||||
{ value: "general", label: "anthropic/claude-haiku", sublabel: "Gateway name: general", recommended: false },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not invent available models or recommend an alias pointing to a different model", () => {
|
||||
expect(
|
||||
estimatorModelOptions({
|
||||
available_models: ["gpt-6-luna"],
|
||||
estimator_models: [{ model_name: "gpt-6-luna", provider_models: ["custom-model"] }],
|
||||
}),
|
||||
).toEqual([{ value: "gpt-6-luna", label: "custom-model", sublabel: "Gateway name: gpt-6-luna", recommended: false }]);
|
||||
expect(estimatorModelOptions({ available_models: [], estimator_models: [] })).toEqual([]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -121,3 +121,25 @@ export const branchCostLabel = (pull: ROIPull): string => {
|
|||
if (pull.branch_cost.status === "unattributed") return "No tagged requests";
|
||||
return formatMoney(pull.branch_cost.spend);
|
||||
};
|
||||
|
||||
export const estimatorModelOptions = (settings: Pick<ROISettings, "available_models" | "estimator_models">) => {
|
||||
const details = new Map(settings.estimator_models?.map((model) => [model.model_name, model]));
|
||||
const isLuna = (name: string) => /(?:^|\/)gpt-6-luna(?:-\d{4}-\d{2}-\d{2})?$/i.test(name);
|
||||
return settings.available_models
|
||||
.map((name) => {
|
||||
const models = details.get(name)?.provider_models ?? [];
|
||||
const recommended = models.length > 0 && models.every(isLuna);
|
||||
const label = [...new Set(models.map((model) => (isLuna(model) ? "GPT-6 Luna" : model)))].join(", ") || name;
|
||||
return {
|
||||
value: name,
|
||||
label,
|
||||
sublabel: [recommended ? "Recommended" : "", label !== name ? `Gateway name: ${name}` : ""]
|
||||
.filter(Boolean)
|
||||
.join(" · "),
|
||||
recommended,
|
||||
};
|
||||
})
|
||||
.sort(
|
||||
(left, right) => Number(right.recommended) - Number(left.recommended) || left.label.localeCompare(right.label),
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -38,11 +38,7 @@ export function CopyButton({
|
|||
<Button
|
||||
variant={iconOnly ? "ghost" : "outline"}
|
||||
size={iconOnly ? "icon-xs" : "xs"}
|
||||
className={cn(
|
||||
"font-mono text-[10px] text-muted-foreground hover:text-foreground",
|
||||
!iconOnly && "gap-1.5",
|
||||
className,
|
||||
)}
|
||||
className={cn("text-xs text-muted-foreground hover:text-foreground", !iconOnly && "gap-1.5", className)}
|
||||
onClick={async () => setCopied(await copyToClipboard(value, copiedLabel))}
|
||||
aria-label={label}
|
||||
>
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ import { errorSource, parseJson, parseMessages, prettyPayload } from "./traceUti
|
|||
|
||||
const ERROR_SOURCE_LABEL: Record<ErrorSource, string> = { tool: "Tool", model: "Model", litellm: "LiteLLM" };
|
||||
const TRACEBACK_MARKER = "Traceback (most recent call last):";
|
||||
const STATUS_TEXT = "px-5 py-2 text-[13px] tracking-[-0.26px] text-trace-duration";
|
||||
const STATUS_TEXT = "px-5 py-2 text-[13px] tracking-[-0.26px] text-muted-foreground";
|
||||
const PAYLOAD_PRE =
|
||||
"font-mono text-[13px] leading-[1.5] tracking-[-0.26px] break-words whitespace-pre-wrap text-trace-text";
|
||||
"font-mono text-[13px] leading-[1.5] tracking-[-0.26px] break-words whitespace-pre-wrap text-foreground";
|
||||
|
||||
/** Exporters record `repr(exc)` + traceback with no separator; keep the exception line. */
|
||||
export const errorHeadline = (error: string): string =>
|
||||
|
|
@ -52,7 +52,7 @@ export function ErrorBlock({ span }: { span: Span }) {
|
|||
return (
|
||||
<section
|
||||
aria-label="Error"
|
||||
className="mx-5 mb-2 rounded-[4px] border-[0.67px] border-destructive/40 bg-destructive/5 px-3 py-2.5"
|
||||
className="mx-3 mb-3 rounded-[4px] border-[0.67px] border-destructive/40 bg-destructive/5 px-3 py-2.5"
|
||||
>
|
||||
<div className="flex items-center gap-2 text-[13px] leading-[1.2] font-medium tracking-[-0.26px] text-destructive">
|
||||
<AlertTriangle className="size-3.5" />
|
||||
|
|
@ -171,7 +171,7 @@ function DiagnosticContent({ accessToken, traceId, traceRef, span }: DetailConte
|
|||
};
|
||||
const query = useQuery(queryOptions);
|
||||
return (
|
||||
<section aria-label="Stored diagnostic" className="mx-5 mb-2 space-y-2">
|
||||
<section aria-label="Stored diagnostic" className="mx-3 mb-3 space-y-2">
|
||||
{span.error_truncated && <p className="text-xs text-muted-foreground">Error preview truncated</p>}
|
||||
{!opened && (
|
||||
<Button variant="outline" size="sm" onClick={() => setOpened(true)}>
|
||||
|
|
@ -215,6 +215,10 @@ export function DetailContent({ accessToken, traceId, traceRef, span }: DetailCo
|
|||
const detailQuery = useSpanDetail(accessToken, traceId, span.span_id, traceRef);
|
||||
const detail = detailQuery.data;
|
||||
const empty = detail && !detail.input && !detail.output;
|
||||
const inputMessages =
|
||||
detail?.input_ui?.kind === "messages"
|
||||
? detail.input_ui.messages.length
|
||||
: parseMessages(detail?.input ?? "")?.length;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col px-2 pt-1 pb-4">
|
||||
|
|
@ -231,7 +235,12 @@ export function DetailContent({ accessToken, traceId, traceRef, span }: DetailCo
|
|||
{detailQuery.isLoading && <div className={STATUS_TEXT}>Loading span…</div>}
|
||||
{detailQuery.isError && <div className={STATUS_TEXT}>Could not load span: {detailQuery.error.message}</div>}
|
||||
{detail?.input ? (
|
||||
<Section title="Input">
|
||||
<Section
|
||||
key={`${span.span_id}:${inputMessages}`}
|
||||
title="Input"
|
||||
count={inputMessages}
|
||||
defaultOpen={!inputMessages || inputMessages <= 8}
|
||||
>
|
||||
<SpanPayload value={detail.input} content={detail.input_ui} span={span} role="input" />
|
||||
</Section>
|
||||
) : null}
|
||||
|
|
@ -241,7 +250,7 @@ export function DetailContent({ accessToken, traceId, traceRef, span }: DetailCo
|
|||
</Section>
|
||||
) : null}
|
||||
{empty && span.status !== "error" && (
|
||||
<div className="py-12 text-center text-[13px] tracking-[-0.26px] text-trace-duration">
|
||||
<div className="py-12 text-center text-[13px] tracking-[-0.26px] text-muted-foreground">
|
||||
No content recorded for this span.
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ const span = (overrides: SpanFields): Span => ({
|
|||
duration_ms: 1300,
|
||||
status: "ok",
|
||||
error: null,
|
||||
error_truncated: false,
|
||||
framework: "",
|
||||
input_preview: "",
|
||||
model: null,
|
||||
input_tokens: 0,
|
||||
|
|
@ -169,13 +171,33 @@ describe("DetailPane", () => {
|
|||
expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1", undefined);
|
||||
});
|
||||
|
||||
it("keeps the output visible when a step contains a long input conversation", async () => {
|
||||
const user = userEvent.setup();
|
||||
const messages = Array.from({ length: 120 }, (_, index) => ({
|
||||
role: "user" as const,
|
||||
content: `Review case ${index}`,
|
||||
}));
|
||||
vi.mocked(agentTraceSpanCall).mockResolvedValue({
|
||||
...details.root,
|
||||
input: JSON.stringify(messages),
|
||||
input_ui: { kind: "messages", messages },
|
||||
});
|
||||
renderPane(spanRow(root));
|
||||
const input = await screen.findByRole("button", { name: "Input 120 messages" });
|
||||
expect(input).toHaveAttribute("aria-expanded", "false");
|
||||
expect(screen.getByText("Customer acme-404 is on the Enterprise plan.")).toBeVisible();
|
||||
expect(screen.getByText("Review case 119")).not.toBeVisible();
|
||||
await user.click(input);
|
||||
expect(screen.getByText("Review case 119")).toBeVisible();
|
||||
});
|
||||
|
||||
it("shows a tool failure as 'Tool · <reason>' with the exception line and no traceback", async () => {
|
||||
renderPane(spanRow(failedTool));
|
||||
const error = screen.getByRole("region", { name: "Error" });
|
||||
expect(error).toHaveTextContent("Tool · ValueError");
|
||||
expect(error).toHaveTextContent("ValueError('customer acme-404 not found in billing DB')");
|
||||
expect(error).not.toHaveTextContent("Traceback");
|
||||
const input = await screen.findByRole("region", { name: "Input" });
|
||||
const input = await screen.findByRole("region", { name: /^Input/ });
|
||||
expect(input).toHaveTextContent("customer_id");
|
||||
expect(input).toHaveTextContent("acme-404");
|
||||
});
|
||||
|
|
@ -231,11 +253,11 @@ describe("DetailPane", () => {
|
|||
expect(writeText.mock.calls[0][0]).toContain("http://proxy.test/v1/traces/t1?format=md&span_id=llm1");
|
||||
});
|
||||
|
||||
it("renders the AI tool call as a card and expands a long argument on click", async () => {
|
||||
it("renders the assistant tool call as a card and expands a long argument on click", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderPane(spanRow(llm));
|
||||
const output = await screen.findByRole("region", { name: "Output" });
|
||||
expect(output).toHaveTextContent("AI");
|
||||
expect(output).toHaveTextContent("Assistant");
|
||||
expect(output).toHaveTextContent("get_customer_plan");
|
||||
const expand = within(output).getAllByRole("button", { name: "Expand note" })[0];
|
||||
expect(within(output).queryAllByText(LONG_NOTE, { selector: "pre", ignore: "[inert] *" })).toHaveLength(0);
|
||||
|
|
@ -250,11 +272,11 @@ describe("DetailPane", () => {
|
|||
it("collapses the Input section without touching Output", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderPane(spanRow(llm));
|
||||
const input = await screen.findByRole("region", { name: "Input" });
|
||||
const input = await screen.findByRole("region", { name: /^Input/ });
|
||||
const systemText = "You are a LiteLLM support agent.";
|
||||
expect(within(input).getByText(systemText, { ignore: "[inert] *" })).toBeInTheDocument();
|
||||
await user.click(within(input).getByRole("button", { name: "Input" }));
|
||||
expect(within(input).getByRole("button", { name: "Input" })).toHaveAttribute("aria-expanded", "false");
|
||||
await user.click(within(input).getByRole("button", { name: /^Input/ }));
|
||||
expect(within(input).getByRole("button", { name: /^Input/ })).toHaveAttribute("aria-expanded", "false");
|
||||
expect(within(input).queryByText(systemText, { ignore: "[inert] *" })).not.toBeInTheDocument();
|
||||
const output = screen.getByRole("region", { name: "Output" });
|
||||
expect(within(output).getAllByText("get_customer_plan", { ignore: "[inert] *" })).not.toHaveLength(0);
|
||||
|
|
@ -263,12 +285,12 @@ describe("DetailPane", () => {
|
|||
it("renders the standard input_ui / output_ui instead of re-parsing the raw payload", async () => {
|
||||
vi.mocked(agentTraceSpanCall).mockResolvedValue(standardDetail);
|
||||
renderPane(spanRow(llm));
|
||||
const input = await screen.findByRole("region", { name: "Input" });
|
||||
const input = await screen.findByRole("region", { name: /^Input/ });
|
||||
expect(input).toHaveTextContent("ticket_id");
|
||||
expect(input).toHaveTextContent("T-981");
|
||||
expect(input).not.toHaveTextContent("raw input left unparsed");
|
||||
const output = screen.getByRole("region", { name: "Output" });
|
||||
expect(output).toHaveTextContent("AI");
|
||||
expect(output).toHaveTextContent("Assistant");
|
||||
expect(output).toHaveTextContent("Refund approved for T-981.");
|
||||
expect(output).toHaveTextContent("issue_refund");
|
||||
expect(output).toHaveTextContent("amount_usd");
|
||||
|
|
@ -281,7 +303,7 @@ describe("DetailPane", () => {
|
|||
const output = await screen.findByRole("region", { name: "Output" });
|
||||
const result = within(output).getByText("permission denied: /etc/shadow");
|
||||
expect(result).toHaveClass("text-destructive");
|
||||
expect(output).not.toHaveTextContent("AI");
|
||||
expect(output).not.toHaveTextContent("Assistant");
|
||||
});
|
||||
|
||||
it("shows a text output_ui as its plain text", async () => {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { PanelRightClose } from "lucide-react";
|
|||
import { useState } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs";
|
||||
|
||||
import { AttributesDetail } from "./AttributesDetail";
|
||||
import { CopyButton } from "./CopyButton";
|
||||
|
|
@ -49,14 +49,12 @@ function PaneHeader({
|
|||
onClose: () => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex h-6 shrink-0 items-center justify-between gap-4 px-7">
|
||||
<div className="flex h-7 shrink-0 items-center justify-between gap-3 px-5">
|
||||
<div className="flex min-w-0 items-center">
|
||||
<span className="mr-2 shrink-0">
|
||||
<SpanIcon type={type} model={model} error={failed} size="md" />
|
||||
</span>
|
||||
<h2 className="min-w-0 truncate text-[14px] leading-5 font-medium tracking-[-0.56px] text-trace-text">
|
||||
{title}
|
||||
</h2>
|
||||
<h2 className="min-w-0 truncate text-sm leading-5 font-semibold text-foreground">{title}</h2>
|
||||
{idValue && (
|
||||
<span className="ml-2 flex shrink-0">
|
||||
<IdChip value={idValue} label="Copy span ID" />
|
||||
|
|
@ -68,7 +66,7 @@ function PaneHeader({
|
|||
size="icon-xs"
|
||||
onClick={onClose}
|
||||
aria-label="Close details"
|
||||
className="size-6 shrink-0 rounded-[4px] text-trace-text-2"
|
||||
className="size-6 shrink-0 rounded-[4px] text-muted-foreground"
|
||||
>
|
||||
<PanelRightClose className="size-4" />
|
||||
</Button>
|
||||
|
|
@ -78,7 +76,7 @@ function PaneHeader({
|
|||
|
||||
function PaneFooter({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<div className="flex h-10 shrink-0 items-center gap-2 border-t border-trace-border bg-trace-surface px-7">
|
||||
<div className="flex min-h-10 shrink-0 flex-wrap items-center gap-x-3 gap-y-1 border-t bg-background px-5 py-2">
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -87,7 +85,7 @@ function PaneFooter({ children }: { children: React.ReactNode }) {
|
|||
function Meta({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<span>
|
||||
<span className="text-trace-key">{label} </span>
|
||||
<span className="text-muted-foreground">{label} </span>
|
||||
{value}
|
||||
</span>
|
||||
);
|
||||
|
|
@ -116,7 +114,7 @@ function SpanPane({
|
|||
const tokens = span.input_tokens + span.output_tokens;
|
||||
return (
|
||||
<aside
|
||||
className="flex h-full min-w-0 animate-view-fade-in flex-col bg-trace-surface pt-4 text-[13px] text-trace-text motion-reduce:animate-none"
|
||||
className="flex h-full min-w-0 animate-view-fade-in flex-col bg-background pt-4 text-[13px] text-foreground motion-reduce:animate-none"
|
||||
aria-label="Span details"
|
||||
>
|
||||
<PaneHeader
|
||||
|
|
@ -127,41 +125,31 @@ function SpanPane({
|
|||
idValue={span.span_id}
|
||||
onClose={onClose}
|
||||
/>
|
||||
<div role="tablist" aria-label="Span detail sections" className="flex shrink-0 items-center gap-2 py-2 pr-2 pl-6">
|
||||
{TABS.map((t) => (
|
||||
<button
|
||||
key={t.id}
|
||||
type="button"
|
||||
role="tab"
|
||||
aria-selected={tab === t.id}
|
||||
onClick={() => setTab(t.id)}
|
||||
className={cn(
|
||||
"rounded-[4px] px-2 py-1 text-[13px] leading-[18px] font-medium transition-colors duration-100 focus-visible:outline-2 focus-visible:outline-trace-brand motion-reduce:transition-none",
|
||||
tab === t.id
|
||||
? "bg-trace-tab-active text-trace-text hover:bg-trace-tab-hover"
|
||||
: "text-trace-key hover:bg-trace-tab-active",
|
||||
)}
|
||||
>
|
||||
{t.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div key={tab} role="tabpanel" className="min-h-0 flex-1 overflow-auto">
|
||||
{tab === "content" && (
|
||||
<Tabs value={tab} onValueChange={(value) => setTab(value as Tab)} className="min-h-0 flex-1 gap-0">
|
||||
<div className="shrink-0 border-b px-5">
|
||||
<TabsList variant="line" aria-label="Span detail sections" className="h-10 gap-4 px-0">
|
||||
{TABS.map((t) => (
|
||||
<TabsTrigger key={t.id} value={t.id} className="flex-none px-0 text-xs">
|
||||
{t.label}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
</div>
|
||||
<TabsContent value="content" className="min-h-0 overflow-auto pt-3">
|
||||
<DetailContent accessToken={accessToken} traceId={traceId} traceRef={trace.summary.trace_ref} span={span} />
|
||||
)}
|
||||
{tab === "request" && (
|
||||
</TabsContent>
|
||||
<TabsContent value="request" className="min-h-0 overflow-auto pt-3">
|
||||
<RequestDetail span={span} accessToken={accessToken} traceStartMs={Date.parse(trace.summary.start_time)} />
|
||||
)}
|
||||
{tab === "attributes" && (
|
||||
</TabsContent>
|
||||
<TabsContent value="attributes" className="min-h-0 overflow-auto pt-3">
|
||||
<AttributesDetail
|
||||
traceId={traceId}
|
||||
span={span}
|
||||
attributes={detailQuery.data?.attributes}
|
||||
isLoading={detailQuery.isLoading}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
<PaneFooter>
|
||||
<CopyButton
|
||||
value={
|
||||
|
|
@ -172,7 +160,7 @@ function SpanPane({
|
|||
label="Copy step"
|
||||
copiedLabel={demo ? "Step copied" : "Command copied"}
|
||||
/>
|
||||
<div className="ml-auto flex items-center gap-3 text-[13px] text-trace-duration tabular-nums">
|
||||
<div className="ml-auto flex items-center gap-3 text-xs text-muted-foreground tabular-nums">
|
||||
<Meta label="time" value={fmtMs(span.duration_ms)} />
|
||||
{tokens > 0 && <Meta label="tokens" value={fmtTok(tokens)} />}
|
||||
</div>
|
||||
|
|
@ -183,9 +171,9 @@ function SpanPane({
|
|||
|
||||
function GroupMetric({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<div className="border-r border-b border-trace-card-border p-3">
|
||||
<div className="text-[13px] font-medium text-trace-key">{label}</div>
|
||||
<div className="mt-1 text-[13px] text-trace-text tabular-nums">{value}</div>
|
||||
<div className="border-b border-border/60 py-3">
|
||||
<div className="text-[13px] font-medium text-muted-foreground">{label}</div>
|
||||
<div className="mt-1 text-[13px] text-foreground tabular-nums">{value}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -197,7 +185,7 @@ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; o
|
|||
const firstFailure = row.members.find((m) => m.status === "error" && m.error);
|
||||
return (
|
||||
<aside
|
||||
className="flex h-full min-w-0 animate-view-fade-in flex-col bg-trace-surface pt-4 text-[13px] text-trace-text motion-reduce:animate-none"
|
||||
className="flex h-full min-w-0 animate-view-fade-in flex-col bg-background pt-4 text-[13px] text-foreground motion-reduce:animate-none"
|
||||
aria-label="Group details"
|
||||
>
|
||||
<PaneHeader
|
||||
|
|
@ -206,13 +194,13 @@ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; o
|
|||
failed={row.failedCount > 0}
|
||||
title={
|
||||
<>
|
||||
{row.name} <span className="text-trace-duration">×{row.members.length}</span>
|
||||
{row.name} <span className="text-muted-foreground">×{row.members.length}</span>
|
||||
</>
|
||||
}
|
||||
onClose={onClose}
|
||||
/>
|
||||
<div className="min-h-0 flex-1 overflow-auto px-7 py-3">
|
||||
<div className="grid grid-cols-2 overflow-hidden rounded-[4px] border border-trace-card-border bg-trace-surface">
|
||||
<div className="min-h-0 flex-1 overflow-auto px-5 py-3">
|
||||
<div className="grid grid-cols-2 gap-x-5">
|
||||
<GroupMetric label="Invocations" value={row.members.length.toLocaleString()} />
|
||||
<GroupMetric label="Failed" value={row.failedCount.toLocaleString()} />
|
||||
<GroupMetric label="p50 latency" value={fmtMs(row.p50Duration)} />
|
||||
|
|
@ -245,7 +233,7 @@ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; o
|
|||
export function DetailPane({ trace, row, accessToken, onClose }: DetailPaneProps) {
|
||||
if (!row || row.kind === "load-more") {
|
||||
return (
|
||||
<div className="grid h-full place-items-center bg-trace-surface text-[13px] text-trace-duration">
|
||||
<div className="grid h-full place-items-center bg-background text-[13px] text-muted-foreground">
|
||||
Select a span to inspect it.
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ export function IdChip({ value, label, showValue = false }: { value: string; lab
|
|||
aria-label={label}
|
||||
title={value}
|
||||
onClick={() => void copyToClipboard(value, "ID copied")}
|
||||
className="inline-flex h-5 min-w-0 shrink-0 items-center gap-1 rounded-[3px] border border-transparent bg-trace-chip px-1 font-mono text-[13px] leading-none font-medium text-trace-key transition-colors duration-100 hover:text-trace-text-2 focus-visible:outline-2 focus-visible:outline-trace-brand motion-reduce:transition-none"
|
||||
className="inline-flex h-6 min-w-0 shrink-0 items-center gap-1 rounded px-1 text-xs text-muted-foreground transition-colors hover:bg-muted hover:text-foreground focus-visible:outline-2 focus-visible:outline-ring"
|
||||
>
|
||||
<span className="truncate">{showValue ? value : "ID"}</span>
|
||||
<Copy className="size-3 shrink-0" />
|
||||
|
|
|
|||
|
|
@ -24,29 +24,26 @@ export const objectEntries = (value: unknown): KeyValue[] | null => {
|
|||
return Object.entries(value).map(([key, v]): KeyValue => [key, displayValue(v)]);
|
||||
};
|
||||
|
||||
const ROW = "flex min-w-0 items-baseline gap-2.5 py-0.5";
|
||||
const KEY = "shrink-0 text-[13px] leading-[1.2] font-medium tracking-[-0.26px] whitespace-nowrap";
|
||||
const ROW = "flex min-w-0 items-baseline gap-3 border-b border-border/60 py-2 last:border-0";
|
||||
const KEY = "w-24 shrink-0 break-words text-xs leading-5 font-normal";
|
||||
|
||||
function Lead({ children }: { children: React.ReactNode }) {
|
||||
return <span className="flex size-4 shrink-0 items-center justify-center self-center">{children}</span>;
|
||||
return <span className="flex size-3 shrink-0 items-center justify-center self-center">{children}</span>;
|
||||
}
|
||||
|
||||
function KeyValueRow({ entry, mono }: { entry: KeyValue; mono: boolean }) {
|
||||
const [key, value] = entry;
|
||||
const [open, setOpen] = useState(false);
|
||||
const valueClass = cn(
|
||||
"min-w-0 flex-1 text-left text-[13px] leading-[1.2] tracking-[-0.26px] text-trace-text",
|
||||
"min-w-0 flex-1 break-words text-left text-[13px] leading-5 text-foreground",
|
||||
mono || ID_KEY.test(key) ? "font-mono" : "font-sans",
|
||||
);
|
||||
|
||||
if (!isLongValue(value)) {
|
||||
return (
|
||||
<li className={ROW}>
|
||||
<Lead>
|
||||
<span className="size-2 rounded-full bg-trace-dot" />
|
||||
</Lead>
|
||||
<span className={cn(KEY, "text-trace-key")}>{key}</span>
|
||||
<span className={cn(valueClass, "truncate")} title={value}>
|
||||
<span className={cn(KEY, "text-muted-foreground")}>{key}</span>
|
||||
<span className={cn(valueClass, "whitespace-pre-wrap")} title={value}>
|
||||
{value}
|
||||
</span>
|
||||
</li>
|
||||
|
|
@ -56,9 +53,9 @@ function KeyValueRow({ entry, mono }: { entry: KeyValue; mono: boolean }) {
|
|||
const head = (
|
||||
<>
|
||||
<Lead>
|
||||
<FoldChevron open={open} className="size-3 text-trace-text" />
|
||||
<FoldChevron open={open} className="size-3 text-foreground" />
|
||||
</Lead>
|
||||
<span className={cn(KEY, "text-trace-duration")}>{key}</span>
|
||||
<span className={cn(KEY, "text-muted-foreground")}>{key}</span>
|
||||
</>
|
||||
);
|
||||
const toggle = {
|
||||
|
|
@ -87,7 +84,6 @@ function KeyValueRow({ entry, mono }: { entry: KeyValue; mono: boolean }) {
|
|||
);
|
||||
}
|
||||
|
||||
/** Dot-bulleted key / value list; long values show a chevron and expand in place on click. */
|
||||
export function KeyValueRows({
|
||||
entries,
|
||||
mono = false,
|
||||
|
|
@ -98,7 +94,7 @@ export function KeyValueRows({
|
|||
className?: string;
|
||||
}) {
|
||||
return (
|
||||
<ul className={cn("flex flex-col gap-3", className)}>
|
||||
<ul className={cn("flex flex-col", className)}>
|
||||
{entries.map((entry) => (
|
||||
<KeyValueRow key={entry[0]} entry={entry} mono={mono} />
|
||||
))}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ describe("MessageCard", () => {
|
|||
expect(screen.getByText("Why was I billed twice?")).toBeVisible();
|
||||
});
|
||||
|
||||
it("lists tool call args as dot rows and expands only the long value in place", async () => {
|
||||
it("lists tool call arguments once and expands only the long value in place", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<MessageCard
|
||||
|
|
|
|||
|
|
@ -1,20 +1,18 @@
|
|||
"use client";
|
||||
|
||||
import { Bot, ScrollText, UserRound, Wrench } from "lucide-react";
|
||||
import { Wrench } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import ReactMarkdown, { type Components } from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
|
||||
import { Logo } from "@/components/molecules/logo/Logo";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import { FoldChevron } from "./Collapse";
|
||||
import { CopyButton } from "./CopyButton";
|
||||
import { displayValue, type KeyValue, KeyValueRows, objectEntries } from "./KeyValueRows";
|
||||
import { useSpanProvider } from "./spanProvider";
|
||||
import type { TraceMessage, TraceToolCall } from "./traceTypes";
|
||||
|
||||
const ROLE_LABEL: Record<string, string> = { user: "User", system: "System", assistant: "AI", tool: "Tool" };
|
||||
const ROLE_LABEL: Record<string, string> = { user: "User", system: "System", assistant: "Assistant", tool: "Tool" };
|
||||
|
||||
const BLOCK_GAP = "mb-[1lh] last:mb-0";
|
||||
|
||||
|
|
@ -27,25 +25,25 @@ const markdownComponents: Components = {
|
|||
h2: ({ node, ...props }) => <h2 className={cn(BLOCK_GAP, "font-semibold")} {...props} />,
|
||||
h3: ({ node, ...props }) => <h3 className={cn(BLOCK_GAP, "font-semibold")} {...props} />,
|
||||
blockquote: ({ node, ...props }) => (
|
||||
<blockquote className={cn(BLOCK_GAP, "border-l-2 border-trace-border pl-3 text-trace-duration")} {...props} />
|
||||
<blockquote className={cn(BLOCK_GAP, "border-l-2 border-border pl-3 text-muted-foreground")} {...props} />
|
||||
),
|
||||
pre: ({ node, ...props }) => (
|
||||
<pre
|
||||
className={cn(
|
||||
BLOCK_GAP,
|
||||
"max-w-full overflow-x-auto rounded-[4px] bg-trace-chip p-2.5 font-mono text-[13px] leading-[1.5] tracking-normal",
|
||||
"max-w-full overflow-x-auto rounded-[4px] bg-muted/40 p-2.5 font-mono text-[13px] leading-[1.5] tracking-normal",
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
),
|
||||
code: ({ node, className, ...props }) => <code className={cn("font-mono", className)} {...props} />,
|
||||
a: ({ node, ...props }) => <a className="text-trace-brand underline" target="_blank" rel="noreferrer" {...props} />,
|
||||
a: ({ node, ...props }) => <a className="text-primary underline" target="_blank" rel="noreferrer" {...props} />,
|
||||
img: ({ alt }) => <span>{alt || "Image omitted"}</span>,
|
||||
};
|
||||
|
||||
export function Markdown({ text }: { text: string }) {
|
||||
return (
|
||||
<div className="text-[14px] leading-[1.65] tracking-[-0.42px] break-words text-trace-text">
|
||||
<div className="text-sm leading-6 break-words text-foreground">
|
||||
<ReactMarkdown remarkPlugins={[remarkGfm]} components={markdownComponents}>
|
||||
{text}
|
||||
</ReactMarkdown>
|
||||
|
|
@ -54,60 +52,16 @@ export function Markdown({ text }: { text: string }) {
|
|||
}
|
||||
|
||||
export function Card({ children, className }: { children: React.ReactNode; className?: string }) {
|
||||
return (
|
||||
<section
|
||||
className={cn(
|
||||
"rounded-[4px] border-[0.67px] border-trace-card-border bg-trace-surface shadow-[0_1px_1px_0_rgba(16,24,40,0.02)]",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
{children}
|
||||
</section>
|
||||
);
|
||||
return <section className={cn("rounded-md border border-border bg-background", className)}>{children}</section>;
|
||||
}
|
||||
|
||||
const TILE = "grid size-4 shrink-0 place-items-center rounded-[2px] p-0.5";
|
||||
|
||||
function RoleGlyph({ role, model, failed }: { role: string; model: string | null; failed: boolean }) {
|
||||
const provider = useSpanProvider(role === "assistant" ? model : null);
|
||||
if (role === "user") return <UserRound className="size-3 fill-current text-trace-human-glyph" strokeWidth={2.5} />;
|
||||
if (role === "tool") return <Wrench className={cn("size-3", failed ? "text-white" : "text-trace-glyph")} />;
|
||||
if (role === "system") return <ScrollText className="size-3 text-trace-glyph" />;
|
||||
if (provider) return <Logo provider={provider} className="size-3" />;
|
||||
return <Bot className="size-3 text-trace-glyph" />;
|
||||
function RoleTile({ failed = false }: { role: string; failed?: boolean }) {
|
||||
return <Wrench className={cn("size-3.5 shrink-0", failed ? "text-destructive" : "text-muted-foreground")} />;
|
||||
}
|
||||
|
||||
const tileTone = (role: string, failed: boolean, hasLogo = false): string => {
|
||||
if (failed) return "bg-destructive";
|
||||
if (role === "user") return "bg-trace-human";
|
||||
if (role === "assistant") return hasLogo ? "bg-white ring-1 ring-trace-line ring-inset" : "bg-trace-llm";
|
||||
if (role === "tool") return "bg-trace-tool";
|
||||
return "bg-trace-duration";
|
||||
};
|
||||
|
||||
function RoleTile({ role, model = null, failed = false }: { role: string; model?: string | null; failed?: boolean }) {
|
||||
const provider = useSpanProvider(role === "assistant" ? model : null);
|
||||
return (
|
||||
<span className={cn(TILE, tileTone(role, failed, provider !== null))}>
|
||||
<RoleGlyph role={role} model={model} failed={failed} />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function FoldTile({
|
||||
role,
|
||||
model,
|
||||
label,
|
||||
open,
|
||||
onToggle,
|
||||
}: {
|
||||
role: string;
|
||||
model: string | null;
|
||||
label: string;
|
||||
open: boolean;
|
||||
onToggle: () => void;
|
||||
}) {
|
||||
const provider = useSpanProvider(role === "assistant" ? model : null);
|
||||
function FoldTile({ label, open, onToggle }: { label: string; open: boolean; onToggle: () => void }) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
|
|
@ -116,53 +70,38 @@ function FoldTile({
|
|||
onClick={onToggle}
|
||||
className={cn(
|
||||
TILE,
|
||||
"group/tile cursor-pointer transition-colors duration-150 group-hover/header:bg-trace-tab-active focus-visible:bg-trace-tab-active motion-reduce:transition-none",
|
||||
tileTone(role, false, provider !== null),
|
||||
"group/tile cursor-pointer transition-colors duration-150 group-hover/header:bg-muted focus-visible:bg-muted motion-reduce:transition-none",
|
||||
"text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
<span className="contents group-hover/header:hidden group-focus-visible/tile:hidden">
|
||||
<RoleGlyph role={role} model={model} failed={false} />
|
||||
</span>
|
||||
<FoldChevron
|
||||
open={open}
|
||||
className="hidden size-3 text-trace-duration group-hover/header:block group-focus-visible/tile:block"
|
||||
/>
|
||||
<FoldChevron open={open} className="size-3 text-muted-foreground" />
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
const HEADER =
|
||||
"group/header flex min-w-0 items-center gap-2.5 border-[0.67px] border-trace-card-border bg-trace-surface py-1.5 pr-2 pl-3 transition-colors duration-150 hover:bg-trace-chip motion-reduce:transition-none dark:bg-trace-chip";
|
||||
const LABEL = "shrink-0 text-[13px] leading-[1.2] font-semibold tracking-[-0.26px] text-trace-text";
|
||||
"group/header flex min-w-0 items-center gap-2 border border-border bg-background py-2 pr-2 pl-3 transition-colors hover:bg-muted/30";
|
||||
const LABEL = "min-w-0 truncate text-xs leading-5 font-medium text-foreground";
|
||||
const CARD_COPY =
|
||||
"ml-auto size-5 shrink-0 rounded-[3px] text-trace-text-2 opacity-60 transition-opacity duration-150 group-hover/header:opacity-100 focus-visible:opacity-100 motion-reduce:transition-none [&_svg]:size-4";
|
||||
const INLINE_COPY = "size-4 shrink-0 rounded-[3px] text-trace-duration";
|
||||
"ml-auto size-5 shrink-0 rounded-[3px] text-muted-foreground opacity-60 transition-opacity duration-150 group-hover/header:opacity-100 focus-visible:opacity-100 motion-reduce:transition-none [&_svg]:size-4";
|
||||
const INLINE_COPY = "size-4 shrink-0 rounded-[3px] text-muted-foreground";
|
||||
|
||||
function ToolCallBlock({ call }: { call: TraceToolCall }) {
|
||||
const raw: KeyValue[] = [
|
||||
["arguments", displayValue(call.args)],
|
||||
["name", call.name],
|
||||
];
|
||||
export function ToolCallBlock({ call }: { call: TraceToolCall }) {
|
||||
const argEntries = objectEntries(call.args);
|
||||
const entries: KeyValue[] = argEntries?.length ? argEntries : [["arguments", displayValue(call.args)]];
|
||||
return (
|
||||
<>
|
||||
<KeyValueRows entries={raw} />
|
||||
<div className="-mx-3 flex h-7 items-center gap-2.5 py-1.5 pr-3 pl-[30px] transition-colors duration-150 hover:bg-trace-row-hover motion-reduce:transition-none">
|
||||
<div className="space-y-1">
|
||||
<div className="flex items-center gap-2 py-1">
|
||||
<RoleTile role="tool" />
|
||||
<span className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="truncate text-[13px] leading-[1.2] font-medium tracking-[-0.26px] text-trace-text">
|
||||
{call.name}
|
||||
</span>
|
||||
<CopyButton value={call.name} label={`Copy ${call.name} name`} iconOnly className={INLINE_COPY} />
|
||||
</span>
|
||||
<span className="min-w-0 truncate text-sm font-medium">{call.name}</span>
|
||||
<CopyButton value={call.name} label={`Copy ${call.name} name`} iconOnly className={INLINE_COPY} />
|
||||
</div>
|
||||
{argEntries && argEntries.length > 0 && <KeyValueRows entries={argEntries} className="pl-10" />}
|
||||
</>
|
||||
<KeyValueRows entries={entries} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** One chat message as a card with a sticky header; hovering the role tile turns it into a fold chevron. */
|
||||
export function MessageCard({ message, model }: { message: TraceMessage; model: string | null }) {
|
||||
export function MessageCard({ message }: { message: TraceMessage; model: string | null }) {
|
||||
const [open, setOpen] = useState(true);
|
||||
const label = message.role === "tool" ? message.name ?? "Tool" : ROLE_LABEL[message.role] ?? message.role;
|
||||
if (message.role === "tool") return <ToolResultCard name={label} result={message.content} />;
|
||||
|
|
@ -171,17 +110,17 @@ export function MessageCard({ message, model }: { message: TraceMessage; model:
|
|||
const hasBody = Boolean(message.content) || calls.length > 0;
|
||||
const expanded = open && hasBody;
|
||||
return (
|
||||
<article className="rounded-[4px] shadow-[0_1px_1px_0_rgba(16,24,40,0.02)]">
|
||||
<article className="rounded-md">
|
||||
<div className={cn(HEADER, "sticky top-10 z-sticky", expanded ? "rounded-t-[4px] border-b-0" : "rounded-[4px]")}>
|
||||
<FoldTile role={message.role} model={model} label={label} open={open} onToggle={() => setOpen((v) => !v)} />
|
||||
<FoldTile label={label} open={open} onToggle={() => setOpen((v) => !v)} />
|
||||
<span className={LABEL}>{label}</span>
|
||||
<CopyButton value={copyValue} label={`Copy ${label}`} iconOnly className={CARD_COPY} />
|
||||
</div>
|
||||
{expanded && (
|
||||
<div
|
||||
className={cn(
|
||||
"flex flex-col gap-2 rounded-b-[4px] border-[0.67px] border-t-0 border-trace-card-border bg-trace-surface px-3 pt-1.5 pb-3",
|
||||
message.role === "system" && "[&_*]:text-trace-duration",
|
||||
"flex flex-col gap-3 rounded-b-md border border-t-0 border-border bg-background px-3 pt-2 pb-3",
|
||||
message.role === "system" && "[&_*]:text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{message.content && <Markdown text={message.content} />}
|
||||
|
|
@ -200,14 +139,12 @@ const LONG_RESULT_CHARS = 120;
|
|||
export function ToolResultCard({ name, result, failed = false }: { name: string; result: string; failed?: boolean }) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const expandable = result.length > LONG_RESULT_CHARS || result.includes("\n");
|
||||
const tone = failed ? "text-destructive" : "text-trace-duration";
|
||||
const tone = failed ? "text-destructive" : "text-muted-foreground";
|
||||
return (
|
||||
<article className="flex flex-col">
|
||||
<div
|
||||
className={cn(HEADER, open ? "rounded-t-[4px]" : "rounded-[4px]", "shadow-[0_1px_1px_0_rgba(16,24,40,0.02)]")}
|
||||
>
|
||||
<div className={cn(HEADER, open ? "rounded-t-[4px]" : "rounded-[4px]")}>
|
||||
<RoleTile role="tool" failed={failed} />
|
||||
<span className="flex shrink-0 items-center gap-1.5">
|
||||
<span className="flex min-w-0 max-w-[50%] shrink-0 items-center gap-1.5">
|
||||
<span className={cn(LABEL, failed && "text-destructive")}>{name}</span>
|
||||
<CopyButton value={name} label={`Copy ${name} name`} iconOnly className={INLINE_COPY} />
|
||||
</span>
|
||||
|
|
@ -223,16 +160,14 @@ export function ToolResultCard({ name, result, failed = false }: { name: string;
|
|||
{!open && <span className="min-w-0 truncate text-[13px] leading-[1.2]">{result}</span>}
|
||||
</button>
|
||||
) : (
|
||||
<span className={cn("min-w-0 truncate text-[13px] leading-[1.2] tracking-[-0.26px]", tone)}>
|
||||
{result || "No output"}
|
||||
</span>
|
||||
<span className={cn("min-w-0 break-words text-[13px] leading-5", tone)}>{result || "No output"}</span>
|
||||
)}
|
||||
<CopyButton value={result} label={`Copy ${name} result`} iconOnly className={CARD_COPY} />
|
||||
</div>
|
||||
{open && (
|
||||
<pre
|
||||
className={cn(
|
||||
"max-h-80 overflow-auto rounded-b-[4px] border-[0.67px] border-t-0 border-trace-card-border bg-trace-chip px-3 py-2 font-mono text-[12px] leading-[1.5] break-words whitespace-pre-wrap",
|
||||
"max-h-80 overflow-auto rounded-b-[4px] border-[0.67px] border-t-0 border-border bg-muted/40 px-3 py-2 font-mono text-[12px] leading-[1.5] break-words whitespace-pre-wrap",
|
||||
tone,
|
||||
)}
|
||||
>
|
||||
|
|
@ -244,20 +179,35 @@ export function ToolResultCard({ name, result, failed = false }: { name: string;
|
|||
}
|
||||
|
||||
/** Collapsible "Input" / "Output" section: sticky header, chevron turns, body snaps open and shut. */
|
||||
export function Section({ title, children }: { title: string; children: React.ReactNode }) {
|
||||
const [open, setOpen] = useState(true);
|
||||
export function Section({
|
||||
title,
|
||||
children,
|
||||
defaultOpen = true,
|
||||
count,
|
||||
}: {
|
||||
title: string;
|
||||
children: React.ReactNode;
|
||||
defaultOpen?: boolean;
|
||||
count?: number;
|
||||
}) {
|
||||
const [open, setOpen] = useState(defaultOpen);
|
||||
return (
|
||||
<section aria-label={title}>
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={open}
|
||||
onClick={() => setOpen((prev) => !prev)}
|
||||
className="sticky top-0 z-sticky-pinned mb-2 flex h-10 w-full cursor-pointer items-center gap-2.5 bg-trace-surface py-1 pr-3 pl-2 text-left transition-colors duration-150 hover:bg-trace-chip motion-reduce:transition-none"
|
||||
className="sticky top-0 z-sticky-pinned mb-2 flex h-10 w-full cursor-pointer items-center gap-2.5 bg-background py-1 pr-3 pl-2 text-left transition-colors duration-150 hover:bg-muted/40 motion-reduce:transition-none"
|
||||
>
|
||||
<FoldChevron open={open} className="size-3 shrink-0 text-trace-text" />
|
||||
<span className="text-[13px] leading-[1.2] font-medium tracking-[-0.26px] text-trace-text">{title}</span>
|
||||
<FoldChevron open={open} className="size-3 shrink-0 text-foreground" />
|
||||
<span className="text-[13px] leading-[1.2] font-medium tracking-[-0.26px] text-foreground">{title}</span>{" "}
|
||||
{count !== undefined && (
|
||||
<span className="text-xs font-normal text-muted-foreground">
|
||||
{count} {count === 1 ? "message" : "messages"}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
<div hidden={!open} inert={!open} className="flex flex-col gap-2.5 pr-4 pb-4 pl-7">
|
||||
<div hidden={!open} inert={!open} className="flex flex-col gap-3 px-3 pb-4">
|
||||
{children}
|
||||
</div>
|
||||
</section>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { screen } from "@testing-library/react";
|
||||
import { fireEvent, screen } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { renderWithProviders } from "../../../../tests/test-utils";
|
||||
|
|
@ -53,6 +53,68 @@ describe("clampDrawerWidth", () => {
|
|||
describe("RunDrawer", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
window.localStorage.clear();
|
||||
});
|
||||
|
||||
it("fills the page across trace navigation and restores the resized drawer width", () => {
|
||||
const runs = [run("a"), run("b")];
|
||||
const onSelect = vi.fn();
|
||||
const { rerender } = renderWithProviders(
|
||||
<RunDrawer trace={runs[0]} runs={runs} accessToken="sk" onSelect={onSelect} />,
|
||||
);
|
||||
const drawer = screen.getByRole("complementary", { name: "Trace details" });
|
||||
fireEvent.keyDown(screen.getByRole("separator", { name: "Resize trace panel" }), { key: "ArrowLeft" });
|
||||
const resizedWidth = drawer.style.width;
|
||||
const storedWidth = window.localStorage.getItem("litellm.agentTraces.drawerWidth");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Enter full screen" }));
|
||||
expect(drawer).toHaveStyle({ width: "100%" });
|
||||
expect(screen.queryByRole("separator", { name: "Resize trace panel" })).not.toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole("button", { name: "Next trace (J)" }));
|
||||
expect(onSelect).toHaveBeenCalledWith(runs[1]);
|
||||
rerender(<RunDrawer trace={runs[1]} runs={runs} accessToken="sk" onSelect={onSelect} />);
|
||||
expect(drawer).toHaveStyle({ width: "100%" });
|
||||
expect(screen.getByTestId("run-view")).toHaveTextContent("run b");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Exit full screen" }));
|
||||
expect(drawer).toHaveStyle({ width: resizedWidth });
|
||||
expect(screen.getByRole("separator", { name: "Resize trace panel" })).toBeInTheDocument();
|
||||
expect(window.localStorage.getItem("litellm.agentTraces.drawerWidth")).toBe(storedWidth);
|
||||
});
|
||||
|
||||
it.each([false, true])("reopens at the saved width after closing full screen (reduced motion: %s)", (reduce) => {
|
||||
mockReducedMotion(reduce);
|
||||
const runs = [run("a"), run("b")];
|
||||
const onSelect = vi.fn();
|
||||
const { rerender } = renderWithProviders(
|
||||
<RunDrawer trace={runs[0]} runs={runs} accessToken="sk" onSelect={onSelect} />,
|
||||
);
|
||||
fireEvent.keyDown(screen.getByRole("separator", { name: "Resize trace panel" }), { key: "ArrowLeft" });
|
||||
const savedWidth = screen.getByRole("complementary", { name: "Trace details" }).style.width;
|
||||
fireEvent.click(screen.getByRole("button", { name: "Enter full screen" }));
|
||||
rerender(<RunDrawer trace={null} runs={runs} accessToken="sk" onSelect={onSelect} />);
|
||||
rerender(<RunDrawer trace={runs[1]} runs={runs} accessToken="sk" onSelect={onSelect} />);
|
||||
expect(screen.getByRole("complementary", { name: "Trace details" })).toHaveStyle({ width: savedWidth });
|
||||
expect(screen.getByRole("button", { name: "Enter full screen" })).toBeVisible();
|
||||
expect(screen.getByRole("separator", { name: "Resize trace panel" })).toBeVisible();
|
||||
});
|
||||
|
||||
it.each(["Close (Esc)", "Close trace (Esc)"])("closes a full-screen trace using %s", (name) => {
|
||||
const runs = [run("a")];
|
||||
const onSelect = vi.fn();
|
||||
renderWithProviders(<RunDrawer trace={runs[0]} runs={runs} accessToken="sk" onSelect={onSelect} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Enter full screen" }));
|
||||
fireEvent.click(screen.getByRole("button", { name }));
|
||||
expect(onSelect).toHaveBeenCalledExactlyOnceWith(null);
|
||||
});
|
||||
|
||||
it("keeps Escape available to close a full-screen trace", () => {
|
||||
const runs = [run("a")];
|
||||
const onSelect = vi.fn();
|
||||
renderWithProviders(<RunDrawer trace={runs[0]} runs={runs} accessToken="sk" onSelect={onSelect} />);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Enter full screen" }));
|
||||
fireEvent.keyDown(window, { key: "Escape" });
|
||||
expect(onSelect).toHaveBeenCalledExactlyOnceWith(null);
|
||||
});
|
||||
|
||||
it("unmounts right away on close when the user prefers reduced motion", () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { ChevronDown, ChevronsRight, ChevronUp } from "lucide-react";
|
||||
import { ChevronDown, ChevronsRight, ChevronUp, Maximize2, Minimize2, X } from "lucide-react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
|
@ -97,7 +97,7 @@ function ResizeHandle({ width, onResize }: { width: number; onResize: (width: nu
|
|||
>
|
||||
<span
|
||||
className={cn(
|
||||
"h-full w-[0.67px] bg-trace-line transition-[width,background-color] duration-150 group-hover/handle:w-0.5 group-focus-visible/handle:w-0.5 group-focus-visible/handle:bg-trace-brand motion-reduce:transition-none",
|
||||
"h-full w-[0.67px] bg-border transition-[width,background-color] duration-150 group-hover/handle:w-0.5 group-focus-visible/handle:w-0.5 group-focus-visible/handle:bg-trace-brand motion-reduce:transition-none",
|
||||
dragging && "w-0.5 bg-trace-brand",
|
||||
)}
|
||||
/>
|
||||
|
|
@ -123,13 +123,21 @@ function HeaderButton({
|
|||
title={label}
|
||||
disabled={disabled}
|
||||
onClick={onClick}
|
||||
className="grid size-7 place-items-center rounded-[4px] text-trace-key transition-colors duration-150 hover:bg-trace-row-hover hover:text-trace-text disabled:pointer-events-none disabled:opacity-40 motion-reduce:transition-none"
|
||||
className="grid size-7 place-items-center rounded-[4px] text-muted-foreground transition-colors duration-150 hover:bg-muted hover:text-foreground disabled:pointer-events-none disabled:opacity-40 motion-reduce:transition-none"
|
||||
>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function FullScreenButton({ fullScreen, onToggle }: { fullScreen: boolean; onToggle: () => void }) {
|
||||
return (
|
||||
<HeaderButton label={fullScreen ? "Exit full screen" : "Enter full screen"} onClick={onToggle}>
|
||||
{fullScreen ? <Minimize2 className="size-4" /> : <Maximize2 className="size-4" />}
|
||||
</HeaderButton>
|
||||
);
|
||||
}
|
||||
|
||||
interface RunDrawerProps {
|
||||
trace: TraceSummary | null;
|
||||
runs: readonly TraceSummary[];
|
||||
|
|
@ -142,6 +150,8 @@ const runKey = (run: TraceSummary): string => run.trace_ref || run.trace_id;
|
|||
/** Right-side drawer over the runs list: resizable, keeps the list clickable, swaps runs in place. */
|
||||
export function RunDrawer({ trace, runs, accessToken, onSelect }: RunDrawerProps) {
|
||||
const [width, setWidth] = useDrawerWidth();
|
||||
const [fullScreen, setFullScreen] = useState(false);
|
||||
if (trace === null && fullScreen) setFullScreen(false);
|
||||
const [lastShown, setLastShown] = useState<TraceSummary | null>(trace);
|
||||
const [exitedKey, setExitedKey] = useState<string | null>(null);
|
||||
if (trace !== null && trace !== lastShown) setLastShown(trace);
|
||||
|
|
@ -183,21 +193,21 @@ export function RunDrawer({ trace, runs, accessToken, onSelect }: RunDrawerProps
|
|||
<aside
|
||||
aria-label="Trace details"
|
||||
data-testid="run-drawer"
|
||||
style={{ width }}
|
||||
style={{ width: fullScreen ? "100%" : width }}
|
||||
onAnimationEnd={(event) => {
|
||||
if (closing && event.target === event.currentTarget) setExitedKey(runKey(shown));
|
||||
}}
|
||||
className={cn(
|
||||
"fixed inset-y-0 right-0 z-overlay flex origin-right flex-col bg-trace-surface shadow-[0_10px_15px_-3px_rgba(16,24,40,0.1),0_4px_6px_-4px_rgba(16,24,40,0.1)] motion-reduce:animate-none",
|
||||
"fixed inset-y-0 right-0 z-overlay flex origin-right flex-col bg-background shadow-[0_10px_15px_-3px_rgba(16,24,40,0.1),0_4px_6px_-4px_rgba(16,24,40,0.1)] motion-reduce:animate-none",
|
||||
closing ? "animate-trace-drawer-out" : "animate-trace-drawer-in",
|
||||
)}
|
||||
>
|
||||
<ResizeHandle width={width} onResize={setWidth} />
|
||||
<div className="flex h-[37px] shrink-0 items-center gap-1 border-b border-trace-line px-2">
|
||||
{!fullScreen && <ResizeHandle width={width} onResize={setWidth} />}
|
||||
<div className="flex h-[37px] shrink-0 items-center gap-1 border-b border-border px-2">
|
||||
<HeaderButton label="Close (Esc)" onClick={() => onSelect(null)}>
|
||||
<ChevronsRight className="size-4" />
|
||||
</HeaderButton>
|
||||
<span className="mx-1 h-4 w-px bg-trace-line" />
|
||||
<span className="mx-1 h-4 w-px bg-border" />
|
||||
<HeaderButton label="Next trace (J)" disabled={index < 0 || index >= runs.length - 1} onClick={() => step(1)}>
|
||||
<ChevronDown className="size-4" />
|
||||
</HeaderButton>
|
||||
|
|
@ -205,10 +215,16 @@ export function RunDrawer({ trace, runs, accessToken, onSelect }: RunDrawerProps
|
|||
<ChevronUp className="size-4" />
|
||||
</HeaderButton>
|
||||
{index >= 0 && (
|
||||
<span className="ml-1 font-mono text-[11px] text-trace-key tabular-nums">
|
||||
<span className="ml-1 font-mono text-[11px] text-muted-foreground tabular-nums">
|
||||
{index + 1} / {runs.length}
|
||||
</span>
|
||||
)}
|
||||
<div className="ml-auto flex items-center gap-1">
|
||||
<FullScreenButton fullScreen={fullScreen} onToggle={() => setFullScreen((current) => !current)} />
|
||||
<HeaderButton label="Close trace (Esc)" onClick={() => onSelect(null)}>
|
||||
<X className="size-4" />
|
||||
</HeaderButton>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
<RunView
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { Bot, BrainCircuit, Link2, Network, Wrench } from "lucide-react";
|
||||
import { MessageSquareText, Link2, Network, Wrench } from "lucide-react";
|
||||
|
||||
import { Logo } from "@/components/molecules/logo/Logo";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
|
@ -15,7 +15,7 @@ const SIZE = {
|
|||
lg: "size-5 rounded-full",
|
||||
} as const;
|
||||
const GLYPH = { sm: "size-3", card: "size-3", md: "size-3", lg: "size-3" } as const;
|
||||
const LOGO_TILE = "bg-white ring-1 ring-trace-line ring-inset dark:bg-white";
|
||||
const LOGO_TILE = "bg-white dark:bg-white";
|
||||
|
||||
export type IconSize = keyof typeof SIZE;
|
||||
|
||||
|
|
@ -26,30 +26,23 @@ interface SpanIconProps {
|
|||
size?: IconSize;
|
||||
}
|
||||
|
||||
const TYPE_GLYPH: Record<SpanType, typeof Bot> = {
|
||||
agent: Bot,
|
||||
llm: BrainCircuit,
|
||||
const TYPE_GLYPH: Record<SpanType, typeof Network> = {
|
||||
agent: Network,
|
||||
llm: MessageSquareText,
|
||||
tool: Wrench,
|
||||
chain: Link2,
|
||||
framework: Network,
|
||||
};
|
||||
|
||||
const tileTone = (type: SpanType, error: boolean): string => {
|
||||
if (error) return "bg-destructive text-white";
|
||||
if (type === "tool") return "bg-trace-tool text-trace-glyph";
|
||||
if (type === "llm") return "bg-trace-llm text-trace-glyph";
|
||||
if (type === "framework") return "bg-trace-key text-trace-glyph";
|
||||
return "bg-trace-chain text-trace-glyph";
|
||||
};
|
||||
const tileTone = (error: boolean): string => (error ? "text-destructive" : "text-muted-foreground");
|
||||
|
||||
/** Solid square type tile; LLM spans show their provider's logo, knocked out to the tile glyph color. */
|
||||
export function SpanIcon({ type, model = null, error = false, size = "md" }: SpanIconProps) {
|
||||
const provider = useSpanProvider(type === "llm" ? model : null);
|
||||
const Glyph = TYPE_GLYPH[type];
|
||||
const showLogo = provider !== null && !error;
|
||||
return (
|
||||
<span
|
||||
className={cn("grid shrink-0 place-items-center", SIZE[size], showLogo ? LOGO_TILE : tileTone(type, error))}
|
||||
className={cn("grid shrink-0 place-items-center", SIZE[size], showLogo ? LOGO_TILE : tileTone(error))}
|
||||
data-testid="span-icon"
|
||||
data-provider={provider ?? undefined}
|
||||
>
|
||||
|
|
|
|||
|
|
@ -1,19 +1,26 @@
|
|||
"use client";
|
||||
|
||||
import { BarChart3, Clock, Coins, ListTree, MoreHorizontal, PanelRightOpen, Timer } from "lucide-react";
|
||||
import { MoreHorizontal, PanelRightOpen, Search, X } from "lucide-react";
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuTrigger,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuCheckboxItem,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
} from "@/components/ui/dropdown-menu";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import { formatCost } from "./AgentTracesTable";
|
||||
import { FoldChevron } from "./Collapse";
|
||||
import { groupFacts, SpanHoverCard, spanFacts } from "./SpanHoverCard";
|
||||
import { SpanIcon } from "./SpanIcon";
|
||||
import type { GroupRowData, SpanRowData, TreeRow } from "./traceTree";
|
||||
import type { TraceSummary } from "./traceTypes";
|
||||
import { fmtMs, fmtTok, type TreeGuide, treeGuides } from "./traceUtils";
|
||||
import { fmtMs, previewText, type TreeGuide, treeGuides } from "./traceUtils";
|
||||
|
||||
interface SpanTreeProps {
|
||||
rows: TreeRow[];
|
||||
|
|
@ -28,61 +35,25 @@ interface SpanTreeProps {
|
|||
onOpenDetails?: () => void;
|
||||
/** Inside the side drawer J/K switch runs, so spans move with the arrow keys. */
|
||||
embedded?: boolean;
|
||||
query: string;
|
||||
onQueryChange: (query: string) => void;
|
||||
errorsOnly: boolean;
|
||||
onErrorsOnlyChange: (enabled: boolean) => void;
|
||||
filtering: boolean;
|
||||
onClearFilters: () => void;
|
||||
onCollapseAll: () => void;
|
||||
}
|
||||
|
||||
const rowClass = (selected: boolean): string =>
|
||||
cn(
|
||||
"relative flex w-full items-stretch border-l-2 px-3.5 text-left outline-none transition-colors duration-150 ease-[cubic-bezier(0.4,0,0.2,1)] focus-visible:shadow-[inset_0_0_0_2px_var(--trace-brand)] motion-reduce:transition-none",
|
||||
selected ? "border-l-trace-brand bg-trace-row-selected" : "border-l-transparent hover:bg-trace-row-hover",
|
||||
"relative flex w-full items-stretch border-l-2 px-3.5 text-left outline-none transition-colors duration-150 ease-[cubic-bezier(0.4,0,0.2,1)] focus-visible:shadow-[inset_0_0_0_2px_var(--ring)] motion-reduce:transition-none",
|
||||
selected ? "border-l-primary bg-accent" : "border-l-transparent hover:bg-muted/60",
|
||||
);
|
||||
|
||||
const LINE = "border-dashed border-trace-line [border-width:0] [border-left-width:1px]";
|
||||
const ELBOW = "border-dashed border-trace-line [border-width:0] [border-left-width:1px] [border-bottom-width:1px]";
|
||||
const NAME = "truncate text-[13px] leading-[1.2] font-medium";
|
||||
const MONO_NAME = "truncate font-mono text-[12px] leading-[1.2]";
|
||||
const META = "font-mono text-[11px] leading-[14px] text-trace-duration tabular-nums";
|
||||
|
||||
function Stat({ icon: Icon, children }: { icon: typeof Clock; children: React.ReactNode }) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1">
|
||||
<Icon className="size-3" />
|
||||
{children}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const Slash = () => <span className="text-trace-key">/</span>;
|
||||
|
||||
function Summary({ summary }: { summary: TraceSummary }) {
|
||||
const tokens = summary.input_tokens + summary.output_tokens;
|
||||
return (
|
||||
<div className="flex flex-wrap items-center gap-2 px-4 py-2 transition-colors duration-150 hover:bg-trace-turn motion-reduce:transition-none">
|
||||
<span className="flex items-center gap-1.5 text-[12px] leading-[13.8px] font-semibold text-trace-text">
|
||||
<span className="grid size-5 place-items-center rounded-[4px] bg-trace-row-selected text-[var(--trace-summary-glyph,#0d3d77)]">
|
||||
<BarChart3 className="size-3" strokeWidth={1.5} />
|
||||
</span>
|
||||
Summary
|
||||
</span>
|
||||
<span className="flex basis-full flex-wrap items-center gap-x-2 pl-6 font-mono text-[11.5px] leading-[13.8px] text-trace-text-secondary tabular-nums">
|
||||
<Stat icon={ListTree}>{`${summary.span_count.toLocaleString()} spans`}</Stat>
|
||||
<Slash />
|
||||
<Stat icon={Timer}>{fmtMs(summary.duration_ms)}</Stat>
|
||||
{tokens > 0 && (
|
||||
<>
|
||||
<Slash />
|
||||
<Stat icon={Coins}>{fmtTok(tokens)}</Stat>
|
||||
</>
|
||||
)}
|
||||
{summary.spend != null && (
|
||||
<>
|
||||
<Slash />
|
||||
<span>{formatCost(summary.spend)}</span>
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const LINE = "border-dashed border-border [border-width:0] [border-left-width:1px]";
|
||||
const ELBOW = "border-dashed border-border [border-width:0] [border-left-width:1px] [border-bottom-width:1px]";
|
||||
const NAME = "line-clamp-2 break-words text-[13px] leading-[1.2] font-medium";
|
||||
const META = "text-xs leading-4 text-muted-foreground tabular-nums";
|
||||
|
||||
/** One 20px column per ancestor: pass-through rails for open branches, an elbow into this row's tile. */
|
||||
function Gutters({ depth, guide }: { depth: number; guide: TreeGuide }) {
|
||||
|
|
@ -119,6 +90,7 @@ interface RowProps {
|
|||
onSelect: (id: string) => void;
|
||||
onToggleSpan: (id: string) => void;
|
||||
onToggleGroup: (id: string) => void;
|
||||
filtering: boolean;
|
||||
}
|
||||
|
||||
function Caret({ open, onToggle, label }: { open: boolean; onToggle: () => void; label: string }) {
|
||||
|
|
@ -127,7 +99,7 @@ function Caret({ open, onToggle, label }: { open: boolean; onToggle: () => void;
|
|||
role="button"
|
||||
tabIndex={-1}
|
||||
aria-label={label}
|
||||
className="grid size-6 shrink-0 place-items-center self-start rounded-[4px] p-0.5 text-trace-text-secondary transition-colors duration-150 hover:bg-trace-tab-active motion-reduce:transition-none"
|
||||
className="grid size-6 shrink-0 place-items-center self-start rounded-[4px] p-0.5 text-muted-foreground transition-colors duration-150 hover:bg-muted motion-reduce:transition-none"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
onToggle();
|
||||
|
|
@ -162,7 +134,7 @@ function Waterfall({ startMs, durationMs, totalMs, tone }: WaterfallProps) {
|
|||
<span
|
||||
aria-hidden="true"
|
||||
data-testid="span-waterfall"
|
||||
className="relative block h-[3px] w-full overflow-hidden rounded-full bg-trace-chip"
|
||||
className="relative block h-[3px] w-full overflow-hidden rounded-full bg-muted"
|
||||
>
|
||||
<span className={cn("absolute inset-y-0 rounded-full", tone)} style={{ left: `${left}%`, width: `${width}%` }} />
|
||||
</span>
|
||||
|
|
@ -171,9 +143,9 @@ function Waterfall({ startMs, durationMs, totalMs, tone }: WaterfallProps) {
|
|||
|
||||
const barTone = (type: string, failed: boolean): string => {
|
||||
if (failed) return "bg-destructive";
|
||||
if (type === "tool") return "bg-trace-tool";
|
||||
if (type === "llm") return "bg-trace-llm";
|
||||
return "bg-trace-chain";
|
||||
if (type === "tool") return "bg-muted-foreground/40";
|
||||
if (type === "llm") return "bg-muted-foreground/50";
|
||||
return "bg-muted-foreground/60";
|
||||
};
|
||||
|
||||
function SpanRow({
|
||||
|
|
@ -184,16 +156,15 @@ function SpanRow({
|
|||
totalMs,
|
||||
onSelect,
|
||||
onToggleSpan,
|
||||
filtering,
|
||||
}: RowProps & { row: SpanRowData }) {
|
||||
const { span } = row;
|
||||
const failed = span.status === "error";
|
||||
const tokens = span.input_tokens + span.output_tokens;
|
||||
const isLlm = span.type === "llm";
|
||||
const duration = <span className={cn(META, "shrink-0")}>{fmtMs(span.duration_ms)}</span>;
|
||||
const caret = row.hasChildren && (
|
||||
<Caret open={!row.collapsed} label={row.collapsed ? "Expand" : "Collapse"} onToggle={() => onToggleSpan(row.id)} />
|
||||
);
|
||||
const leafDuration = !row.hasChildren && !isLlm && <span className="flex min-h-5 items-center">{duration}</span>;
|
||||
const leafDuration = !row.hasChildren && <span className="flex min-h-5 items-center">{duration}</span>;
|
||||
return (
|
||||
<SpanHoverCard facts={spanFacts(span)} traceStartMs={traceStartMs}>
|
||||
<button
|
||||
|
|
@ -210,19 +181,20 @@ function SpanRow({
|
|||
<TileColumn tile={<SpanIcon type={span.type} model={span.model} error={failed} />} stem={guide.stem} />
|
||||
<RowBody trailing={caret || leafDuration}>
|
||||
<span className={cn("flex min-w-0 items-center gap-2", row.hasChildren ? "min-h-6" : "min-h-5")}>
|
||||
<span
|
||||
className={cn(span.type === "tool" ? MONO_NAME : NAME, failed ? "text-destructive" : "text-trace-text")}
|
||||
>
|
||||
{span.name}
|
||||
</span>
|
||||
{isLlm && span.model && (
|
||||
<span className="h-4 max-w-[150px] shrink-0 truncate rounded-[3px] bg-trace-chip px-1 font-mono text-[11px] leading-4 text-trace-text-secondary">
|
||||
{span.model.split("/").pop()}
|
||||
</span>
|
||||
)}
|
||||
{(row.hasChildren || isLlm) && duration}
|
||||
{isLlm && tokens > 0 && <span className={META}>{fmtTok(tokens)} tok</span>}
|
||||
<span className={cn(NAME, failed ? "text-destructive" : "text-foreground")}>{span.name}</span>
|
||||
{row.hasChildren && duration}
|
||||
</span>
|
||||
{span.type === "agent" && span.parent_span_id && span.input_preview ? (
|
||||
<span className="truncate text-xs text-muted-foreground" title={previewText(span.input_preview)}>
|
||||
{previewText(span.input_preview)}
|
||||
</span>
|
||||
) : (
|
||||
filtering && (
|
||||
<span className="truncate text-xs text-muted-foreground">
|
||||
{previewText(span.input_preview) || span.agent}
|
||||
</span>
|
||||
)
|
||||
)}
|
||||
<Waterfall
|
||||
startMs={span.start_offset_ms}
|
||||
durationMs={span.duration_ms}
|
||||
|
|
@ -275,18 +247,10 @@ function GroupRow({
|
|||
}
|
||||
>
|
||||
<span className="flex min-h-6 min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
row.type === "tool" ? MONO_NAME : NAME,
|
||||
row.isFailureGroup ? "text-destructive" : "text-trace-text",
|
||||
)}
|
||||
>
|
||||
{row.name}
|
||||
</span>
|
||||
<span className={META}>×{row.members.length}</span>
|
||||
<span className={META}>p50 {fmtMs(row.p50Duration)}</span>
|
||||
{row.failedCount > 0 && <span className={cn(META, "text-destructive")}>{row.failedCount} failed</span>}
|
||||
<span className={cn(NAME, row.isFailureGroup ? "text-destructive" : "text-foreground")}>{row.name}</span>
|
||||
<span className={cn(META, "shrink-0")}>×{row.members.length}</span>
|
||||
</span>
|
||||
{row.failedCount > 0 && <span className={cn(META, "text-destructive")}>{row.failedCount} failed</span>}
|
||||
<Waterfall
|
||||
startMs={groupStart}
|
||||
durationMs={groupEnd - groupStart}
|
||||
|
|
@ -299,7 +263,6 @@ function GroupRow({
|
|||
);
|
||||
}
|
||||
|
||||
/** Span tree with connector lines, a run summary on top, a framework toggle and keyboard hints. */
|
||||
export function SpanTree({
|
||||
rows,
|
||||
summary,
|
||||
|
|
@ -312,6 +275,13 @@ export function SpanTree({
|
|||
onLoadMore,
|
||||
onOpenDetails,
|
||||
embedded = false,
|
||||
query,
|
||||
onQueryChange,
|
||||
errorsOnly,
|
||||
onErrorsOnlyChange,
|
||||
filtering,
|
||||
onClearFilters,
|
||||
onCollapseAll,
|
||||
}: SpanTreeProps) {
|
||||
const scrollRef = useRef<HTMLDivElement>(null);
|
||||
const guides = useMemo(() => treeGuides(rows.map((row) => row.depth)), [rows]);
|
||||
|
|
@ -325,26 +295,88 @@ export function SpanTree({
|
|||
|
||||
return (
|
||||
<section className="flex h-full min-h-0 min-w-0 flex-col border-r border-border bg-card" aria-label="Run spans">
|
||||
<div className="mt-3.5 mr-2 mb-2 ml-4 flex h-8 shrink-0 items-center gap-1">
|
||||
<span className="text-[13px] leading-[15.6px] font-semibold tracking-[-0.26px] text-trace-key">Spans</span>
|
||||
<label className="ml-auto flex shrink-0 cursor-pointer items-center gap-2 text-[12px] text-muted-foreground">
|
||||
Hide framework
|
||||
<Switch
|
||||
size="sm"
|
||||
checked={hideFramework}
|
||||
onCheckedChange={(checked) => onToggleHideFramework(checked)}
|
||||
aria-label="Hide framework spans"
|
||||
<div className="shrink-0 space-y-3 border-b px-3 py-3">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="text-sm font-medium">
|
||||
Steps{" "}
|
||||
<span className="ml-1 text-xs font-normal text-muted-foreground">
|
||||
{summary.span_count.toLocaleString()}
|
||||
</span>
|
||||
</span>
|
||||
<div className="flex items-center gap-1">
|
||||
{onOpenDetails && (
|
||||
<Button variant="ghost" size="icon-xs" onClick={onOpenDetails} aria-label="Show details">
|
||||
<PanelRightOpen className="size-4" />
|
||||
</Button>
|
||||
)}
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger render={<Button variant="ghost" size="icon-xs" aria-label="Step display options" />}>
|
||||
<MoreHorizontal className="size-4" />
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-52">
|
||||
<DropdownMenuCheckboxItem checked={hideFramework} onCheckedChange={onToggleHideFramework}>
|
||||
Hide framework spans
|
||||
</DropdownMenuCheckboxItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem onClick={onCollapseAll}>Collapse branches</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
</div>
|
||||
<div className="relative">
|
||||
<Search className="pointer-events-none absolute top-2.5 left-2.5 size-3.5 text-muted-foreground" />
|
||||
<Input
|
||||
aria-label="Search steps"
|
||||
placeholder="Search steps"
|
||||
value={query}
|
||||
onChange={(event) => onQueryChange(event.target.value)}
|
||||
className="h-8 pr-8 pl-8 text-xs shadow-none md:text-xs"
|
||||
/>
|
||||
</label>
|
||||
{onOpenDetails && (
|
||||
<Button variant="ghost" size="sm" onClick={onOpenDetails} className="ml-3 shrink-0 text-xs text-trace-text-2">
|
||||
<PanelRightOpen className="size-4" />
|
||||
Show details
|
||||
{query && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon-xs"
|
||||
aria-label="Clear step search"
|
||||
className="absolute top-1 right-1"
|
||||
onClick={() => onQueryChange("")}
|
||||
>
|
||||
<X className="size-3" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
variant={!errorsOnly ? "secondary" : "ghost"}
|
||||
size="xs"
|
||||
aria-pressed={!errorsOnly}
|
||||
onClick={() => onErrorsOnlyChange(false)}
|
||||
>
|
||||
All steps
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
variant={errorsOnly ? "secondary" : "ghost"}
|
||||
size="xs"
|
||||
aria-pressed={errorsOnly}
|
||||
onClick={() => onErrorsOnlyChange(true)}
|
||||
>
|
||||
Errors <span className="ml-1 text-muted-foreground">{summary.error_count}</span>
|
||||
</Button>
|
||||
{filtering && (
|
||||
<span className="ml-auto text-xs text-muted-foreground" role="status">
|
||||
{rows.length} found
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div ref={scrollRef} className="min-h-0 flex-1 overflow-auto pb-4">
|
||||
<Summary summary={summary} />
|
||||
<div ref={scrollRef} className="min-h-0 flex-1 overflow-auto py-2">
|
||||
{rows.length === 0 && (
|
||||
<div className="px-4 py-8 text-center text-sm text-muted-foreground">
|
||||
<p>No matching steps</p>
|
||||
<Button variant="link" size="sm" onClick={onClearFilters}>
|
||||
Clear filters
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
<div role="tree" aria-label="Spans in time order">
|
||||
{rows.map((row, i) => {
|
||||
if (row.kind === "load-more") {
|
||||
|
|
@ -353,12 +385,12 @@ export function SpanTree({
|
|||
key={row.id}
|
||||
type="button"
|
||||
onClick={() => onLoadMore(row.groupId)}
|
||||
className={cn(rowClass(false), "h-8 text-[13px] tracking-[-0.26px] text-trace-text-secondary")}
|
||||
className={cn(rowClass(false), "h-8 text-[13px] tracking-[-0.26px] text-muted-foreground")}
|
||||
>
|
||||
<Gutters depth={row.depth} guide={guides[i]} />
|
||||
<span className="ml-0.5 flex items-center gap-2">
|
||||
<MoreHorizontal className="relative z-raised size-3.5" /> Load 20 more
|
||||
<span className="text-trace-duration">({row.remaining} remaining)</span>
|
||||
<span className="text-muted-foreground">({row.remaining} remaining)</span>
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
|
|
@ -371,6 +403,7 @@ export function SpanTree({
|
|||
onSelect,
|
||||
onToggleSpan,
|
||||
onToggleGroup,
|
||||
filtering,
|
||||
};
|
||||
return row.kind === "group" ? (
|
||||
<GroupRow key={row.id} row={row} {...shared} />
|
||||
|
|
@ -380,25 +413,25 @@ export function SpanTree({
|
|||
})}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex h-8 shrink-0 items-center gap-3 border-t border-border px-3.5 text-[11px] text-muted-foreground">
|
||||
<div className="flex min-h-8 shrink-0 flex-wrap items-center gap-x-3 gap-y-1 border-t border-border px-3 py-2 text-[10px] text-muted-foreground">
|
||||
{embedded ? (
|
||||
<>
|
||||
<span>
|
||||
<span className="whitespace-nowrap">
|
||||
<Kbd>↑</Kbd>/<Kbd>↓</Kbd> step
|
||||
</span>
|
||||
<span>
|
||||
<span className="whitespace-nowrap">
|
||||
<Kbd>J</Kbd>/<Kbd>K</Kbd> trace
|
||||
</span>
|
||||
</>
|
||||
) : (
|
||||
<span>
|
||||
<span className="whitespace-nowrap">
|
||||
<Kbd>J</Kbd>/<Kbd>K</Kbd> move
|
||||
</span>
|
||||
)}
|
||||
<span>
|
||||
<span className="whitespace-nowrap">
|
||||
<Kbd>←</Kbd>/<Kbd>→</Kbd> fold
|
||||
</span>
|
||||
<span>
|
||||
<span className="whitespace-nowrap">
|
||||
<Kbd>Esc</Kbd> close
|
||||
</span>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,195 @@
|
|||
import { act, screen, waitFor, within } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
|
||||
import { RunView } from "./TraceDrawer";
|
||||
import { TraceConversation } from "./TraceConversation";
|
||||
import type { SpanDetail, Trace } from "./traceTypes";
|
||||
import research from "./__fixtures__/research_trace.json";
|
||||
|
||||
vi.mock("../../networking", () => ({
|
||||
agentTraceCall: vi.fn(),
|
||||
agentTraceSpanCall: vi.fn(),
|
||||
getProxyBaseUrl: () => "http://proxy.test",
|
||||
}));
|
||||
import { agentTraceCall, agentTraceSpanCall } from "../../networking";
|
||||
|
||||
const root = { ...research.spans[0], span_id: "root", parent_span_id: null };
|
||||
const tool = { ...root, span_id: "tool", name: "read_file", parent_span_id: "root", type: "tool", start_offset_ms: 1 };
|
||||
const trace = { ...research, spans: [root, tool] } as Trace;
|
||||
const rootDetail: SpanDetail = {
|
||||
span_id: "root",
|
||||
input: '[{"role":"user","content":"Read the release notes"}]',
|
||||
output: '[{"role":"assistant","content":"The release is ready"}]',
|
||||
attributes: {},
|
||||
};
|
||||
const toolDetail: SpanDetail = {
|
||||
span_id: "tool",
|
||||
input: '{"path":"CHANGELOG.md"}',
|
||||
output: "All checks passed",
|
||||
attributes: {},
|
||||
};
|
||||
|
||||
describe("TraceConversation", () => {
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.mocked(agentTraceCall).mockReset().mockResolvedValue(trace);
|
||||
vi.mocked(agentTraceSpanCall)
|
||||
.mockReset()
|
||||
.mockImplementation(async (_token, _trace, id) => (id === "root" ? rootDetail : { ...toolDetail, span_id: id }));
|
||||
});
|
||||
|
||||
it("switches to a readable transcript and opens the exact tool step from it", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<RunView traceId={trace.summary.trace_id} accessToken="test" onBack={vi.fn()} embedded />);
|
||||
const conversationTab = await screen.findByRole("tab", { name: "Conversation", selected: false });
|
||||
await user.click(conversationTab);
|
||||
expect(conversationTab).toHaveAttribute("aria-selected", "true");
|
||||
const conversation = await screen.findByRole("region", { name: "Trace conversation" });
|
||||
expect(await within(conversation).findByText("Read the release notes")).toBeVisible();
|
||||
await user.click(await within(conversation).findByRole("button", { name: "Expand read_file tool call" }));
|
||||
expect(await within(conversation).findByText("CHANGELOG.md")).toBeVisible();
|
||||
expect(await within(conversation).findByText("All checks passed")).toBeVisible();
|
||||
expect(await within(conversation).findByText("The release is ready")).toBeVisible();
|
||||
const toolStep = within(conversation).getByRole("region", { name: "Conversation step read_file" });
|
||||
await user.click(within(toolStep).getByRole("button", { name: "Inspect step read_file" }));
|
||||
expect(screen.getByRole("tab", { name: "Steps", selected: true })).toBeVisible();
|
||||
expect(screen.getByRole("treeitem", { selected: true })).toHaveAttribute("data-row-id", "tool");
|
||||
expect(screen.getByRole("heading", { name: "read_file" })).toBeVisible();
|
||||
});
|
||||
|
||||
it("loads only twenty full steps at a time and fetches the remainder on demand", async () => {
|
||||
const user = userEvent.setup();
|
||||
const spans = [
|
||||
root,
|
||||
...Array.from({ length: 30 }, (_, index) => ({ ...tool, span_id: `tool-${index}`, start_offset_ms: index + 1 })),
|
||||
];
|
||||
const long = { ...trace, spans } as Trace;
|
||||
renderWithProviders(<TraceConversation trace={long} accessToken="test" onOpenStep={vi.fn()} />);
|
||||
const more = await screen.findByRole("button", { name: "Load next 11 steps" });
|
||||
await waitFor(() => expect(more).toBeEnabled());
|
||||
expect(agentTraceSpanCall).toHaveBeenCalledTimes(20);
|
||||
expect(screen.queryByText("The release is ready")).not.toBeInTheDocument();
|
||||
await user.click(more);
|
||||
expect(await screen.findByText("The release is ready")).toBeVisible();
|
||||
expect(agentTraceSpanCall).toHaveBeenCalledTimes(31);
|
||||
expect(screen.queryByRole("button", { name: /Load next/ })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each([rootDetail.output, ""])(
|
||||
"keeps the root failure visible while loading, then places it in order (output: %s)",
|
||||
async (output) => {
|
||||
const user = userEvent.setup();
|
||||
const rootFetch = Promise.withResolvers<SpanDetail>();
|
||||
const failedRoot = { ...root, status: "error", error: "Agent exceeded its execution limit" };
|
||||
const spans = [
|
||||
failedRoot,
|
||||
...Array.from({ length: 24 }, (_, index) => ({
|
||||
...tool,
|
||||
span_id: `tool-${index}`,
|
||||
start_offset_ms: index + 1,
|
||||
})),
|
||||
];
|
||||
vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, id) =>
|
||||
id === "root" ? rootFetch.promise : { ...toolDetail, span_id: id },
|
||||
);
|
||||
renderWithProviders(
|
||||
<TraceConversation trace={{ ...trace, spans } as Trace} accessToken="test" onOpenStep={vi.fn()} />,
|
||||
);
|
||||
expect(screen.getAllByText("Agent exceeded its execution limit")).toHaveLength(1);
|
||||
await act(async () => rootFetch.resolve({ ...rootDetail, output }));
|
||||
const more = screen.getByRole("button", { name: "Load next 5 steps" });
|
||||
await waitFor(() => expect(more).toBeEnabled());
|
||||
expect(screen.getAllByText("Agent exceeded its execution limit")).toHaveLength(1);
|
||||
expect(screen.queryByText("The release is ready")).not.toBeInTheDocument();
|
||||
await user.click(more);
|
||||
expect(await screen.findByText("End of conversation")).toBeVisible();
|
||||
expect(screen.getAllByText("Agent exceeded its execution limit")).toHaveLength(1);
|
||||
const rootSteps = screen.getAllByRole("region", { name: `Conversation step ${root.name}` });
|
||||
expect(within(rootSteps.at(-1)!).getByText("Agent exceeded its execution limit")).toBeVisible();
|
||||
},
|
||||
);
|
||||
|
||||
it("shows a missing step explicitly and lets the user retry it", async () => {
|
||||
const user = userEvent.setup();
|
||||
vi.mocked(agentTraceSpanCall).mockRejectedValueOnce(new Error("temporarily unavailable"));
|
||||
renderWithProviders(<TraceConversation trace={trace} accessToken="test" onOpenStep={vi.fn()} />);
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Retry this step to continue the conversation");
|
||||
await user.click(screen.getByRole("button", { name: "Retry step" }));
|
||||
expect(await screen.findByText("Read the release notes")).toBeVisible();
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("holds later turns and the final answer behind a failed step until retry succeeds", async () => {
|
||||
const user = userEvent.setup();
|
||||
const first = { ...tool, span_id: "first", name: "First response", type: "llm", start_offset_ms: 1 };
|
||||
const failedTool = { ...tool, start_offset_ms: 2 };
|
||||
const last = { ...first, span_id: "last", name: "Final response", start_offset_ms: 3 };
|
||||
const traced = { ...trace, spans: [root, first, failedTool, last] } as Trace;
|
||||
const question = { role: "user", content: "Read the release notes" };
|
||||
const checking = {
|
||||
role: "assistant",
|
||||
content: "Checking the release",
|
||||
tool_calls: [{ name: "read_file", args: { path: "CHANGELOG.md" } }],
|
||||
};
|
||||
const firstDetail = { ...rootDetail, span_id: "first", output: JSON.stringify([checking]) };
|
||||
const lastDetail = {
|
||||
...rootDetail,
|
||||
span_id: "last",
|
||||
input: JSON.stringify([question, checking, { role: "tool", content: "All checks passed" }]),
|
||||
};
|
||||
const toolFetch = vi.fn().mockRejectedValueOnce(new Error("unavailable")).mockResolvedValue(toolDetail);
|
||||
vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, id) => {
|
||||
if (id === "tool") return toolFetch();
|
||||
if (id === "first") return firstDetail;
|
||||
if (id === "last") return lastDetail;
|
||||
return rootDetail;
|
||||
});
|
||||
renderWithProviders(<TraceConversation trace={traced} accessToken="test" onOpenStep={vi.fn()} />);
|
||||
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("read_file");
|
||||
expect(await screen.findByText("Checking the release")).toBeVisible();
|
||||
expect(screen.queryByText("The release is ready")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("End of conversation")).not.toBeInTheDocument();
|
||||
expect(screen.getByText("2 of 4 steps loaded")).toBeVisible();
|
||||
|
||||
await user.click(screen.getByRole("button", { name: "Retry step" }));
|
||||
expect(await screen.findByText("The release is ready")).toBeVisible();
|
||||
expect(screen.getAllByText("Checking the release")).toHaveLength(1);
|
||||
expect(screen.getAllByText("Read the release notes")).toHaveLength(1);
|
||||
expect(screen.getAllByRole("button", { name: "Expand read_file tool call" })).toHaveLength(1);
|
||||
expect(screen.getByText("End of conversation")).toBeVisible();
|
||||
});
|
||||
|
||||
it.each(["Partial investigation", ""])(
|
||||
"shows a failed child agent's error once after its work, with output %j",
|
||||
async (output) => {
|
||||
const agent = {
|
||||
...root,
|
||||
span_id: "child",
|
||||
parent_span_id: "root",
|
||||
name: "Investigate release",
|
||||
type: "agent",
|
||||
start_offset_ms: 1,
|
||||
duration_ms: 10,
|
||||
status: "error",
|
||||
error: "Investigation timed out",
|
||||
};
|
||||
const childTool = { ...tool, parent_span_id: "child", start_offset_ms: 2, duration_ms: 2 };
|
||||
const traced = { ...trace, spans: [root, agent, childTool] } as Trace;
|
||||
vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, id) => {
|
||||
if (id === "child") return { ...rootDetail, span_id: id, input: "Investigate failed checks", output };
|
||||
return id === "root" ? rootDetail : toolDetail;
|
||||
});
|
||||
renderWithProviders(<TraceConversation trace={traced} accessToken="test" onOpenStep={vi.fn()} />);
|
||||
|
||||
expect(await screen.findByText("End of conversation")).toBeVisible();
|
||||
expect(screen.getAllByText("Investigation timed out")).toHaveLength(1);
|
||||
const entries = screen.getAllByRole("region", { name: "Conversation step Investigate release" });
|
||||
expect(within(entries[0]).getByText("Investigate failed checks")).toBeVisible();
|
||||
expect(within(entries[0]).queryByText("Investigation timed out")).not.toBeInTheDocument();
|
||||
expect(within(entries[1]).getByText("Investigation timed out")).toBeVisible();
|
||||
if (output) expect(within(entries[1]).getByText(output)).toBeVisible();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
"use client";
|
||||
|
||||
import { useQueries } from "@tanstack/react-query";
|
||||
import { useState } from "react";
|
||||
import { ChevronRight, Wrench } from "lucide-react";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
import { CopyButton } from "./CopyButton";
|
||||
import { useLensDemo } from "@/components/lens/LensDemoContext";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { agentTraceSpanCall } from "../../networking";
|
||||
import { buildConversation, conversationSteps, CONVERSATION_PAGE_SIZE, type ConversationItem } from "./conversation";
|
||||
import { ErrorBlock } from "./DetailContent";
|
||||
import { Markdown, ToolCallBlock } from "./MessageCard";
|
||||
import type { SpanDetail, Trace, TraceMessage } from "./traceTypes";
|
||||
import { fmtMs } from "./traceUtils";
|
||||
|
||||
export function TraceConversation({
|
||||
trace,
|
||||
accessToken,
|
||||
onOpenStep,
|
||||
}: {
|
||||
trace: Trace;
|
||||
accessToken: string;
|
||||
onOpenStep: (id: string) => void;
|
||||
}) {
|
||||
const demo = useLensDemo();
|
||||
const [limit, setLimit] = useState(CONVERSATION_PAGE_SIZE);
|
||||
const steps = conversationSteps(trace.spans);
|
||||
const visible = steps.slice(0, limit);
|
||||
const { trace_id: traceId, trace_ref: traceRef } = trace.summary;
|
||||
const queries = useQueries({
|
||||
queries: visible.map((span) => ({
|
||||
queryKey: ["agentTraceSpan", traceId, traceRef, span.span_id, accessToken],
|
||||
queryFn: (): Promise<SpanDetail> =>
|
||||
demo
|
||||
? demo.client.get<SpanDetail>(
|
||||
`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(span.span_id)}`,
|
||||
)
|
||||
: agentTraceSpanCall(accessToken, traceId, span.span_id, traceRef),
|
||||
staleTime: Infinity,
|
||||
retry: false,
|
||||
})),
|
||||
});
|
||||
const loading = queries.some((query) => query.isPending);
|
||||
const failed = queries.some((query) => query.isError);
|
||||
const unresolvedIndex = queries.findIndex((query) => !query.isSuccess);
|
||||
const loadedCount = unresolvedIndex < 0 ? queries.length : unresolvedIndex;
|
||||
const details = new Map(
|
||||
queries.slice(0, loadedCount).map((query, index) => [visible[index].span_id, query.data!] as const),
|
||||
);
|
||||
const complete = loadedCount === steps.length;
|
||||
const items = buildConversation(trace.spans, details, complete);
|
||||
const inlineErrorIds = new Set(items.filter((item) => item.showError).map((item) => item.span.span_id));
|
||||
const rootErrors = trace.spans.filter((span) => {
|
||||
const failedRoot = span.parent_span_id === null && span.status === "error" && span.type !== "tool";
|
||||
return failedRoot && !inlineErrorIds.has(span.span_id);
|
||||
});
|
||||
return (
|
||||
<section aria-label="Trace conversation" className="min-h-0 flex-1 overflow-y-auto">
|
||||
<div className="mx-auto max-w-3xl space-y-6 px-6 py-5">
|
||||
{rootErrors.map((span) => (
|
||||
<ErrorBlock key={span.span_id} span={span} />
|
||||
))}
|
||||
{items.map((item) => (
|
||||
<section
|
||||
key={item.id}
|
||||
className="group/conversation relative space-y-3"
|
||||
aria-label={`Conversation step ${item.span.name}`}
|
||||
>
|
||||
{item.showError && <ErrorBlock span={item.span} />}
|
||||
{item.messages.map((message, index) => (
|
||||
<ConversationMessage key={index} message={message} />
|
||||
))}
|
||||
{item.toolResult !== undefined && <ConversationTool item={item} />}
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
title={`${item.span.agent || item.span.name} · ${fmtMs(item.span.start_offset_ms)}`}
|
||||
aria-label={`Inspect step ${item.span.name}`}
|
||||
className="absolute right-0 -bottom-5 z-raised bg-background text-muted-foreground opacity-0 group-hover/conversation:opacity-100 focus-visible:opacity-100"
|
||||
onClick={() => onOpenStep(item.span.span_id)}
|
||||
>
|
||||
Inspect step
|
||||
</Button>
|
||||
</section>
|
||||
))}
|
||||
{queries.map(
|
||||
(query, index) =>
|
||||
query.isError && (
|
||||
<div key={visible[index].span_id} role="alert" className="rounded-md border p-3 text-sm">
|
||||
Could not load {visible[index].name}. Retry this step to continue the conversation.
|
||||
<Button variant="outline" size="sm" className="mt-2" onClick={() => query.refetch()}>
|
||||
Retry step
|
||||
</Button>
|
||||
</div>
|
||||
),
|
||||
)}
|
||||
{loading && (
|
||||
<p role="status" className="py-4 text-sm text-muted-foreground">
|
||||
Loading conversation…
|
||||
</p>
|
||||
)}
|
||||
{complete && items.length === 0 && (
|
||||
<p className="text-sm text-muted-foreground">No conversation content recorded.</p>
|
||||
)}
|
||||
<div className="flex items-center justify-between gap-3 border-t pt-4 text-xs text-muted-foreground">
|
||||
<span>{complete ? "End of conversation" : `${loadedCount} of ${steps.length} steps loaded`}</span>
|
||||
{visible.length < steps.length && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={loading || failed}
|
||||
onClick={() => setLimit((current) => current + CONVERSATION_PAGE_SIZE)}
|
||||
>
|
||||
Load next {Math.min(CONVERSATION_PAGE_SIZE, steps.length - visible.length)} steps
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function ConversationTool({ item }: { item: ConversationItem }) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const failed = item.span.status === "error";
|
||||
return (
|
||||
<div className="rounded-md border">
|
||||
<button
|
||||
type="button"
|
||||
aria-expanded={open}
|
||||
aria-label={`${open ? "Collapse" : "Expand"} ${item.span.name} tool call`}
|
||||
onClick={() => setOpen((value) => !value)}
|
||||
className="flex w-full items-center gap-2 px-3 py-2.5 text-left text-sm hover:bg-muted/40"
|
||||
>
|
||||
<ChevronRight
|
||||
className={cn("size-3.5 shrink-0 text-muted-foreground transition-transform", open && "rotate-90")}
|
||||
/>
|
||||
<Wrench className="size-3.5 shrink-0 text-muted-foreground" />
|
||||
<span className="min-w-0 truncate font-medium">{item.span.name}</span>
|
||||
<span className={cn("ml-auto shrink-0 text-xs", failed ? "text-destructive" : "text-muted-foreground")}>
|
||||
{failed ? "Failed" : "Completed"}
|
||||
</span>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="space-y-3 border-t px-3 py-3">
|
||||
{item.toolCall && <ToolCallBlock call={item.toolCall} />}
|
||||
{item.span.error && item.span.error !== item.toolResult && (
|
||||
<p className="text-sm text-destructive">{item.span.error}</p>
|
||||
)}
|
||||
<div className="flex items-center justify-between text-xs text-muted-foreground">
|
||||
<span>Result</span>
|
||||
<CopyButton value={item.toolResult ?? ""} label={`Copy ${item.span.name} result`} iconOnly />
|
||||
</div>
|
||||
<pre className="max-h-80 overflow-auto whitespace-pre-wrap break-words text-xs leading-5">
|
||||
{item.toolResult || "No output recorded"}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ConversationMessage({ message }: { message: TraceMessage }) {
|
||||
if (message.role === "system")
|
||||
return (
|
||||
<details className="text-sm text-muted-foreground">
|
||||
<summary className="cursor-pointer">System instructions</summary>
|
||||
<div className="pt-3">
|
||||
<Markdown text={message.content} />
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
if (message.role === "tool")
|
||||
return (
|
||||
<details className="rounded-md border p-3 text-sm">
|
||||
<summary className="cursor-pointer">{message.name || "Tool result"}</summary>
|
||||
<pre className="max-h-80 overflow-auto whitespace-pre-wrap pt-3 text-xs">{message.content}</pre>
|
||||
</details>
|
||||
);
|
||||
return (
|
||||
<div className={message.role === "user" ? "flex justify-end" : "space-y-3"}>
|
||||
{message.content && (
|
||||
<div className={message.role === "user" ? "max-w-[85%] rounded-xl bg-muted px-4 py-3" : "px-1 py-1"}>
|
||||
<Markdown text={message.content} />
|
||||
</div>
|
||||
)}
|
||||
{message.tool_calls?.map((call, index) => (
|
||||
<details key={index} className="rounded-md border p-3 text-sm">
|
||||
<summary className="cursor-pointer">{call.name}</summary>
|
||||
<div className="pt-3">
|
||||
<ToolCallBlock call={call} />
|
||||
</div>
|
||||
</details>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -49,14 +49,17 @@ describe("RunView", () => {
|
|||
vi.mocked(copyToClipboard).mockClear();
|
||||
});
|
||||
|
||||
it("shows a one-line run header: agent name, trace id, duration and steps", async () => {
|
||||
it("shows the run name, copyable ID and run totals", async () => {
|
||||
renderRun(research);
|
||||
|
||||
const header = await screen.findByRole("banner");
|
||||
expect(screen.getByRole("heading", { level: 1 })).toHaveTextContent(traceDisplayName(research.summary));
|
||||
expect(header).toHaveTextContent(research.summary.trace_id);
|
||||
expect(header).toHaveTextContent("duration 40.20s");
|
||||
expect(header).toHaveTextContent(`steps ${research.summary.span_count}`);
|
||||
expect(within(header).getByRole("button", { name: "Copy trace ID" })).toHaveAttribute(
|
||||
"title",
|
||||
research.summary.trace_id,
|
||||
);
|
||||
expect(header).toHaveTextContent("Duration 40.20s");
|
||||
expect(header).toHaveTextContent(`Steps ${research.summary.span_count}`);
|
||||
expect(header).not.toHaveTextContent("failed");
|
||||
});
|
||||
|
||||
|
|
@ -86,7 +89,7 @@ describe("RunView", () => {
|
|||
|
||||
const tree = await screen.findByRole("tree", { name: "Spans in time order" });
|
||||
expect(tree).toHaveTextContent("researcher×12");
|
||||
expect(screen.getByRole("banner")).toHaveTextContent(`failed ${swarm.summary.error_count}`);
|
||||
expect(screen.getByRole("banner")).toHaveTextContent(`Step errors ${swarm.summary.error_count}`);
|
||||
});
|
||||
|
||||
it("opens a failed run on its first failed span", async () => {
|
||||
|
|
@ -163,6 +166,57 @@ describe("RunView", () => {
|
|||
expect(onBack).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("finds a step beyond a folded group's first page and reveals it after search clears", async () => {
|
||||
const user = userEvent.setup();
|
||||
const root = research.spans.find((span) => span.parent_span_id === null)!;
|
||||
const children = Array.from(
|
||||
{ length: 45 },
|
||||
(_, index): Span => ({
|
||||
...root,
|
||||
span_id: `case-${index}`,
|
||||
parent_span_id: root.span_id,
|
||||
type: "tool",
|
||||
name: "check_case",
|
||||
input_preview: `Case ${index}`,
|
||||
start_offset_ms: index + 1,
|
||||
status: index === 44 ? "error" : "ok",
|
||||
}),
|
||||
);
|
||||
renderRun({ ...research, spans: [root, ...children] });
|
||||
const search = await screen.findByRole("textbox", { name: "Search steps" });
|
||||
await user.type(search, "case 43");
|
||||
expect(screen.getAllByRole("treeitem")).toHaveLength(1);
|
||||
await user.click(screen.getByRole("treeitem"));
|
||||
expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", "case-43");
|
||||
await user.click(screen.getByRole("button", { name: "Clear step search" }));
|
||||
expect(screen.getByRole("treeitem", { selected: true })).toHaveAttribute("data-row-id", "case-43");
|
||||
await user.click(screen.getByRole("button", { name: /^Errors/ }));
|
||||
expect(screen.getAllByRole("treeitem")).toHaveLength(1);
|
||||
expect(screen.getByRole("treeitem")).toHaveAttribute("data-row-id", "case-44");
|
||||
await user.type(search, "not present");
|
||||
expect(screen.getByText("No matching steps")).toBeVisible();
|
||||
await user.click(screen.getByRole("button", { name: "Clear filters" }));
|
||||
expect(screen.getAllByRole("treeitem").length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("does not navigate steps while typing or moving the search cursor", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderRun(research);
|
||||
const search = await screen.findByRole("textbox", { name: "Search steps" });
|
||||
const selected = screen.getByTestId("detail-pane").getAttribute("data-row-id");
|
||||
await user.type(search, "jk{ArrowDown}{ArrowUp}");
|
||||
expect(search).toHaveValue("jk");
|
||||
expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", selected);
|
||||
});
|
||||
|
||||
it("distinguishes a completed run with recovered step errors from a failed run", async () => {
|
||||
renderRun({ ...research, summary: { ...research.summary, status: "ok", error_count: 2 } });
|
||||
const header = await screen.findByRole("banner");
|
||||
expect(header).toHaveTextContent("Completed");
|
||||
expect(header).toHaveTextContent("Step errors 2");
|
||||
expect(header).not.toHaveTextContent("Failed");
|
||||
});
|
||||
|
||||
it("copies a curl one-liner for Claude / Codex", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderRun(research);
|
||||
|
|
@ -190,6 +244,16 @@ describe("initialRunSelection", () => {
|
|||
expect(initialRunSelection(trace).selectedId).toBe("tool");
|
||||
});
|
||||
|
||||
it("folds other agent branches while revealing the failed step", () => {
|
||||
const first = child({ span_id: "first", type: "agent" });
|
||||
const second = child({ span_id: "second", type: "agent" });
|
||||
const failure = child({ span_id: "failed", parent_span_id: "second", type: "tool", status: "error" });
|
||||
const { selectedId, state } = initialRunSelection({ ...research, spans: [base, first, second, failure] });
|
||||
expect(selectedId).toBe("failed");
|
||||
expect(state.collapsedSpanIds.has("first")).toBe(true);
|
||||
expect(state.collapsedSpanIds.has("second")).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to the nearest visible ancestor when only a hidden span failed", () => {
|
||||
const agent = child({ span_id: "agent", type: "agent", name: "researcher" });
|
||||
const hiddenFields: Partial<Span> = { span_id: "mw", parent_span_id: "agent", type: "framework", status: "error" };
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { ArrowLeft, Check, Copy } from "lucide-react";
|
|||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
import { copyToClipboard } from "@/utils/dataUtils";
|
||||
|
|
@ -16,6 +17,7 @@ import { IdChip } from "./IdChip";
|
|||
import { formatCost } from "./AgentTracesTable";
|
||||
import { SpanIcon } from "./SpanIcon";
|
||||
import { SpanTree } from "./SpanTree";
|
||||
import { TraceConversation } from "./TraceConversation";
|
||||
import { FrameworkLogo, traceFramework } from "./TraceFramework";
|
||||
import type { SpanTreeState, TreeRow } from "./traceTree";
|
||||
import type { Trace } from "./traceTypes";
|
||||
|
|
@ -23,6 +25,8 @@ import {
|
|||
buildTreeRows,
|
||||
firstErrorSpan,
|
||||
fmtMs,
|
||||
fmtTok,
|
||||
findTraceSteps,
|
||||
GROUP_PAGE_SIZE,
|
||||
isFrameworkSpan,
|
||||
nearestVisibleSpanId,
|
||||
|
|
@ -56,16 +60,22 @@ export function initialRunSelection(
|
|||
const state = revealSpanInState(trace.spans, { ...INITIAL_STATE, hideFramework: false }, selectedId);
|
||||
return { selectedId, state };
|
||||
}
|
||||
const initialState = {
|
||||
...INITIAL_STATE,
|
||||
collapsedSpanIds: new Set(
|
||||
trace.spans.filter((span) => span.type === "agent" && span.parent_span_id !== null).map((span) => span.span_id),
|
||||
),
|
||||
};
|
||||
const failed = firstErrorSpan(trace.spans);
|
||||
if (!failed || failed.parent_span_id === null) {
|
||||
const root = trace.spans.find((s) => s.parent_span_id === null);
|
||||
return { selectedId: root?.span_id ?? "", state: INITIAL_STATE };
|
||||
return { selectedId: root?.span_id ?? "", state: initialState };
|
||||
}
|
||||
const visibleFailure = trace.spans
|
||||
.filter((s) => s.status === "error" && s.parent_span_id !== null && !isFrameworkSpan(s))
|
||||
.sort((a, b) => a.start_offset_ms - b.start_offset_ms)[0];
|
||||
const selectedId = visibleFailure?.span_id ?? nearestVisibleSpanId(trace.spans, failed.span_id, true);
|
||||
return { selectedId, state: revealSpanInState(trace.spans, INITIAL_STATE, selectedId) };
|
||||
return { selectedId, state: revealSpanInState(trace.spans, initialState, selectedId) };
|
||||
}
|
||||
|
||||
const toggle = (set: ReadonlySet<string>, id: string): Set<string> => {
|
||||
|
|
@ -107,12 +117,11 @@ function Stat({ label, value, error = false }: { label: string; value: string; e
|
|||
return (
|
||||
<span
|
||||
className={cn(
|
||||
"inline-flex shrink-0 items-center gap-1 rounded-[5px] border border-border bg-card px-1.5 py-px text-[12px] tabular-nums",
|
||||
error && "border-destructive/40 bg-destructive/10 text-destructive",
|
||||
"inline-flex items-center gap-1.5 text-xs tabular-nums",
|
||||
error ? "text-destructive" : "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
<span className={cn("text-muted-foreground", error && "text-destructive/80")}>{label} </span>
|
||||
{value}
|
||||
<span>{label}</span> <span className={cn("font-medium", !error && "text-foreground")}>{value}</span>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
|
@ -122,7 +131,7 @@ function RunIcon({ summary, failed }: { summary: Trace["summary"]; failed: boole
|
|||
if (!framework) return <SpanIcon type="agent" error={failed} size="lg" />;
|
||||
return (
|
||||
<span
|
||||
className="inline-flex shrink-0 items-center gap-1 rounded-[5px] border border-border bg-card px-1.5 py-px text-[12px] text-foreground"
|
||||
className="inline-flex shrink-0 items-center gap-1.5 text-sm text-foreground"
|
||||
data-testid="run-framework"
|
||||
title={framework.label}
|
||||
>
|
||||
|
|
@ -134,33 +143,39 @@ function RunIcon({ summary, failed }: { summary: Trace["summary"]; failed: boole
|
|||
|
||||
function RunHeader({ trace, onBack, embedded }: { trace: Trace; onBack: () => void; embedded: boolean }) {
|
||||
const { summary } = trace;
|
||||
const failed = summary.error_count > 0;
|
||||
const failed = summary.status === "error";
|
||||
return (
|
||||
<header className="flex min-h-11 shrink-0 flex-wrap items-center gap-x-2 gap-y-1.5 border-b border-border bg-card px-3 py-1.5">
|
||||
{!embedded && (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
className="grid size-7 shrink-0 place-items-center rounded-md text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
aria-label="Back to runs"
|
||||
>
|
||||
<header className="shrink-0 border-b bg-background px-4 py-3">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
{!embedded && (
|
||||
<Button variant="ghost" size="icon-xs" onClick={onBack} aria-label="Back to runs">
|
||||
<ArrowLeft className="size-4" />
|
||||
</button>
|
||||
<span className="mx-1 h-[18px] w-px bg-border" />
|
||||
</>
|
||||
)}
|
||||
<RunIcon summary={summary} failed={failed} />
|
||||
<h1 className="min-w-0 truncate text-[14px] font-medium text-foreground">{traceDisplayName(summary)}</h1>
|
||||
<IdChip value={summary.trace_id} label="Copy trace ID" showValue />
|
||||
<div className="flex min-w-0 flex-wrap items-center gap-1.5">
|
||||
<Stat label="duration" value={fmtMs(summary.duration_ms)} />
|
||||
<Stat label="steps" value={summary.span_count.toLocaleString()} />
|
||||
<Stat label="cost" value={summary.spend == null ? "—" : formatCost(summary.spend)} />
|
||||
{failed && <Stat label="failed" value={summary.error_count.toLocaleString()} error />}
|
||||
</Button>
|
||||
)}
|
||||
<RunIcon summary={summary} failed={failed} />
|
||||
<h1 className="min-w-0 truncate text-base font-semibold">{traceDisplayName(summary)}</h1>
|
||||
<IdChip value={summary.trace_id} label="Copy trace ID" />
|
||||
<TabsList aria-label="Trace view" className="ml-auto shrink-0 group-data-horizontal/tabs:h-8">
|
||||
<TabsTrigger value="steps" className="text-xs">
|
||||
Steps
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="conversation" className="text-xs">
|
||||
Conversation
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<div className="shrink-0">
|
||||
<CopyForAgent traceId={summary.trace_id} traceRef={summary.trace_ref} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="ml-auto">
|
||||
<CopyForAgent traceId={summary.trace_id} traceRef={summary.trace_ref} />
|
||||
<div className="mt-3 flex flex-wrap items-center gap-x-5 gap-y-2">
|
||||
<span className={cn("text-xs font-medium", failed ? "text-destructive" : "text-muted-foreground")}>
|
||||
{failed ? "Failed" : "Completed"}
|
||||
</span>
|
||||
<Stat label="Duration" value={fmtMs(summary.duration_ms)} />
|
||||
<Stat label="Steps" value={summary.span_count.toLocaleString()} />
|
||||
<Stat label="Tokens" value={fmtTok(summary.input_tokens + summary.output_tokens)} />
|
||||
<Stat label="Cost" value={summary.spend == null ? "Not reported" : formatCost(summary.spend)} />
|
||||
{summary.error_count > 0 && <Stat label="Step errors" value={summary.error_count.toLocaleString()} error />}
|
||||
</div>
|
||||
</header>
|
||||
);
|
||||
|
|
@ -170,27 +185,60 @@ function RunHeader({ trace, onBack, embedded }: { trace: Trace; onBack: () => vo
|
|||
const SPAN_KEYS = { down: ["j", "J", "ArrowDown"], up: ["k", "K", "ArrowUp"] } as const;
|
||||
const EMBEDDED_SPAN_KEYS = { down: ["ArrowDown"], up: ["ArrowUp"] } as const;
|
||||
|
||||
function ignoreStepKey(event: KeyboardEvent): boolean {
|
||||
const control = (event.target as HTMLElement | null)?.closest(
|
||||
"input, textarea, select, [contenteditable='true'], [role='combobox'], [role='tablist'], [role='menu'], [role='separator']",
|
||||
);
|
||||
return event.defaultPrevented || ignoresLetterShortcut(event) || Boolean(control);
|
||||
}
|
||||
|
||||
type TraceView = "steps" | "conversation";
|
||||
|
||||
interface RunBodyProps {
|
||||
trace: Trace;
|
||||
accessToken: string;
|
||||
initialSpanId?: string;
|
||||
embedded: boolean;
|
||||
view: TraceView;
|
||||
onViewChange: (view: TraceView) => void;
|
||||
}
|
||||
|
||||
function RunBody({ trace, accessToken, initialSpanId, embedded }: RunBodyProps) {
|
||||
function RunBody({ trace, accessToken, initialSpanId, embedded, view, onViewChange }: RunBodyProps) {
|
||||
const spanKeys = embedded ? EMBEDDED_SPAN_KEYS : SPAN_KEYS;
|
||||
const initial = useMemo(() => initialRunSelection(trace, initialSpanId), [trace, initialSpanId]);
|
||||
const [state, setState] = useState<SpanTreeState>(initial.state);
|
||||
const [selectedId, setSelectedId] = useState<string>(initial.selectedId);
|
||||
const [detailOpen, setDetailOpen] = useState(true);
|
||||
const [query, setQuery] = useState("");
|
||||
const [errorsOnly, setErrorsOnly] = useState(false);
|
||||
const filtering = Boolean(query.trim()) || errorsOnly;
|
||||
|
||||
const rows = useMemo(() => buildTreeRows(trace.spans, state), [trace, state]);
|
||||
const selectedRow: TreeRow | undefined = rows.find((row) => row.id === selectedId) ?? rows[0];
|
||||
const treeRows = useMemo(() => buildTreeRows(trace.spans, state), [trace, state]);
|
||||
const rows = useMemo<TreeRow[]>(
|
||||
() =>
|
||||
filtering
|
||||
? findTraceSteps(trace.spans, query, errorsOnly, state.hideFramework).map((span) => ({
|
||||
kind: "span",
|
||||
id: span.span_id,
|
||||
span,
|
||||
depth: 0,
|
||||
hasChildren: false,
|
||||
collapsed: false,
|
||||
}))
|
||||
: treeRows,
|
||||
[trace, state.hideFramework, query, errorsOnly, filtering, treeRows],
|
||||
);
|
||||
const selectedRow: TreeRow | undefined =
|
||||
rows.find((row) => row.id === selectedId) ?? treeRows.find((row) => row.id === selectedId) ?? rows[0];
|
||||
|
||||
const select = useCallback((id: string) => {
|
||||
setSelectedId(id);
|
||||
setDetailOpen(true);
|
||||
}, []);
|
||||
const select = useCallback(
|
||||
(id: string) => {
|
||||
setSelectedId(id);
|
||||
setDetailOpen(true);
|
||||
setState((prev) => revealSpanInState(trace.spans, prev, id));
|
||||
},
|
||||
[trace.spans],
|
||||
);
|
||||
const toggleSpan = useCallback(
|
||||
(id: string) => setState((prev) => ({ ...prev, collapsedSpanIds: toggle(prev.collapsedSpanIds, id) })),
|
||||
[],
|
||||
|
|
@ -212,8 +260,13 @@ function RunBody({ trace, accessToken, initialSpanId, embedded }: RunBodyProps)
|
|||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (view !== "steps") return;
|
||||
const setRowExpanded = (row: TreeRow, expand: boolean) => {
|
||||
if (row.kind === "span" && row.hasChildren && row.collapsed === expand) toggleSpan(row.id);
|
||||
if (row.kind === "group" && row.expanded !== expand) toggleGroup(row.id);
|
||||
};
|
||||
const onKeyDown = (event: KeyboardEvent) => {
|
||||
if (ignoresLetterShortcut(event)) return;
|
||||
if (ignoreStepKey(event)) return;
|
||||
const index = rows.findIndex((row) => row.id === selectedRow?.id);
|
||||
const row = rows[index];
|
||||
if (event.key === "Escape" && detailOpen) {
|
||||
|
|
@ -231,23 +284,36 @@ function RunBody({ trace, accessToken, initialSpanId, embedded }: RunBodyProps)
|
|||
const next = rows[Math.max(0, index - 1)];
|
||||
if (next) select(next.id);
|
||||
} else if (event.key === "ArrowLeft" && row) {
|
||||
if (row.kind === "span" && row.hasChildren && !row.collapsed) toggleSpan(row.id);
|
||||
if (row.kind === "group" && row.expanded) toggleGroup(row.id);
|
||||
setRowExpanded(row, false);
|
||||
} else if (event.key === "ArrowRight" && row) {
|
||||
if (row.kind === "span" && row.hasChildren && row.collapsed) toggleSpan(row.id);
|
||||
if (row.kind === "group" && !row.expanded) toggleGroup(row.id);
|
||||
setRowExpanded(row, true);
|
||||
}
|
||||
};
|
||||
window.addEventListener("keydown", onKeyDown, true);
|
||||
return () => window.removeEventListener("keydown", onKeyDown, true);
|
||||
}, [rows, selectedRow, detailOpen, select, toggleSpan, toggleGroup, spanKeys]);
|
||||
}, [rows, selectedRow, detailOpen, select, toggleSpan, toggleGroup, spanKeys, view]);
|
||||
|
||||
if (view === "conversation")
|
||||
return (
|
||||
<TabsContent value="conversation" className="flex min-h-0 flex-1">
|
||||
<TraceConversation
|
||||
trace={trace}
|
||||
accessToken={accessToken}
|
||||
onOpenStep={(id) => {
|
||||
select(id);
|
||||
onViewChange("steps");
|
||||
}}
|
||||
/>
|
||||
</TabsContent>
|
||||
);
|
||||
|
||||
return (
|
||||
<div
|
||||
<TabsContent
|
||||
value="steps"
|
||||
className={cn(
|
||||
"grid min-h-0 flex-1",
|
||||
detailOpen
|
||||
? "grid-cols-1 grid-rows-2 lg:grid-cols-[minmax(340px,400px)_minmax(0,1fr)] lg:grid-rows-1"
|
||||
? "grid-cols-1 grid-rows-2 @[640px]/trace:grid-cols-[clamp(280px,38%,340px)_minmax(0,1fr)] @[640px]/trace:grid-rows-1"
|
||||
: "grid-cols-1",
|
||||
)}
|
||||
>
|
||||
|
|
@ -263,13 +329,31 @@ function RunBody({ trace, accessToken, initialSpanId, embedded }: RunBodyProps)
|
|||
onLoadMore={loadMore}
|
||||
onOpenDetails={detailOpen ? undefined : () => setDetailOpen(true)}
|
||||
embedded={embedded}
|
||||
query={query}
|
||||
onQueryChange={setQuery}
|
||||
errorsOnly={errorsOnly}
|
||||
onErrorsOnlyChange={setErrorsOnly}
|
||||
filtering={filtering}
|
||||
onClearFilters={() => {
|
||||
setQuery("");
|
||||
setErrorsOnly(false);
|
||||
}}
|
||||
onCollapseAll={() =>
|
||||
setState((prev) => ({
|
||||
...prev,
|
||||
collapsedSpanIds: new Set(
|
||||
trace.spans.filter((span) => span.parent_span_id !== null).map((span) => span.span_id),
|
||||
),
|
||||
expandedGroupIds: new Set(),
|
||||
}))
|
||||
}
|
||||
/>
|
||||
{detailOpen && (
|
||||
<div className="min-h-0 min-w-0 animate-slide-left motion-reduce:animate-none">
|
||||
<DetailPane trace={trace} row={selectedRow} accessToken={accessToken} onClose={() => setDetailOpen(false)} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</TabsContent>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -286,6 +370,7 @@ interface RunViewProps {
|
|||
/** One agent run: header with totals and "Copy for agent", span tree on the left, span details on the right. */
|
||||
export function RunView({ traceId, traceRef, initialSpanId, accessToken, onBack, embedded = false }: RunViewProps) {
|
||||
const demo = useLensDemo();
|
||||
const [view, setView] = useState<TraceView>("steps");
|
||||
const traceQuery = useQuery({
|
||||
queryKey: ["agentTrace", traceId, traceRef, accessToken],
|
||||
queryFn: () =>
|
||||
|
|
@ -329,9 +414,11 @@ export function RunView({ traceId, traceRef, initialSpanId, accessToken, onBack,
|
|||
);
|
||||
}
|
||||
return (
|
||||
<div
|
||||
<Tabs
|
||||
value={view}
|
||||
onValueChange={(value) => setView(value as TraceView)}
|
||||
className={cn(
|
||||
"flex flex-1 flex-col overflow-hidden bg-background",
|
||||
"@container/trace flex flex-1 flex-col gap-0 overflow-hidden bg-background",
|
||||
embedded ? "min-h-0 animate-view-fade-in motion-reduce:animate-none" : "min-h-[560px] border-y border-border",
|
||||
)}
|
||||
data-testid="run-view"
|
||||
|
|
@ -343,7 +430,9 @@ export function RunView({ traceId, traceRef, initialSpanId, accessToken, onBack,
|
|||
accessToken={accessToken}
|
||||
initialSpanId={initialSpanId}
|
||||
embedded={embedded}
|
||||
view={view}
|
||||
onViewChange={setView}
|
||||
/>
|
||||
</div>
|
||||
</Tabs>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,254 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { buildConversation, conversationSteps, newConversationMessages } from "./conversation";
|
||||
import type { Span, SpanDetail, TraceMessage } from "./traceTypes";
|
||||
import research from "./__fixtures__/research_trace.json";
|
||||
|
||||
const root = { ...research.spans[0], span_id: "root", parent_span_id: null, type: "agent" } as Span;
|
||||
const user: TraceMessage = { role: "user", content: "Find my order" };
|
||||
const call: TraceMessage = {
|
||||
role: "assistant",
|
||||
content: "Checking the order",
|
||||
tool_calls: [{ name: "lookup", args: { order: 42 } }],
|
||||
};
|
||||
const result: TraceMessage = { role: "tool", content: "Shipped" };
|
||||
const answer: TraceMessage = { role: "assistant", content: "Your order shipped" };
|
||||
const detail = (span_id: string, input: unknown, output: unknown): SpanDetail => ({
|
||||
span_id,
|
||||
input: JSON.stringify(input),
|
||||
output: JSON.stringify(output),
|
||||
attributes: {},
|
||||
});
|
||||
|
||||
describe("trace conversation", () => {
|
||||
it("removes repeated prefixes and trimmed context, but preserves a genuinely repeated question", () => {
|
||||
expect(newConversationMessages([user, call, result], [user, call, result, answer])).toEqual([answer]);
|
||||
expect(newConversationMessages([user, call, result], [call, result, answer])).toEqual([answer]);
|
||||
expect(newConversationMessages([user, answer], [user, answer, user])).toEqual([user]);
|
||||
expect(newConversationMessages([user, call, result], [user])).toEqual([]);
|
||||
});
|
||||
|
||||
it("reads a tool exchange once, without repeating the root's final answer", () => {
|
||||
const first = { ...root, span_id: "first", parent_span_id: "root", type: "llm", start_offset_ms: 1 } as Span;
|
||||
const tool = { ...first, span_id: "tool", name: "lookup", type: "tool", start_offset_ms: 2 } as Span;
|
||||
const last = { ...first, span_id: "last", start_offset_ms: 3 };
|
||||
const details = new Map([
|
||||
["root", detail("root", [user], [answer])],
|
||||
["first", detail("first", [user], [call])],
|
||||
["tool", { ...detail("tool", { order: 42 }, "Shipped"), output: "Shipped" }],
|
||||
["last", detail("last", [user, call, result], [answer])],
|
||||
]);
|
||||
const items = buildConversation([root, first, tool, last], details, true);
|
||||
expect(items.flatMap((item) => item.messages)).toEqual([user, { ...call, tool_calls: [] }, answer]);
|
||||
expect(items.find((item) => item.span.span_id === "tool")).toMatchObject({
|
||||
toolCall: { name: "lookup", args: { order: 42 } },
|
||||
toolResult: "Shipped",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses normalized message content and structured tool arguments from the gateway", () => {
|
||||
const model = { ...root, span_id: "model", parent_span_id: "root", type: "llm" } as Span;
|
||||
const normalized: SpanDetail = {
|
||||
span_id: "model",
|
||||
input: "unparsed input",
|
||||
output: "unparsed output",
|
||||
attributes: {},
|
||||
input_ui: { kind: "messages", messages: [{ role: "user", content: user.content }] },
|
||||
output_ui: {
|
||||
kind: "messages",
|
||||
messages: [
|
||||
{ role: "assistant", content: "Checking", tool_calls: [{ name: "lookup", arguments: '{"order":42}' }] },
|
||||
],
|
||||
},
|
||||
};
|
||||
const details = new Map([
|
||||
["root", detail("root", [], [])],
|
||||
["model", normalized],
|
||||
]);
|
||||
const items = buildConversation([root, model], details, true);
|
||||
expect(items[0].messages).toEqual([
|
||||
user,
|
||||
{ role: "assistant", content: "Checking", tool_calls: [{ name: "lookup", args: { order: 42 } }] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps parallel agent histories separate and excludes framework scaffolding", () => {
|
||||
const agent = { ...root, span_id: "agent", parent_span_id: "root" };
|
||||
const otherAgent = { ...agent, span_id: "other" };
|
||||
const first = { ...root, span_id: "first", parent_span_id: "agent", type: "llm", start_offset_ms: 1 } as Span;
|
||||
const second = { ...first, span_id: "second", parent_span_id: "other", start_offset_ms: 2 };
|
||||
const framework = { ...first, span_id: "framework", type: "framework" } as Span;
|
||||
const spans = [root, agent, otherAgent, first, second, framework];
|
||||
expect(conversationSteps(spans).map((span) => span.span_id)).toEqual(["root", "agent", "other", "first", "second"]);
|
||||
const details = new Map([
|
||||
["root", detail("root", [], [])],
|
||||
["agent", detail("agent", [], [])],
|
||||
["other", detail("other", [], [])],
|
||||
["first", detail("first", [user], [answer])],
|
||||
["second", detail("second", [user], [answer])],
|
||||
]);
|
||||
expect(buildConversation(spans, details, true).flatMap((item) => item.messages)).toEqual([
|
||||
user,
|
||||
answer,
|
||||
user,
|
||||
answer,
|
||||
]);
|
||||
});
|
||||
|
||||
it("defers the root's final answer until all sections have been loaded", () => {
|
||||
const details = new Map([["root", detail("root", [user], [answer])]]);
|
||||
expect(buildConversation([root], details, false).flatMap((item) => item.messages)).toEqual([user]);
|
||||
expect(buildConversation([root], details, true).flatMap((item) => item.messages)).toEqual([user, answer]);
|
||||
});
|
||||
|
||||
it("pairs typed tool arguments without coercing strings that resemble JSON", () => {
|
||||
const model = { ...root, span_id: "model", parent_span_id: "root", type: "llm", start_offset_ms: 1 } as Span;
|
||||
const tool = { ...model, span_id: "tool", name: "lookup", type: "tool", start_offset_ms: 2 } as Span;
|
||||
const args = { order: 42, active: true, filters: { tags: ["paid"] }, empty: null, label: "42", text: "true" };
|
||||
const typedCall = { ...call, tool_calls: [{ name: "lookup", args }] };
|
||||
const toolDetail: SpanDetail = {
|
||||
...detail("tool", args, "Shipped"),
|
||||
input_ui: {
|
||||
kind: "fields",
|
||||
fields: Object.entries(args).map(([key, value]) => ({
|
||||
key,
|
||||
value: typeof value === "string" ? value : JSON.stringify(value),
|
||||
})),
|
||||
},
|
||||
};
|
||||
const details = new Map([
|
||||
["root", detail("root", [user], [])],
|
||||
["model", detail("model", [user], [typedCall])],
|
||||
["tool", toolDetail],
|
||||
]);
|
||||
const items = buildConversation([root, model, tool], details, true);
|
||||
expect(items.flatMap((item) => item.messages.flatMap((message) => message.tool_calls ?? []))).toEqual([]);
|
||||
expect(items.find((item) => item.toolCall)?.toolCall?.args).toEqual(args);
|
||||
|
||||
const differentType = { ...typedCall, tool_calls: [{ name: "lookup", args: { ...args, order: "42" } }] };
|
||||
details.set("model", detail("model", [user], [differentType]));
|
||||
const unmatched = buildConversation([root, model, tool], details, true);
|
||||
expect(unmatched.flatMap((item) => item.messages.flatMap((message) => message.tool_calls ?? []))).toEqual(
|
||||
differentType.tool_calls,
|
||||
);
|
||||
});
|
||||
|
||||
it("loads child agents and places their own answers after their tools", () => {
|
||||
const parent = { ...root, start_offset_ms: 0, duration_ms: 100 };
|
||||
const agent = { ...parent, span_id: "agent", parent_span_id: "root", start_offset_ms: 1, duration_ms: 20 };
|
||||
const tool = {
|
||||
...agent,
|
||||
span_id: "tool",
|
||||
parent_span_id: "agent",
|
||||
type: "tool",
|
||||
start_offset_ms: 2,
|
||||
duration_ms: 2,
|
||||
} as Span;
|
||||
const next = { ...tool, span_id: "next", parent_span_id: "root", type: "llm", start_offset_ms: 30 } as Span;
|
||||
const final = { role: "assistant", content: "Finished the whole run" };
|
||||
const details = new Map([
|
||||
["root", detail("root", [], [final])],
|
||||
["agent", detail("agent", [user], [answer])],
|
||||
["tool", detail("tool", { order: 42 }, "Shipped")],
|
||||
["next", detail("next", [], [final])],
|
||||
]);
|
||||
const spans = [parent, agent, tool, next];
|
||||
expect(conversationSteps(spans)).toContain(agent);
|
||||
const items = buildConversation(spans, details, true);
|
||||
expect(items.map((item) => item.id)).toEqual(["agent", "tool", "agent-output", "next"]);
|
||||
expect(items.flatMap((item) => item.messages)).toEqual([user, answer, final]);
|
||||
|
||||
details.delete("next");
|
||||
expect(buildConversation(spans, details, false).flatMap((item) => item.messages)).toEqual([user, answer]);
|
||||
details.delete("tool");
|
||||
expect(buildConversation(spans, details, false).flatMap((item) => item.messages)).toEqual([user]);
|
||||
});
|
||||
|
||||
it("does not repeat a child agent answer already recorded by its model", () => {
|
||||
const parent = { ...root, start_offset_ms: 0, duration_ms: 100 };
|
||||
const agent = { ...parent, span_id: "agent", parent_span_id: "root", start_offset_ms: 1, duration_ms: 20 };
|
||||
const model = {
|
||||
...agent,
|
||||
span_id: "model",
|
||||
parent_span_id: "agent",
|
||||
type: "llm",
|
||||
start_offset_ms: 2,
|
||||
duration_ms: 2,
|
||||
} as Span;
|
||||
const spans = [parent, agent, model];
|
||||
const details = new Map([
|
||||
["root", detail("root", [], [])],
|
||||
["agent", detail("agent", [user], [answer])],
|
||||
["model", detail("model", [user], [answer])],
|
||||
]);
|
||||
expect(buildConversation(spans, details, true).flatMap((item) => item.messages)).toEqual([user, answer]);
|
||||
});
|
||||
|
||||
it("stops at a missing step even if later details and completion are supplied", () => {
|
||||
const first = { ...root, span_id: "first", parent_span_id: "root", type: "llm", start_offset_ms: 1 } as Span;
|
||||
const missing = { ...first, span_id: "missing", start_offset_ms: 2 };
|
||||
const last = { ...first, span_id: "last", start_offset_ms: 3 };
|
||||
const details = new Map([
|
||||
["root", detail("root", [user], [answer])],
|
||||
["first", detail("first", [user], [call])],
|
||||
["last", detail("last", [user, call, result], [answer])],
|
||||
]);
|
||||
expect(buildConversation([root, first, missing, last], details, true).flatMap((item) => item.messages)).toEqual([
|
||||
user,
|
||||
call,
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps nested and parallel agent answers in their own histories", () => {
|
||||
const parent = { ...root, start_offset_ms: 0, duration_ms: 100 };
|
||||
const agent = { ...parent, span_id: "agent", parent_span_id: "root", start_offset_ms: 1, duration_ms: 20 };
|
||||
const nested = { ...agent, span_id: "nested", parent_span_id: "agent", start_offset_ms: 2, duration_ms: 5 };
|
||||
const other = { ...agent, span_id: "other", start_offset_ms: 4, duration_ms: 30 };
|
||||
const tool = {
|
||||
...nested,
|
||||
span_id: "tool",
|
||||
parent_span_id: "nested",
|
||||
type: "tool",
|
||||
start_offset_ms: 3,
|
||||
duration_ms: 1,
|
||||
} as Span;
|
||||
const otherTool = { ...tool, span_id: "other-tool", parent_span_id: "other", start_offset_ms: 5 };
|
||||
const summary = { role: "assistant", content: "Nested work complete" };
|
||||
const details = new Map([
|
||||
["root", detail("root", [], [])],
|
||||
["agent", detail("agent", [], [summary])],
|
||||
["nested", detail("nested", [], [answer])],
|
||||
["other", detail("other", [], [answer])],
|
||||
["tool", detail("tool", {}, "First result")],
|
||||
["other-tool", detail("other-tool", {}, "Second result")],
|
||||
]);
|
||||
const items = buildConversation([parent, agent, nested, tool, other, otherTool], details, true);
|
||||
expect(items.map((item) => item.id)).toEqual([
|
||||
"tool",
|
||||
"other-tool",
|
||||
"nested-output",
|
||||
"agent-output",
|
||||
"other-output",
|
||||
]);
|
||||
expect(items.flatMap((item) => item.messages)).toEqual([answer, summary, answer]);
|
||||
});
|
||||
|
||||
it("renders a tool-only trace as one exchange", () => {
|
||||
const tool = { ...root, type: "tool" } as Span;
|
||||
const items = buildConversation([tool], new Map([["root", detail("root", { order: 42 }, "Shipped")]]), true);
|
||||
expect(items).toHaveLength(1);
|
||||
expect(items[0].toolResult).toBe("Shipped");
|
||||
expect(items[0].messages).toEqual([]);
|
||||
});
|
||||
|
||||
it("removes forwarded child answers from ancestors while preserving identical answers in sibling branches", () => {
|
||||
const parent = { ...root, start_offset_ms: 0, duration_ms: 100 };
|
||||
const agent = { ...parent, span_id: "agent", parent_span_id: "root", start_offset_ms: 1, duration_ms: 20 };
|
||||
const nested = { ...agent, span_id: "nested", parent_span_id: "agent", start_offset_ms: 2, duration_ms: 5 };
|
||||
const other = { ...agent, span_id: "other", start_offset_ms: 4, duration_ms: 30 };
|
||||
const spans = [parent, agent, nested, other];
|
||||
const details = new Map(spans.map((span) => [span.span_id, detail(span.span_id, [], [answer])]));
|
||||
const items = buildConversation(spans, details, true);
|
||||
expect(items.map((item) => item.id)).toEqual(["nested-output", "other-output"]);
|
||||
expect(items.flatMap((item) => item.messages)).toEqual([answer, answer]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
import type { Span, SpanDetail, TraceMessage, TraceToolCall, UIContent } from "./traceTypes";
|
||||
import { isFrameworkSpan, parseJson, parseMessages, prettyPayload } from "./traceUtils";
|
||||
|
||||
export const CONVERSATION_PAGE_SIZE = 20;
|
||||
|
||||
export function conversationSteps(spans: readonly Span[]): Span[] {
|
||||
const parents = new Set(spans.map((span) => span.parent_span_id));
|
||||
return spans
|
||||
.filter((span) => {
|
||||
const isEvent = ["agent", "llm", "tool"].includes(span.type) || !parents.has(span.span_id);
|
||||
return !isFrameworkSpan(span) && (span.parent_span_id === null || isEvent);
|
||||
})
|
||||
.sort((a, b) => a.start_offset_ms - b.start_offset_ms);
|
||||
}
|
||||
|
||||
function contentText(value: string, content?: UIContent): string {
|
||||
if (content?.kind === "text") return content.text;
|
||||
if (content?.kind === "fields")
|
||||
return JSON.stringify(Object.fromEntries(content.fields.map((field) => [field.key, field.value])), null, 2);
|
||||
if (content?.kind === "messages") return content.messages.map((message) => message.content).join("\n");
|
||||
return prettyPayload(value);
|
||||
}
|
||||
|
||||
function messages(value: string, content: UIContent | undefined, role: string): TraceMessage[] {
|
||||
if (content?.kind === "messages")
|
||||
return content.messages.map((message) => ({
|
||||
...message,
|
||||
tool_calls: message.tool_calls?.map((call) => ({
|
||||
name: call.name,
|
||||
args: parseJson(call.arguments) ?? call.arguments,
|
||||
})),
|
||||
}));
|
||||
const parsed = !content ? parseMessages(value) : null;
|
||||
if (parsed) return parsed;
|
||||
const text = contentText(value, content);
|
||||
return text ? [{ role, content: text }] : [];
|
||||
}
|
||||
|
||||
function stableValue(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(stableValue);
|
||||
if (value !== null && typeof value === "object")
|
||||
return Object.fromEntries(
|
||||
Object.entries(value)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, item]) => [key, stableValue(item)]),
|
||||
);
|
||||
return value;
|
||||
}
|
||||
|
||||
const messageKey = (message: TraceMessage): string =>
|
||||
JSON.stringify(
|
||||
stableValue({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
tool_calls: message.tool_calls?.length ? message.tool_calls : undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
export function newConversationMessages(
|
||||
previous: readonly TraceMessage[],
|
||||
current: readonly TraceMessage[],
|
||||
): TraceMessage[] {
|
||||
const oldKeys = previous.map(messageKey);
|
||||
const newKeys = current.map(messageKey);
|
||||
if (newKeys.every((key, index) => key === oldKeys[index])) return [];
|
||||
for (let overlap = Math.min(oldKeys.length, newKeys.length); overlap > 0; overlap--) {
|
||||
if (oldKeys.slice(-overlap).every((key, index) => key === newKeys[index])) return current.slice(overlap);
|
||||
}
|
||||
return [...current];
|
||||
}
|
||||
|
||||
export interface ConversationItem {
|
||||
id: string;
|
||||
span: Span;
|
||||
messages: TraceMessage[];
|
||||
toolCall?: TraceToolCall;
|
||||
toolResult?: string;
|
||||
showError?: boolean;
|
||||
}
|
||||
|
||||
function toolItem(
|
||||
span: Span,
|
||||
detail: SpanDetail,
|
||||
pending: TraceToolCall[],
|
||||
items: ConversationItem[],
|
||||
): ConversationItem {
|
||||
const args =
|
||||
parseJson(detail.input) ??
|
||||
(detail.input_ui?.kind === "fields"
|
||||
? Object.fromEntries(detail.input_ui.fields.map((field) => [field.key, field.value]))
|
||||
: detail.input);
|
||||
const call = { name: span.name, args };
|
||||
const match = pending.findIndex(
|
||||
(candidate) =>
|
||||
candidate.name === call.name &&
|
||||
JSON.stringify(stableValue(candidate.args)) === JSON.stringify(stableValue(call.args)),
|
||||
);
|
||||
if (match >= 0) {
|
||||
const [matched] = pending.splice(match, 1);
|
||||
for (const item of items)
|
||||
for (const message of item.messages) {
|
||||
if (message.tool_calls?.includes(matched))
|
||||
message.tool_calls = message.tool_calls.filter((call) => call !== matched);
|
||||
}
|
||||
}
|
||||
const result = contentText(detail.output, detail.output_ui);
|
||||
return { id: span.span_id, span, messages: [], toolCall: call, toolResult: result };
|
||||
}
|
||||
|
||||
interface ConversationEvent {
|
||||
span: Span;
|
||||
time: number;
|
||||
output: boolean;
|
||||
}
|
||||
|
||||
function conversationEvents(
|
||||
steps: readonly Span[],
|
||||
byId: ReadonlyMap<string, Span>,
|
||||
details: ReadonlyMap<string, SpanDetail>,
|
||||
complete: boolean,
|
||||
): ConversationEvent[] {
|
||||
const missingIndex = steps.findIndex((span) => !details.has(span.span_id));
|
||||
const loaded = missingIndex < 0 ? steps : steps.slice(0, missingIndex);
|
||||
const boundary = missingIndex < 0 ? -Infinity : steps[missingIndex].start_offset_ms;
|
||||
const ends = new Map<string, number>();
|
||||
const depths = new Map<string, number>();
|
||||
for (const span of steps) {
|
||||
const end = span.start_offset_ms + span.duration_ms;
|
||||
const visited = new Set<string>();
|
||||
let ancestor: Span | undefined = span;
|
||||
while (ancestor && !visited.has(ancestor.span_id)) {
|
||||
visited.add(ancestor.span_id);
|
||||
ends.set(ancestor.span_id, Math.max(ends.get(ancestor.span_id) ?? -Infinity, end));
|
||||
ancestor = ancestor.parent_span_id ? byId.get(ancestor.parent_span_id) : undefined;
|
||||
}
|
||||
depths.set(span.span_id, visited.size);
|
||||
}
|
||||
return loaded
|
||||
.flatMap((span) => {
|
||||
const start = { span, time: span.start_offset_ms, output: false };
|
||||
if (span.type === "tool" || (span.type !== "agent" && span.parent_span_id !== null)) return [start];
|
||||
const end = ends.get(span.span_id)!;
|
||||
return (complete && missingIndex < 0) || end < boundary ? [start, { span, time: end, output: true }] : [start];
|
||||
})
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.time - b.time ||
|
||||
Number(a.output) - Number(b.output) ||
|
||||
(a.output ? -1 : 1) * (depths.get(a.span.span_id)! - depths.get(b.span.span_id)!),
|
||||
);
|
||||
}
|
||||
|
||||
function isDescendant(id: string, ancestorId: string, byId: ReadonlyMap<string, Span>): boolean {
|
||||
let parentId = byId.get(id)?.parent_span_id;
|
||||
const visited = new Set<string>();
|
||||
while (parentId && !visited.has(parentId)) {
|
||||
if (parentId === ancestorId) return true;
|
||||
visited.add(parentId);
|
||||
parentId = byId.get(parentId)?.parent_span_id;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function withoutForwardedAnswers(
|
||||
spanId: string,
|
||||
output: TraceMessage[],
|
||||
completedOutputs: ReadonlyMap<string, TraceMessage[]>,
|
||||
byId: ReadonlyMap<string, Span>,
|
||||
): TraceMessage[] {
|
||||
return [...completedOutputs].reduce(
|
||||
(fresh, [childId, childOutput]) =>
|
||||
isDescendant(childId, spanId, byId) ? newConversationMessages(childOutput, fresh) : fresh,
|
||||
output,
|
||||
);
|
||||
}
|
||||
|
||||
export function buildConversation(
|
||||
spans: readonly Span[],
|
||||
details: ReadonlyMap<string, SpanDetail>,
|
||||
complete: boolean,
|
||||
): ConversationItem[] {
|
||||
const byId = new Map(spans.map((span) => [span.span_id, span]));
|
||||
const histories = new Map<string, TraceMessage[]>();
|
||||
const completedOutputs = new Map<string, TraceMessage[]>();
|
||||
const pendingCalls = new Map<string, TraceToolCall[]>();
|
||||
const items: ConversationItem[] = [];
|
||||
const events = conversationEvents(conversationSteps(spans), byId, details, complete);
|
||||
const branch = (span: Span): string => {
|
||||
if (span.type === "agent" || span.parent_span_id === null) return span.span_id;
|
||||
let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined;
|
||||
const visited = new Set<string>();
|
||||
while (parent && !visited.has(parent.span_id)) {
|
||||
visited.add(parent.span_id);
|
||||
if (parent.type === "agent") return parent.span_id;
|
||||
parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined;
|
||||
}
|
||||
return span.parent_span_id ?? span.span_id;
|
||||
};
|
||||
for (const event of events) {
|
||||
const { span } = event;
|
||||
const detail = details.get(span.span_id)!;
|
||||
const key = branch(span);
|
||||
const history = histories.get(key) ?? [];
|
||||
if (event.output) {
|
||||
const output = messages(detail.output, detail.output_ui, "assistant");
|
||||
const fresh = withoutForwardedAnswers(
|
||||
span.span_id,
|
||||
newConversationMessages(history, output),
|
||||
completedOutputs,
|
||||
byId,
|
||||
);
|
||||
const item = { id: `${span.span_id}-output`, span, messages: fresh, showError: span.status === "error" };
|
||||
if (fresh.length || item.showError) items.push(item);
|
||||
completedOutputs.set(span.span_id, output);
|
||||
histories.set(key, [...history, ...fresh]);
|
||||
continue;
|
||||
}
|
||||
if (span.type === "tool") {
|
||||
const item = toolItem(span, detail, pendingCalls.get(key) ?? [], items);
|
||||
items.push(item);
|
||||
histories.set(key, [...history, { role: "tool", name: span.name, content: item.toolResult ?? "" }]);
|
||||
continue;
|
||||
}
|
||||
const input = messages(detail.input, detail.input_ui, "user");
|
||||
const output = messages(detail.output, detail.output_ui, "assistant");
|
||||
const fresh = newConversationMessages(history, input);
|
||||
if (span.type === "agent" || span.parent_span_id === null) {
|
||||
histories.set(key, input);
|
||||
if (fresh.length) items.push({ id: span.span_id, span, messages: fresh });
|
||||
continue;
|
||||
}
|
||||
const combined = [...fresh, ...output];
|
||||
const retained = input.length && (fresh.length || input.length >= history.length) ? input : history;
|
||||
histories.set(key, [...retained, ...output]);
|
||||
pendingCalls.set(
|
||||
key,
|
||||
output.flatMap((message) => message.tool_calls ?? []),
|
||||
);
|
||||
const item = {
|
||||
id: span.span_id,
|
||||
span,
|
||||
showError: span.status === "error",
|
||||
messages: combined.map((message) => ({
|
||||
...message,
|
||||
tool_calls: message.tool_calls ? [...message.tool_calls] : undefined,
|
||||
})),
|
||||
};
|
||||
if (combined.length || item.showError) items.push(item);
|
||||
}
|
||||
return items
|
||||
.map((item) => ({
|
||||
...item,
|
||||
messages: item.messages.filter((message) => Boolean(message.content) || Boolean(message.tool_calls?.length)),
|
||||
}))
|
||||
.filter((item) => item.messages.length || item.toolResult !== undefined || item.showError);
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import {
|
|||
buildVisibleTree,
|
||||
errorSource,
|
||||
firstErrorSpan,
|
||||
findTraceSteps,
|
||||
fmtMs,
|
||||
GROUP_PAGE_SIZE,
|
||||
groupRowId,
|
||||
|
|
@ -45,6 +46,8 @@ const span = (overrides: SpanOverrides): Span => ({
|
|||
duration_ms: 1,
|
||||
status: "ok",
|
||||
error: null,
|
||||
error_truncated: false,
|
||||
framework: "",
|
||||
input_preview: "",
|
||||
model: null,
|
||||
input_tokens: 0,
|
||||
|
|
@ -322,3 +325,38 @@ describe("treeGuides", () => {
|
|||
expect(guides.map((g) => g.stem)).toEqual([true, true, false, false, false, false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findTraceSteps", () => {
|
||||
it("searches names, agents, models, IDs and inputs in time order without changing the trace", () => {
|
||||
const spans = [
|
||||
span({ span_id: "late", name: "Check", start_offset_ms: 20 }),
|
||||
span({ span_id: "early", model: "check-model", start_offset_ms: 1 }),
|
||||
span({ span_id: "agent", agent: "check-agent", start_offset_ms: 2 }),
|
||||
span({ span_id: "check-id", start_offset_ms: 3 }),
|
||||
span({ span_id: "input", input_preview: "Check this case", start_offset_ms: 4 }),
|
||||
];
|
||||
expect(findTraceSteps(spans, " CHECK ", false, true).map((item) => item.span_id)).toEqual([
|
||||
"early",
|
||||
"agent",
|
||||
"check-id",
|
||||
"input",
|
||||
"late",
|
||||
]);
|
||||
expect(spans[0].span_id).toBe("late");
|
||||
});
|
||||
|
||||
it("combines search and errors while respecting the framework display setting", () => {
|
||||
const tool: SpanOverrides = { span_id: "tool", name: "check", type: "tool", status: "error" };
|
||||
const framework: SpanOverrides = {
|
||||
span_id: "framework",
|
||||
parent_span_id: "root",
|
||||
name: "check",
|
||||
type: "framework",
|
||||
status: "error",
|
||||
};
|
||||
const spans = [span(tool), span({ span_id: "ok", name: "check", type: "tool" }), span(framework)];
|
||||
expect(findTraceSteps(spans, "check", true, true).map((item) => item.span_id)).toEqual(["tool"]);
|
||||
expect(findTraceSteps(spans, "check", true, false).map((item) => item.span_id)).toEqual(["tool", "framework"]);
|
||||
expect(findTraceSteps(spans, "missing", false, false)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -210,6 +210,27 @@ export function buildTreeRows(spans: readonly Span[], state: SpanTreeState): Tre
|
|||
return ctx.rows;
|
||||
}
|
||||
|
||||
export function findTraceSteps(
|
||||
spans: readonly Span[],
|
||||
query: string,
|
||||
errorsOnly: boolean,
|
||||
hideFramework: boolean,
|
||||
): Span[] {
|
||||
const search = query.trim().toLowerCase();
|
||||
return spans
|
||||
.filter((span) => {
|
||||
const visible = !hideFramework || !isFrameworkSpan(span);
|
||||
const matchesStatus = !errorsOnly || span.status === "error";
|
||||
const matchesQuery =
|
||||
!search ||
|
||||
[span.name, span.agent, span.model, span.input_preview, span.span_id].some((value) =>
|
||||
value?.toLowerCase().includes(search),
|
||||
);
|
||||
return visible && matchesStatus && matchesQuery;
|
||||
})
|
||||
.sort(byStart);
|
||||
}
|
||||
|
||||
/** Tree state with every visible ancestor of `spanId` expanded and any group holding it paged far enough. */
|
||||
export function revealSpanInState(spans: readonly Span[], state: SpanTreeState, spanId: string): SpanTreeState {
|
||||
const { children } = buildVisibleTree(spans, !state.hideFramework);
|
||||
|
|
|
|||
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -41334,6 +41334,13 @@ export interface components {
|
|||
*/
|
||||
status: "estimated" | "needs_review" | "error";
|
||||
};
|
||||
/** ROIEstimatorModel */
|
||||
ROIEstimatorModel: {
|
||||
/** Model Name */
|
||||
model_name: string;
|
||||
/** Provider Models */
|
||||
provider_models: string[];
|
||||
};
|
||||
/** ROIIdentityMapResponse */
|
||||
ROIIdentityMapResponse: {
|
||||
/** Identity Map */
|
||||
|
|
@ -41486,6 +41493,11 @@ export interface components {
|
|||
default_prompt: string;
|
||||
/** Estimator Model */
|
||||
estimator_model: string;
|
||||
/**
|
||||
* Estimator Models
|
||||
* @default []
|
||||
*/
|
||||
estimator_models: components["schemas"]["ROIEstimatorModel"][];
|
||||
/** Estimator Prompt */
|
||||
estimator_prompt: string;
|
||||
/** Github Api Url */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue