diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md
index 1b97e159105..d9047b675ce 100644
--- a/litellm/integrations/otel/README.md
+++ b/litellm/integrations/otel/README.md
@@ -213,6 +213,15 @@ nothing here imports outside it:
`config.yaml` — the latter reach the config through the logger's constructor
kwargs. `baggage_team_metadata_keys` is empty by default, so none of a team's
free-form metadata is promoted until each sub-key is explicitly allowlisted.
+ `excluded_services` withholds datastore spans from key/team `callback_vars`
+ destinations while the operator's own exporters keep them: set
+ `LITELLM_OTEL_EXCLUDED_SERVICES` (comma-separated) or `excluded_services`
+ (a YAML list) under `callback_settings.otel`, naming the datastore services
+ to withhold (`redis`, `postgres`, `batch_write_to_db`, `redis_*`, or their
+ `db.system.name` spellings `redis` / `postgresql`). Unknown names are logged
+ as an error and ignored. A span is withheld when its `db.system.name` /
+ `db.system` attribute is in the set, so request root, auth, guardrail and
+ model spans can never be excluded.
- [`baggage.py`](./model/baggage.py) — the single definition of which request-identity
values are promoted into Baggage (so child spans inherit them) and under which
attribute keys.
diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py
index e21711c2708..55eb8e8fb71 100644
--- a/litellm/integrations/otel/logger.py
+++ b/litellm/integrations/otel/logger.py
@@ -30,7 +30,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.model.baggage import promoted_baggage
-from litellm.integrations.otel.model.config import OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from
from litellm.integrations.otel.model.metadata import (
LLMCallEvent,
RequestIdentity,
@@ -898,12 +898,29 @@ def publish_global_otel_v2_provider(
"""
global _published_v2_provider
logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
- attach_tenant_fan_out(logger.tracer_provider, *_v2_configs(in_memory_loggers, logger))
+ attach_tenant_fan_out(
+ logger.tracer_provider,
+ *_v2_configs(in_memory_loggers, logger),
+ excluded_db_systems=_excluded_db_systems(logger),
+ )
set_global_provider(logger.tracer_provider)
_published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out
return logger
+def _excluded_db_systems(logger: "OpenTelemetryV2") -> frozenset[str]:
+ """The datastore services withheld from tenant destinations.
+
+ ``callback_settings.otel.excluded_services`` wins over the env var whichever
+ logger got published: with ``callbacks: [langfuse_otel, otel]`` the ``otel``
+ callback folds into the preset, whose config is env-only.
+ """
+ configured: Final = litellm.callback_settings.get("otel", {}).get("excluded_services")
+ if configured is None:
+ return logger.config.excluded_services
+ return excluded_db_systems_from(configured)
+
+
def _v2_configs(in_memory_loggers: Sequence[object], logger: "OpenTelemetryV2") -> tuple[OpenTelemetryV2Config, ...]:
"""Every v2 logger's config, the published logger's first.
@@ -963,7 +980,11 @@ def fan_out_provider() -> ApiTracerProvider:
return published
logger: Final = _registered_v2_logger()
if logger is not None:
- attach_tenant_fan_out(logger.tracer_provider, logger.config)
+ attach_tenant_fan_out(
+ logger.tracer_provider,
+ logger.config,
+ excluded_db_systems=_excluded_db_systems(logger),
+ )
return logger.tracer_provider
return get_tracer_provider()
diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py
index 5a3965862e0..9eb29157d6f 100644
--- a/litellm/integrations/otel/model/config.py
+++ b/litellm/integrations/otel/model/config.py
@@ -4,14 +4,16 @@ from enum import Enum
from functools import lru_cache
from typing import Annotated, Any, Final
-from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
+from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, ValidationError, field_validator, model_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
+from litellm._logging import verbose_logger
from litellm.integrations.otel.model.baggage import (
BAGGAGE_PROMOTED_KEYS,
DEFAULT_BAGGAGE_METADATA_KEYS,
DEFAULT_BAGGAGE_TEAM_METADATA_KEYS,
)
+from litellm.integrations.otel.model.spans import POSTGRESQL, db_system
from litellm.types.utils import OtelSpanScope
#: Master feature-flag env var. The logger is inert until this is truthy.
@@ -174,6 +176,19 @@ class OpenTelemetryV2Config(BaseSettings):
"key/team destinations are not affected."
),
)
+ excluded_services: Annotated[frozenset[str], NoDecode] = Field(
+ default_factory=frozenset,
+ validation_alias=AliasChoices("excluded_services", "LITELLM_OTEL_EXCLUDED_SERVICES"),
+ description=(
+ "Datastore services whose spans are withheld from key/team ``callback_vars`` "
+ "OTel destinations (the operator's own exporters still receive them). Accepted "
+ "values are the datastore ``ServiceTypes`` names (``redis``, ``postgres``, "
+ "``batch_write_to_db``, ``redis_*``) or their ``db.system.name`` spellings "
+ "(``redis``, ``postgresql``); stored normalized to ``db.system.name`` values. "
+ "Configure via the ``LITELLM_OTEL_EXCLUDED_SERVICES`` env var (comma-separated) "
+ "or ``callback_settings.otel.excluded_services`` in config.yaml (a YAML list)."
+ ),
+ )
# ----- explicit multi-destination / vocabulary configuration ------------ #
@@ -284,6 +299,11 @@ class OpenTelemetryV2Config(BaseSettings):
return [item.strip() for item in value.split(",") if item.strip()]
return value
+ @field_validator("excluded_services", mode="before")
+ @classmethod
+ def _read_excluded_services(cls, value: object) -> frozenset[str]:
+ return excluded_service_names(value)
+
@model_validator(mode="after")
def _normalize(self) -> "OpenTelemetryV2Config":
# An endpoint with the default exporter kind implies OTLP/HTTP.
@@ -316,6 +336,7 @@ class OpenTelemetryV2Config(BaseSettings):
if self.legacy_compat and "legacy" not in names:
names.append("legacy")
self.mapper_names = names
+ self.excluded_services = _normalize_excluded_services(self.excluded_services)
return self
@property
@@ -334,3 +355,55 @@ class OpenTelemetryV2Config(BaseSettings):
@classmethod
def from_env(cls) -> "OpenTelemetryV2Config":
return cls()
+
+
+_EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...])
+
+
+def excluded_db_systems_from(value: object) -> frozenset[str]:
+ """Normalize a raw ``excluded_services`` value without building a settings model that rereads the env"""
+ return _normalize_excluded_services(excluded_service_names(value))
+
+
+def excluded_service_names(value: object) -> frozenset[str]:
+ """Read a YAML list or comma-separated string of service names, logging and dropping unusable input
+ so a malformed value cannot stop the OTel logger from being built"""
+ if value is None:
+ return frozenset()
+ try:
+ parsed: Final = _EXCLUDED_SERVICES_INPUT.validate_python(value)
+ except ValidationError:
+ verbose_logger.error("excluded_services must be a list or comma-separated string; %r ignored", value)
+ return frozenset()
+ items: Final = tuple(parsed.split(",")) if isinstance(parsed, str) else parsed
+ return frozenset(name for item in items if (name := _service_name(item)))
+
+
+def _service_name(item: object) -> str:
+ if not isinstance(item, str):
+ verbose_logger.error("excluded_services must be a list of service names; %r ignored", item)
+ return ""
+ return item.strip().lower()
+
+
+def _normalize_excluded_services(services: frozenset[str]) -> frozenset[str]:
+ """Fold each accepted spelling to its ``db.system.name`` value.
+
+ ``postgres`` and ``postgresql`` name the same system, as do every
+ ``ServiceTypes`` member that ``db_system`` maps. Anything else means the
+ operator pointed the setting at a span family it cannot cover; those names
+ are logged and dropped so a typo cannot take the proxy down.
+ """
+ resolved: Final = frozenset(
+ system for service in services if (system := _db_system_for_excluded_service(service)) is not None
+ )
+ return resolved
+
+
+def _db_system_for_excluded_service(service: str) -> str | None:
+ resolved: Final = db_system(service) if service != POSTGRESQL else POSTGRESQL
+ if resolved is None:
+ verbose_logger.error(
+ "excluded_services: %r is not a datastore service; ignored. Allowed: postgres, redis", service
+ )
+ return resolved
diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py
index 8bac36aad76..25878e8a302 100644
--- a/litellm/integrations/otel/plumbing/providers.py
+++ b/litellm/integrations/otel/plumbing/providers.py
@@ -418,6 +418,13 @@ def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool:
return any(key in attributes for key in _DB_SYSTEM_KEYS)
+def _is_excluded_database_span(attributes: Mapping[str, AttributeValue], excluded: frozenset[str]) -> bool:
+ if not excluded:
+ return False
+ system: Final = attributes.get(DB.SYSTEM_NAME) or attributes.get(DB.SYSTEM_LEGACY)
+ return isinstance(system, str) and system in excluded
+
+
def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool:
return any(key in attributes for key in _TENANT_OWNED_KEYS)
@@ -549,10 +556,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None,
shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS,
operator_sinks: 'Mapping[_SinkKey, "OtelSpanScope"]' = MappingProxyType({}),
+ excluded_db_systems: frozenset[str] = frozenset(),
pending_drains: int = _MAX_PENDING_DRAINS,
drain_pool: _DrainPool | None = None,
) -> None:
self._operator_sinks: Final = operator_sinks
+ self._excluded_db_systems: Final = excluded_db_systems
self._drain_seconds: Final = shutdown_drain_seconds
self._lock: Final = threading.Condition()
self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates
@@ -567,9 +576,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
def on_end(self, span: ReadableSpan) -> None:
suppressed: Final = suppressed_backends()
+ attributes: Final = span.attributes or _NO_ATTRIBUTES
for destination in request_destinations():
- if self._operator_already_writes(span, destination, suppressed) or not _in_scope(
- span, destination.span_scope
+ if (
+ self._operator_already_writes(span, destination, suppressed)
+ or not _in_scope(span, destination.span_scope)
+ or _is_excluded_database_span(attributes, self._excluded_db_systems)
):
continue
processor = self._acquire(destination)
@@ -1155,7 +1167,9 @@ def build_tracer_provider(
_FAN_OUT_ATTACH_LOCK: Final = threading.Lock()
-def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Config) -> None:
+def attach_tenant_fan_out(
+ provider: TracerProvider, *configs: OpenTelemetryV2Config, excluded_db_systems: frozenset[str] = frozenset()
+) -> None:
"""Give ``provider`` the fan-out that delivers spans to key/team destinations.
Called on the one provider published as the OTel global, and idempotent so a
@@ -1164,12 +1178,18 @@ def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Con
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
config per v2 logger since each keeps its own provider and still writes its
account, so an additive destination pointing at any of them is delivered once
- rather than twice.
+ rather than twice. ``excluded_db_systems`` only filters what the fan-out
+ delivers, never the operator's own exporters.
"""
with _FAN_OUT_ATTACH_LOCK:
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
return
- provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_scopes(*configs)))
+ provider.add_span_processor(
+ TenantFanOutSpanProcessor(
+ operator_sinks=operator_sink_scopes(*configs),
+ excluded_db_systems=excluded_db_systems,
+ )
+ )
def deliverable_destinations(
diff --git a/tests/integration/_support/otlp_sink.py b/tests/integration/_support/otlp_sink.py
new file mode 100644
index 00000000000..eeabe9d886f
--- /dev/null
+++ b/tests/integration/_support/otlp_sink.py
@@ -0,0 +1,528 @@
+"""OTLP/HTTP trace sink: records exported spans and exposes them over a control API.
+
+Accepts ``application/x-protobuf`` ``ExportTraceServiceRequest`` bodies and OTLP
+``http/json`` bodies on any path. Tests read spans through ``recorded_spans`` and
+steer the sink through ``configure``; the process can also be frozen with
+``SIGSTOP``/``SIGCONT`` after reading its pid from ``/__pid``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import os
+import signal
+import socket
+import ssl
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import ExitStack, contextmanager
+from dataclasses import dataclass, field
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import httpx
+import psutil
+from pydantic import JsonValue, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
+
+INTERNAL_MARKERS: Final = ("gen_ai.operation.name", "mcp.method.name", "litellm.guardrail_name")
+
+
+class Span(TypedDict):
+ trace_id: ReadOnly[str]
+ span_id: ReadOnly[str]
+ parent_span_id: ReadOnly[str]
+ kind: ReadOnly[int]
+ name: ReadOnly[str]
+ attributes: ReadOnly[Mapping[str, JsonValue]]
+ resource: ReadOnly[Mapping[str, JsonValue]]
+
+
+class _SpanListing(TypedDict):
+ next: ReadOnly[int]
+ spans: ReadOnly[list[Span]]
+
+
+_SPAN_LISTING: Final = TypeAdapter(_SpanListing)
+
+
+def _proto_spans(body: bytes) -> list[Span]:
+ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+ from opentelemetry.proto.common.v1.common_pb2 import AnyValue
+
+ def scalar(value: AnyValue) -> JsonValue:
+ match value.WhichOneof("value"):
+ case "string_value":
+ return value.string_value
+ case "bool_value":
+ return value.bool_value
+ case "int_value":
+ return int(value.int_value)
+ case "double_value":
+ return value.double_value
+ case "bytes_value":
+ return value.bytes_value.decode("utf-8", errors="replace")
+ case "array_value":
+ return [scalar(item) for item in value.array_value.values]
+ case "kvlist_value":
+ return {pair.key: scalar(pair.value) for pair in value.kvlist_value.values}
+ case _:
+ return None
+
+ request: Final = ExportTraceServiceRequest()
+ request.ParseFromString(body)
+ return [
+ Span(
+ trace_id=span.trace_id.hex(),
+ span_id=span.span_id.hex(),
+ parent_span_id=span.parent_span_id.hex(),
+ kind=span.kind,
+ name=span.name,
+ attributes={attribute.key: scalar(attribute.value) for attribute in span.attributes},
+ resource={attribute.key: scalar(attribute.value) for attribute in resource.resource.attributes},
+ )
+ for resource in request.resource_spans
+ for scope in resource.scope_spans
+ for span in scope.spans
+ ]
+
+
+def _json_spans(body: bytes) -> list[Span]:
+ payload: Final = json.loads(body)
+
+ def scalar(value: object) -> JsonValue:
+ if not isinstance(value, dict):
+ return value if isinstance(value, (str, int, float, bool)) or value is None else str(value)
+ for key in ("stringValue", "intValue", "doubleValue", "boolValue", "bytesValue"):
+ if key in value:
+ return value[key]
+ if "arrayValue" in value:
+ return [scalar(item) for item in value["arrayValue"].get("values", [])]
+ if "kvlistValue" in value:
+ return {pair["key"]: scalar(pair["value"]) for pair in value["kvlistValue"].get("values", [])}
+ return None
+
+ return [
+ Span(
+ trace_id=str(span.get("traceId", "")),
+ span_id=str(span.get("spanId", "")),
+ parent_span_id=str(span.get("parentSpanId", "")),
+ kind=int(span.get("kind", 0)),
+ name=str(span.get("name", "")),
+ attributes={attribute["key"]: scalar(attribute.get("value")) for attribute in span.get("attributes", [])},
+ resource={
+ attribute["key"]: scalar(attribute.get("value"))
+ for attribute in resource.get("resource", {}).get("attributes", [])
+ },
+ )
+ for resource in payload.get("resourceSpans", [])
+ for scope in resource.get("scopeSpans", [])
+ for span in scope.get("spans", [])
+ ]
+
+
+def decode_spans(body: bytes, content_type: str) -> list[Span]:
+ if "protobuf" in content_type:
+ return _proto_spans(body)
+ return _json_spans(body)
+
+
+def span_class(span: Span) -> str:
+ if span["kind"] == 2:
+ return "root"
+ if any(marker in span["attributes"] for marker in INTERNAL_MARKERS):
+ return "tenant"
+ return "internal"
+
+
+def spans_for_trace(spans: tuple[Span, ...], trace_id: str) -> tuple[Span, ...]:
+ return tuple(span for span in spans if span["trace_id"] == trace_id)
+
+
+@dataclass(slots=True)
+class _State:
+ spans: list[Span] = field(default_factory=list)
+ requests: list[dict[str, JsonValue]] = field(default_factory=list)
+ status: int = 200
+ delay_seconds: float = 0.0
+ pause: threading.Event = field(default_factory=threading.Event)
+
+ def __post_init__(self) -> None:
+ self.pause.set()
+
+
+class _Handler(BaseHTTPRequestHandler):
+ state: _State
+ protocol_version = "HTTP/1.1"
+
+ def _read_body(self) -> bytes:
+ return self.rfile.read(int(self.headers.get("content-length", "0")))
+
+ def _send_json(self, payload: object, status: int = 200) -> None:
+ body: Final = json.dumps(payload).encode()
+ self.send_response(status)
+ self.send_header("content-type", "application/json")
+ self.send_header("content-length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def _record(self) -> None:
+ body: Final = self._read_body()
+ self.state.pause.wait(timeout=120)
+ if self.state.delay_seconds > 0:
+ time.sleep(self.state.delay_seconds)
+ recorded: Final = decode_spans(body, self.headers.get("content-type", ""))
+ self.state.spans.extend(recorded)
+ self.state.requests.append(
+ {
+ "path": self.path,
+ "count": len(recorded),
+ "host": self.headers.get("host", ""),
+ "headers": dict(self.headers),
+ }
+ )
+ self._send_json({"recorded": len(recorded)}, status=self.state.status)
+
+ do_POST = _record
+ do_PUT = _record
+
+ def do_GET(self) -> None:
+ parsed: Final = urlparse(self.path)
+ if parsed.path == "/__spans":
+ since: Final = int(dict(part.split("=", 1) for part in parsed.query.split("&") if part).get("since", "0"))
+ self._send_json({"next": len(self.state.spans), "spans": self.state.spans[since:]})
+ return
+ if parsed.path == "/__pid":
+ self._send_json({"pid": os.getpid()})
+ return
+ if parsed.path == "/__requests":
+ self._send_json({"requests": self.state.requests})
+ return
+ self._send_json({"error": "unknown"}, status=404)
+
+ def do_DELETE(self) -> None:
+ if urlparse(self.path).path == "/__spans":
+ self.state.spans.clear()
+ self.state.requests.clear()
+ self._send_json({"cleared": True})
+ return
+ self._send_json({"error": "unknown"}, status=404)
+
+ def do_PATCH(self) -> None:
+ if urlparse(self.path).path != "/__control":
+ self._send_json({"error": "unknown"}, status=404)
+ return
+ fields: Final = json.loads(self._read_body() or b"{}")
+ if "status" in fields:
+ self.state.status = int(fields["status"])
+ if "delay_seconds" in fields:
+ self.state.delay_seconds = float(fields["delay_seconds"])
+ if fields.get("paused") is True:
+ self.state.pause.clear()
+ if fields.get("paused") is False:
+ self.state.pause.set()
+ self._send_json({"status": self.state.status, "delay_seconds": self.state.delay_seconds})
+
+ def log_message(self, format: str, *args: object) -> None:
+ pass
+
+
+class _ConnectHandler(_Handler):
+ tunnel_context: ssl.SSLContext
+
+ def do_CONNECT(self) -> None:
+ self.state.requests.append({"connect": self.path})
+ self.connection.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n")
+ wrapped: Final = self.tunnel_context.wrap_socket(self.connection, server_side=True)
+ self.close_connection = True
+ type(self)(wrapped, self.client_address, self.server)
+
+
+_MITM_HOSTS: Final = ("otlp.nr-data.net", "otlp.eu01.nr-data.net")
+
+
+def _mitm_context(directory: Path) -> ssl.SSLContext:
+ from cryptography import x509
+ from cryptography.hazmat.primitives import hashes, serialization
+ from cryptography.hazmat.primitives.asymmetric import rsa
+ from cryptography.x509.oid import NameOID
+
+ directory.mkdir(parents=True, exist_ok=True)
+ now: Final = datetime.datetime.now(datetime.timezone.utc)
+ window: Final = datetime.timedelta(days=2)
+ ca_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+ ca_name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "otlp-sink test CA")])
+ ca_cert: Final = (
+ x509.CertificateBuilder()
+ .subject_name(ca_name)
+ .issuer_name(ca_name)
+ .public_key(ca_key.public_key())
+ .serial_number(x509.random_serial_number())
+ .not_valid_before(now - window)
+ .not_valid_after(now + window)
+ .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+ .sign(ca_key, hashes.SHA256())
+ )
+ leaf_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+ leaf_cert: Final = (
+ x509.CertificateBuilder()
+ .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, _MITM_HOSTS[0])]))
+ .issuer_name(ca_cert.subject)
+ .public_key(leaf_key.public_key())
+ .serial_number(x509.random_serial_number())
+ .not_valid_before(now - window)
+ .not_valid_after(now + window)
+ .add_extension(x509.SubjectAlternativeName([x509.DNSName(host) for host in _MITM_HOSTS]), critical=False)
+ .sign(ca_key, hashes.SHA256())
+ )
+ ca_pem: Final = directory / "ca.pem"
+ ca_pem.write_bytes(ca_cert.public_bytes(serialization.Encoding.PEM))
+ leaf_pem: Final = directory / "leaf.pem"
+ leaf_pem.write_bytes(leaf_cert.public_bytes(serialization.Encoding.PEM))
+ leaf_key_pem: Final = directory / "leaf-key.pem"
+ leaf_key_pem.write_bytes(
+ leaf_key.private_bytes(
+ serialization.Encoding.PEM,
+ serialization.PrivateFormat.TraditionalOpenSSL,
+ serialization.NoEncryption(),
+ )
+ )
+ context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ context.load_cert_chain(str(leaf_pem), str(leaf_key_pem))
+ return context
+
+
+def _grpc_trace_server(state: _State, port: int) -> object:
+ from concurrent import futures
+
+ import grpc
+ from opentelemetry.proto.collector.trace.v1 import trace_service_pb2, trace_service_pb2_grpc
+
+ class _TraceService(trace_service_pb2_grpc.TraceServiceServicer):
+ def Export(self, request: object, context: grpc.ServicerContext) -> object:
+ state.pause.wait(timeout=120)
+ if state.delay_seconds > 0:
+ time.sleep(state.delay_seconds)
+ recorded: Final = _proto_spans(request.SerializeToString())
+ state.spans.extend(recorded)
+ state.requests.append(
+ {
+ "grpc": "Export",
+ "metadata": {key: value for key, value in context.invocation_metadata()},
+ "count": len(recorded),
+ }
+ )
+ return trace_service_pb2.ExportTraceServiceResponse()
+
+ server: Final = grpc.server(futures.ThreadPoolExecutor(max_workers=4))
+ trace_service_pb2_grpc.add_TraceServiceServicer_to_server(_TraceService(), server)
+ server.add_insecure_port(f"127.0.0.1:{port}")
+ server.start()
+ return server
+
+
+def recorded_spans(url: str, since: int = 0) -> tuple[int, tuple[Span, ...]]:
+ response: Final = httpx.get(f"{url}/__spans", params={"since": since}, trust_env=False, timeout=15)
+ response.raise_for_status()
+ listing: Final = _SPAN_LISTING.validate_python(response.json())
+ return listing["next"], tuple(listing["spans"])
+
+
+def configure_sink(url: str, **fields: JsonValue) -> None:
+ httpx.request("PATCH", f"{url}/__control", json=dict(fields), trust_env=False, timeout=15).raise_for_status()
+
+
+def reset_sink(url: str) -> None:
+ httpx.delete(f"{url}/__spans", trust_env=False, timeout=15).raise_for_status()
+
+
+def sink_pid(url: str) -> int:
+ return int(httpx.get(f"{url}/__pid", trust_env=False, timeout=15).json()["pid"])
+
+
+_REQUEST_LISTING: Final = TypeAdapter(list[dict[str, JsonValue]])
+
+
+def recorded_requests(url: str) -> tuple[Mapping[str, JsonValue], ...]:
+ response: Final = httpx.get(f"{url}/__requests", trust_env=False, timeout=15)
+ response.raise_for_status()
+ return tuple(_REQUEST_LISTING.validate_python(response.json()["requests"]))
+
+
+@dataclass(frozen=True, slots=True)
+class SpanSinks:
+ operator: str
+ tenant: str
+ arize: str
+
+
+@dataclass(frozen=True, slots=True)
+class GrpcSink:
+ url: str
+ control_url: str
+
+
+@dataclass(frozen=True, slots=True)
+class ConnectSink:
+ proxy_url: str
+ control_url: str
+ ca_pem: str
+
+
+def _free_port() -> int:
+ with socket.socket() as reserve:
+ reserve.bind(("127.0.0.1", 0))
+ return int(reserve.getsockname()[1])
+
+
+def _pid_reachable(url: str) -> bool:
+ try:
+ return httpx.get(f"{url}/__pid", trust_env=False, timeout=2).status_code == 200
+ except httpx.TransportError:
+ return False
+
+
+@contextmanager
+def owned_sinks(directory: Path) -> Iterator[SpanSinks]:
+ from integration._support.process import group_members, signal_group, stop_root_process
+
+ directory.mkdir(parents=True, exist_ok=True)
+ ports: Final = tuple(_free_port() for _ in range(3))
+ root: Final = Path(__file__).resolve().parents[3]
+ with ExitStack() as stack:
+ processes: Final = tuple(
+ subprocess.Popen(
+ [sys.executable, "-P", "-m", "integration._support.otlp_sink", "--port", str(port)],
+ cwd=root,
+ stdout=stack.enter_context((directory / f"otlp-sink-{port}.log").open("w")),
+ stderr=subprocess.STDOUT,
+ start_new_session=True,
+ )
+ for port in ports
+ )
+ try:
+ urls: Final = tuple(f"http://127.0.0.1:{port}" for port in ports)
+ deadline: Final = time.monotonic() + 30
+ while True:
+ alive: Final = all(process.poll() is None for process in processes)
+ assert alive, "OTLP sink exited before readiness"
+ if all(_pid_reachable(url) for url in urls):
+ break
+ assert time.monotonic() < deadline, "OTLP sink readiness deadline exceeded"
+ time.sleep(0.05)
+ yield SpanSinks(operator=urls[0], tenant=urls[1], arize=urls[2])
+ finally:
+ for process in processes:
+ stopped: Final = stop_root_process(process)
+ residual: Final = group_members(process.pid)
+ if residual:
+ signal_group(process.pid, signal.SIGKILL)
+ psutil.wait_procs(residual, timeout=5)
+ survivors: Final = group_members(process.pid)
+ assert not survivors and stopped, "OTLP sink required forced cleanup"
+
+
+@contextmanager
+def _spawn_sink(directory: Path, log_name: str, argv: Sequence[str]) -> Iterator[None]:
+ from integration._support.process import group_members, signal_group, stop_root_process
+
+ directory.mkdir(parents=True, exist_ok=True)
+ root: Final = Path(__file__).resolve().parents[3]
+ with (directory / log_name).open("w") as log:
+ process: Final = subprocess.Popen(
+ [sys.executable, "-P", "-m", "integration._support.otlp_sink", *argv],
+ cwd=root,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ start_new_session=True,
+ )
+ try:
+ yield
+ finally:
+ stopped: Final = stop_root_process(process)
+ residual: Final = group_members(process.pid)
+ if residual:
+ signal_group(process.pid, signal.SIGKILL)
+ psutil.wait_procs(residual, timeout=5)
+ survivors: Final = group_members(process.pid)
+ assert not survivors and stopped, "OTLP sink required forced cleanup"
+
+
+def _await_sink(url: str) -> None:
+ deadline: Final = time.monotonic() + 30
+ while not _pid_reachable(url):
+ assert time.monotonic() < deadline, "OTLP sink readiness deadline exceeded"
+ time.sleep(0.05)
+
+
+@contextmanager
+def owned_grpc_sink(directory: Path) -> Iterator[GrpcSink]:
+ http_port: Final = _free_port()
+ grpc_port: Final = _free_port()
+ with _spawn_sink(
+ directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)]
+ ):
+ control_url: Final = f"http://127.0.0.1:{http_port}"
+ _await_sink(control_url)
+ yield GrpcSink(url=f"http://127.0.0.1:{grpc_port}", control_url=control_url)
+
+
+@contextmanager
+def owned_connect_sink(directory: Path) -> Iterator[ConnectSink]:
+ http_port: Final = _free_port()
+ tunnel_port: Final = _free_port()
+ ca_dir: Final = directory / "mitm"
+ with _spawn_sink(
+ directory,
+ "otlp-connect-sink.log",
+ ["--port", str(http_port), "--connect-port", str(tunnel_port), "--ca-dir", str(ca_dir)],
+ ):
+ control_url: Final = f"http://127.0.0.1:{http_port}"
+ _await_sink(control_url)
+ yield ConnectSink(
+ proxy_url=f"http://127.0.0.1:{tunnel_port}",
+ control_url=control_url,
+ ca_pem=str(ca_dir / "ca.pem"),
+ )
+
+
+def main() -> None:
+ parser: Final = argparse.ArgumentParser()
+ parser.add_argument("--port", type=int, required=True)
+ parser.add_argument("--grpc-port", type=int, default=0)
+ parser.add_argument("--connect-port", type=int, default=0)
+ parser.add_argument("--ca-dir", type=Path, default=None)
+ arguments: Final = parser.parse_args()
+ bound_state: Final = _State()
+
+ class BoundHandler(_Handler):
+ state = bound_state
+
+ if arguments.grpc_port:
+ grpc_server: Final = _grpc_trace_server(bound_state, arguments.grpc_port)
+ assert grpc_server is not None
+ if arguments.connect_port:
+ assert arguments.ca_dir is not None, "--connect-port needs --ca-dir"
+ bound_context: Final = _mitm_context(arguments.ca_dir)
+
+ class BoundConnectHandler(_ConnectHandler):
+ state = bound_state
+ tunnel_context = bound_context # pyright: ignore[reportIncompatibleVariableOverride] # bound context, not a new field
+
+ tunnel: Final = ThreadingHTTPServer(("127.0.0.1", arguments.connect_port), BoundConnectHandler)
+ tunnel.daemon_threads = True
+ threading.Thread(target=tunnel.serve_forever, daemon=True).start()
+ server: Final = ThreadingHTTPServer(("127.0.0.1", arguments.port), BoundHandler)
+ server.daemon_threads = True
+ server.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/integration/observability/conftest.py b/tests/integration/observability/conftest.py
new file mode 100644
index 00000000000..f09a703f047
--- /dev/null
+++ b/tests/integration/observability/conftest.py
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import pytest
+import yaml
+from integration._support.otlp_sink import SpanSinks, owned_sinks
+from pydantic import JsonValue
+
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+
+@pytest.fixture(scope="module")
+def audit_sinks(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SpanSinks]:
+ directory: Final = tmp_path_factory.mktemp("otel-audit-sinks")
+ with owned_sinks(directory) as sinks:
+ yield sinks
+
+
+@pytest.fixture(scope="module")
+def otel_audit_config(audit_sinks: SpanSinks) -> AuditConfigWriter:
+ tenant_host: Final = urlparse(audit_sinks.tenant).netloc
+
+ def write(directory: Path, litellm_settings: Mapping[str, JsonValue] = {}) -> Path:
+ config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+ config["litellm_settings"] = {
+ **config.get("litellm_settings", {}),
+ "callbacks": ["otel"],
+ "provider_url_destination_allowed_hosts": [tenant_host],
+ **dict(litellm_settings),
+ }
+ config["callback_settings"] = {
+ "otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True}
+ }
+ config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2}
+ path: Final = directory / f"otel-audit-{uuid.uuid4().hex}.yaml"
+ path.write_text(yaml.safe_dump(config))
+ return path
+
+ return write
+
+
+@pytest.fixture(scope="module")
+def langfuse_vars(audit_sinks: SpanSinks) -> dict[str, JsonValue]:
+ return {
+ "langfuse_public_key": "pk-lf-audit",
+ "langfuse_secret_key": "sk-lf-audit",
+ "langfuse_host": audit_sinks.tenant,
+ }
diff --git a/tests/integration/observability/test_otel_excluded_services.py b/tests/integration/observability/test_otel_excluded_services.py
new file mode 100644
index 00000000000..48b20651b97
--- /dev/null
+++ b/tests/integration/observability/test_otel_excluded_services.py
@@ -0,0 +1,392 @@
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import (
+ Gateway,
+ eventually,
+ gateway_from_environment,
+)
+from integration._support.otlp_sink import (
+ Span,
+ SpanSinks,
+ recorded_spans,
+ spans_for_trace,
+)
+from integration._support.process import owned_proxy, owned_proxy_process
+from pydantic import JsonValue
+
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+DB_SYSTEM_KEYS: Final = frozenset({"db.system.name", "db.system"})
+
+
+@pytest.fixture(scope="module")
+def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]:
+ with gateway_from_environment() as base:
+ yield base
+
+
+def _config_with(
+ directory: Path,
+ otel_audit_config: AuditConfigWriter,
+ *,
+ otel: Mapping[str, JsonValue] = MappingProxyType({}),
+ extra: Callable[[dict[str, JsonValue]], None] | None = None,
+) -> Path:
+ config: Final = yaml.safe_load(otel_audit_config(directory, {}).read_text())
+ config["callback_settings"]["otel"].update(dict(otel))
+ if extra is not None:
+ extra(config)
+ path: Final = directory / f"otel-excl-{uuid.uuid4().hex}.yaml"
+ path.write_text(yaml.safe_dump(config))
+ return path
+
+
+def _operator_langfuse(audit_sinks: SpanSinks) -> dict[str, str]:
+ return {
+ "LANGFUSE_HOST": audit_sinks.operator,
+ "LANGFUSE_PUBLIC_KEY": "pk-lf-operator",
+ "LANGFUSE_SECRET_KEY": "sk-lf-operator",
+ "OTEL_EXPORTER": "http/json",
+ "OTEL_ENDPOINT": audit_sinks.operator,
+ }
+
+
+def _add_callback(gateway: Gateway, team_id: str, callback_vars: Mapping[str, JsonValue]) -> httpx.Response:
+ return gateway.request(
+ "POST",
+ f"/team/{team_id}/callback",
+ {"callback_name": "langfuse_otel", "callback_vars": dict(callback_vars)},
+ )
+
+
+def _drive(candidate: Gateway, langfuse_vars: Mapping[str, JsonValue]) -> httpx.Response:
+ with candidate.scenario() as scenario:
+ model: Final = scenario.model(model="openai/audit-chat", api_base=f"{candidate.upstream_url}/v1")
+ team_id: Final = scenario.team()
+ callback: Final = _add_callback(candidate, team_id, langfuse_vars)
+ assert callback.status_code == 200, callback.text
+ key: Final = scenario.key(team_id=team_id)
+ response: Final = candidate.request(
+ "POST",
+ "/v1/chat/completions",
+ {"model": model, "messages": [{"role": "user", "content": f"otel-excl-{uuid.uuid4().hex}"}]},
+ key=key,
+ )
+ assert response.status_code == 200, response.text
+ return response
+
+
+def _trace_id(sink_url: str, response: httpx.Response, seconds: float = 40) -> str:
+ call_id: Final = response.headers.get("x-litellm-call-id")
+ response_id: Final = response.json().get("id")
+
+ def look() -> str | None:
+ _, spans = recorded_spans(sink_url)
+ return next(
+ (
+ str(span["trace_id"])
+ for span in spans
+ if (call_id is not None and span["attributes"].get("litellm.call_id") == call_id)
+ or (response_id is not None and span["attributes"].get("gen_ai.response.id") == response_id)
+ ),
+ None,
+ )
+
+ found: Final = eventually(look, lambda value: value is not None, seconds=seconds)
+ assert found is not None
+ return found
+
+
+def _trace_spans(sink_url: str, trace_id: str, seconds: float = 30) -> tuple[Span, ...]:
+ """The trace's spans once the post-call tail has landed.
+
+ The spend-writer and other post-response spans flush after the request
+ answers, so absence assertions poll for the whole window instead of
+ settling at the first glimpse of the root span.
+ """
+ deadline: Final = time.monotonic() + seconds
+ group: tuple[Span, ...] = () # rebind-ok: drains samples until the post-call tail lands
+ while time.monotonic() < deadline:
+ _, spans = recorded_spans(sink_url)
+ group = spans_for_trace(spans, trace_id)
+ time.sleep(0.5)
+ assert group, f"trace {trace_id} never reached {sink_url}"
+ return group
+
+
+def _await_db_span(sink_url: str, trace_id: str | None, needle: str, seconds: float = 40, since: int = 0) -> None:
+ def seen() -> bool:
+ _, spans = recorded_spans(sink_url, since)
+ group: Final = spans if trace_id is None else spans_for_trace(spans, trace_id)
+ return any(
+ needle in str(span["name"]) or needle in {str(span["attributes"].get(k)) for k in DB_SYSTEM_KEYS}
+ for span in group
+ )
+
+ landed: Final = eventually(seen, bool, seconds=seconds)
+ assert landed, f"{needle} span never landed at {sink_url}"
+
+
+def _db_systems(spans: tuple[Span, ...]) -> set[str]:
+ return {str(span["attributes"][key]) for span in spans for key in DB_SYSTEM_KEYS if key in span["attributes"]}
+
+
+def _assert_core_spans_present(spans: tuple[Span, ...]) -> None:
+ attributes_by_span: Final = tuple(span["attributes"] for span in spans)
+ assert any(span["kind"] == 2 for span in spans), "request root span missing"
+ assert any("gen_ai.operation.name" in attrs for attrs in attributes_by_span), "model span missing"
+ assert any("litellm.guardrail.name" in attrs for attrs in attributes_by_span), "guardrail span missing"
+ names: Final = sorted(str(span["name"]) for span in spans)
+ assert any(name.startswith("auth") for name in names), f"auth span missing in {names}"
+
+
+def _assert_tenant_keeps_redis_without_postgres(
+ candidate: Gateway, audit_sinks: SpanSinks, langfuse_vars: Mapping[str, JsonValue]
+) -> None:
+ tenant_start, _ = recorded_spans(audit_sinks.tenant)
+ operator_start, _ = recorded_spans(audit_sinks.operator)
+ traffic: Final = _drive(candidate, langfuse_vars)
+ _await_db_span(audit_sinks.operator, None, "batch_write_to_db", seconds=60, since=operator_start)
+ tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+ _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+ systems: Final = _db_systems(_trace_spans(audit_sinks.tenant, tenant_trace, seconds=15))
+ assert "redis" in systems, f"redis spans missing at tenant: {systems}"
+ _, all_tenant = recorded_spans(audit_sinks.tenant, tenant_start)
+ assert "postgresql" not in _db_systems(all_tenant), f"postgresql spans reached tenant: {_db_systems(all_tenant)}"
+
+
+def _guardrail_block(config: dict) -> None:
+ config["guardrails"] = [
+ {
+ "guardrail_name": f"excl-filter-{uuid.uuid4().hex[:8]}",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "default_on": True,
+ "patterns": [
+ {
+ "pattern_type": "regex",
+ "pattern_name": "excl_secret",
+ "pattern": "TOPSECRET\\d{9}",
+ "action": "BLOCK",
+ }
+ ],
+ },
+ }
+ ]
+
+
+@pytest.mark.timeout(180)
+def test_excluded_services_drops_db_spans_at_tenant_only(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(
+ tmp_path, otel_audit_config, otel={"excluded_services": ["redis", "postgres"]}, extra=_guardrail_block
+ )
+ with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+ ten_start, _ = recorded_spans(audit_sinks.tenant)
+ op_start, _ = recorded_spans(audit_sinks.operator)
+ traffic: Final = _drive(candidate, langfuse_vars)
+ _await_db_span(audit_sinks.operator, None, "postgresql", seconds=60, since=op_start)
+ tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+ tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace)
+ _assert_core_spans_present(tenant_spans)
+ assert _db_systems(tenant_spans) == set(), (
+ f"db spans reached tenant: {sorted(str(s['name']) for s in tenant_spans)}"
+ )
+ operator_trace: Final = _trace_id(audit_sinks.operator, traffic)
+ assert operator_trace == tenant_trace
+ trace_systems: Final = _db_systems(_trace_spans(audit_sinks.operator, operator_trace))
+ assert "redis" in trace_systems, f"operator trace lost redis spans: {trace_systems}"
+ _, all_operator = recorded_spans(audit_sinks.operator, op_start)
+ operator_systems: Final = _db_systems(all_operator)
+ assert "postgresql" in operator_systems, f"operator lost aux db spans: {operator_systems}"
+ _, all_tenant = recorded_spans(audit_sinks.tenant, ten_start)
+ names: Final = sorted(str(span["name"]) for span in all_tenant)
+ assert _db_systems(all_tenant) == set(), f"aux db spans reached tenant: {names}"
+ assert not any("batch_write_to_db" in name for name in names), f"spend writer reached tenant: {names}"
+
+
+@pytest.mark.timeout(180)
+def test_without_excluded_services_the_tenant_still_gets_redis_and_postgres_spans(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(tmp_path, otel_audit_config, extra=_guardrail_block)
+ with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+ tenant_start, _ = recorded_spans(audit_sinks.tenant)
+ traffic: Final = _drive(candidate, langfuse_vars)
+ _await_db_span(audit_sinks.tenant, None, "batch_write_to_db", seconds=60, since=tenant_start)
+ tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+ _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+ _assert_core_spans_present(_trace_spans(audit_sinks.tenant, tenant_trace, seconds=15))
+ _, all_tenant = recorded_spans(audit_sinks.tenant, tenant_start)
+ systems: Final = _db_systems(all_tenant)
+ assert {"redis", "postgresql"} <= systems, f"datastore spans missing at tenant: {systems}"
+
+
+def test_env_excluded_services_drops_only_redis(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(tmp_path, otel_audit_config)
+ with owned_proxy(
+ gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "redis"}, config=config, workers=2
+ ) as candidate:
+ start, _ = recorded_spans(audit_sinks.tenant)
+ _drive(candidate, langfuse_vars)
+ _await_db_span(audit_sinks.tenant, None, "postgresql", seconds=60, since=start)
+ _, tenant_spans = recorded_spans(audit_sinks.tenant, start)
+ systems: Final = _db_systems(tenant_spans)
+ assert "postgresql" in systems, f"postgresql spans missing at tenant: {systems}"
+ assert "redis" not in systems, f"redis spans reached tenant: {sorted(str(s['name']) for s in tenant_spans)}"
+
+
+@pytest.mark.timeout(180)
+def test_config_excluded_services_wins_over_env(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ def with_langfuse_otel(config: dict) -> None:
+ config["litellm_settings"]["callbacks"] = ["otel", "langfuse_otel"]
+
+ config: Final = _config_with(
+ tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=with_langfuse_otel
+ )
+ with owned_proxy(
+ gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "redis"}, config=config, workers=2
+ ) as candidate:
+ _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_excluded_services_applies_with_preset_ordered_first(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ def preset_first(config: dict) -> None:
+ config["litellm_settings"]["callbacks"] = ["langfuse_otel", "otel"]
+
+ config: Final = _config_with(
+ tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=preset_first
+ )
+ overrides: Final = {"LITELLM_OTEL_V2": "1", **_operator_langfuse(audit_sinks)}
+ with owned_proxy(gateway, tmp_path, overrides, config=config, workers=2) as candidate:
+ _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_service_logs_error_and_drops_at_proxy_start(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["auth", "postgres"]})
+ with owned_proxy_process(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as owned:
+ assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+ _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_valid_config_excluded_services_tolerates_bogus_env(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]})
+ with owned_proxy(
+ gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth"}, config=config, workers=2
+ ) as candidate:
+ _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_services_env_logs_and_drops_with_preset_alongside_otel(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ def with_langfuse_otel(config: dict) -> None:
+ config["litellm_settings"]["callbacks"] = ["otel", "langfuse_otel"]
+
+ config: Final = _config_with(tmp_path, otel_audit_config, extra=with_langfuse_otel)
+ overrides: Final = {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth,postgres"}
+ with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned:
+ assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+ _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_services_env_logs_and_drops_without_otel_callback(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ def presets_only(config: dict) -> None:
+ config["litellm_settings"]["callbacks"] = ["langfuse_otel"]
+
+ config: Final = _config_with(tmp_path, otel_audit_config, extra=presets_only)
+ overrides: Final = {
+ "LITELLM_OTEL_V2": "1",
+ "LITELLM_OTEL_EXCLUDED_SERVICES": "auth,postgres",
+ **_operator_langfuse(audit_sinks),
+ }
+ with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned:
+ assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+ _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+def test_postgres_exclusion_covers_batch_write_to_db(
+ gateway: Gateway,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]})
+ with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+ op_start, _ = recorded_spans(audit_sinks.operator)
+ ten_start, _ = recorded_spans(audit_sinks.tenant)
+ traffic: Final = _drive(candidate, langfuse_vars)
+ _await_db_span(audit_sinks.operator, None, "batch_write_to_db", seconds=60, since=op_start)
+ tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+ _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+ tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace, seconds=15)
+ _, all_tenant = recorded_spans(audit_sinks.tenant, ten_start)
+ names: Final = sorted(str(span["name"]) for span in all_tenant)
+ assert "redis" in _db_systems(tenant_spans), f"redis spans missing at tenant: {names}"
+ assert not any("batch_write_to_db" in name for name in names), f"spend writer reached tenant: {names}"
diff --git a/tests/integration/observability/test_otel_excluded_services_matrix.py b/tests/integration/observability/test_otel_excluded_services_matrix.py
new file mode 100644
index 00000000000..0d4b5d087c6
--- /dev/null
+++ b/tests/integration/observability/test_otel_excluded_services_matrix.py
@@ -0,0 +1,713 @@
+import asyncio
+import json
+import os
+import re
+import signal
+import uuid
+from collections.abc import Callable, Generator, Iterator, Mapping
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, Literal
+
+import anthropic
+import httpx
+import openai
+import psutil
+import pytest
+import yaml
+from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value
+from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_spans, spans_for_trace
+from integration._support.process import OwnedProxy, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+MARKER: Final = re.compile(rb"excl-[0-9a-f]{32}")
+FAILING: Final = re.compile(rb"excl-fail-[0-9a-f]{32}")
+JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+REPLY_TEXT: Final = "excluded ok"
+SERVER: Final = 2
+INVALID_NAME_LOG: Final = "is not a datastore service"
+INVALID_VALUE_LOG: Final = "excluded_services must be"
+Endpoint = Literal["chat", "responses", "messages"]
+Client = Literal["raw", "sdk", "async_sdk"]
+ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "responses", "messages")
+CLIENTS: Final[tuple[Client, ...]] = ("raw", "sdk", "async_sdk")
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+
+def _marker() -> str:
+ return "excl-" + uuid.uuid4().hex
+
+
+def _chat_reply(identity: str, stream: bool) -> Reply:
+ if not stream:
+ return Reply(
+ body=json.dumps(
+ {
+ "id": identity,
+ "object": "chat.completion",
+ "created": 1,
+ "model": "gpt-4o-mini",
+ "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": REPLY_TEXT}, "finish_reason": "stop"}
+ ],
+ "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9},
+ }
+ ).encode()
+ )
+ chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"}
+ first, _, rest = REPLY_TEXT.partition(" ")
+ deltas: Final[tuple[dict[str, JsonValue], ...]] = (
+ {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": first}}]},
+ {**chunk, "choices": [{"index": 0, "delta": {"content": " " + rest}, "finish_reason": "stop"}]},
+ {**chunk, "choices": [], "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}},
+ )
+ return Reply(
+ content_type="text/event-stream",
+ chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"),
+ )
+
+
+def _responses_reply(identity: str, stream: bool) -> Reply:
+ response: Final[dict[str, JsonValue]] = {
+ "id": identity,
+ "object": "response",
+ "created_at": 1,
+ "status": "completed",
+ "model": "gpt-4o-mini",
+ "output": [
+ {
+ "id": "msg_" + identity,
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": REPLY_TEXT, "annotations": []}],
+ }
+ ],
+ "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9},
+ }
+ if not stream:
+ return Reply(body=json.dumps(response).encode())
+ events: Final[tuple[dict[str, JsonValue], ...]] = (
+ {
+ "type": "response.created",
+ "sequence_number": 0,
+ "response": {**response, "status": "in_progress", "output": []},
+ },
+ {
+ "type": "response.output_text.delta",
+ "sequence_number": 1,
+ "item_id": "msg_" + identity,
+ "output_index": 0,
+ "content_index": 0,
+ "delta": REPLY_TEXT,
+ },
+ {"type": "response.completed", "sequence_number": 2, "response": response},
+ )
+ return Reply(
+ content_type="text/event-stream",
+ chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events),
+ )
+
+
+def _upstream(request: Request) -> Reply:
+ if FAILING.search(request.body) is not None:
+ return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}')
+ found: Final = MARKER.search(request.body)
+ if found is None:
+ return Reply(status=404, body=b'{"error":"no marker"}')
+ marker: Final = found.group(0).decode()
+ stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True
+ if request.target.endswith("/responses"):
+ return _responses_reply(f"resp_{marker}", stream)
+ return _chat_reply(f"chatcmpl-{marker}", stream)
+
+
+def _at(payload: JsonValue, *path: str | int) -> JsonValue:
+ if not path:
+ return payload
+ step: Final = path[0]
+ if isinstance(step, int):
+ assert isinstance(payload, list), payload
+ return _at(payload[step], *path[1:])
+ return _at(object_value(payload)[step], *path[1:])
+
+
+def _sse(body: str) -> tuple[JsonValue, ...]:
+ return tuple(
+ JSON.validate_json(line[6:])
+ for line in body.splitlines()
+ if line.startswith("data: ") and line != "data: [DONE]"
+ )
+
+
+def _raw_text(endpoint: Endpoint, stream: bool, body: str) -> str:
+ if not stream:
+ path: Final[tuple[str | int, ...]] = {
+ "chat": ("choices", 0, "message", "content"),
+ "responses": ("output", 0, "content", 0, "text"),
+ "messages": ("content", 0, "text"),
+ }[endpoint]
+ return str(_at(JSON.validate_json(body), *path))
+ events: Final = _sse(body)
+ if endpoint == "chat":
+ return "".join(
+ str(object_value(_at(event, "choices", 0, "delta")).get("content") or "")
+ for event in events
+ if _at(event, "choices")
+ )
+ if endpoint == "responses":
+ return "".join(
+ str(_at(event, "delta")) for event in events if _at(event, "type") == "response.output_text.delta"
+ )
+ return "".join(
+ str(_at(event, "delta", "text"))
+ for event in events
+ if _at(event, "type") == "content_block_delta" and _at(event, "delta", "type") == "text_delta"
+ )
+
+
+def _body(model: str, endpoint: Endpoint, marker: str, stream: bool) -> tuple[str, dict[str, JsonValue]]:
+ if endpoint == "chat":
+ return "/v1/chat/completions", {
+ "model": model,
+ "messages": [{"role": "user", "content": marker}],
+ "stream": stream,
+ }
+ if endpoint == "responses":
+ return "/v1/responses", {"model": model, "input": marker, "stream": stream}
+ return "/v1/messages", {
+ "model": model,
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": marker}],
+ "stream": stream,
+ }
+
+
+@dataclass(frozen=True, slots=True)
+class Sent:
+ call_id: str
+ text: str
+
+
+@dataclass(frozen=True, slots=True)
+class Cursors:
+ operator: int
+ tenant: int
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+ proxy: Gateway
+ owned: OwnedProxy
+ scenario: Scenario
+ model: str
+ key: str
+ upstream: Wire
+ sinks: SpanSinks
+
+ def cursors(self) -> Cursors:
+ self.upstream.drain()
+ return Cursors(recorded_spans(self.sinks.operator)[0], recorded_spans(self.sinks.tenant)[0])
+
+ def upstream_hits(self, marker: str) -> int:
+ return sum(1 for request in self.upstream.drain() if marker.encode() in request.body)
+
+ def base_url(self) -> str:
+ return str(self.proxy.client.base_url)
+
+ def raw(
+ self, endpoint: Endpoint, marker: str, stream: bool, key: str | None = None, trace_id: str | None = None
+ ) -> Sent:
+ path, body = _body(self.model, endpoint, marker, stream)
+ auth: Final = {"Authorization": f"Bearer {key or self.key}"}
+ parent: Final = {} if trace_id is None else {"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}
+ with self.proxy.client.stream("POST", path, json=body, headers={**auth, **parent}) as response:
+ text: Final = response.read().decode()
+ assert response.status_code == 200, text
+ return Sent(response.headers["x-litellm-call-id"], _raw_text(endpoint, stream, text))
+
+ def sdk(self, endpoint: Endpoint, marker: str, stream: bool) -> Sent:
+ if endpoint == "messages":
+ messages: Final = anthropic.Anthropic(base_url=self.base_url(), api_key=self.key, max_retries=0).messages
+ if not stream:
+ reply: Final = messages.with_raw_response.create(
+ model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}]
+ )
+ block: Final = reply.parse().content[0]
+ assert isinstance(block, anthropic.types.TextBlock), block
+ return Sent(reply.headers["x-litellm-call-id"], block.text)
+ with messages.with_streaming_response.create(
+ model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}], stream=True
+ ) as streamed:
+ return Sent(
+ streamed.headers["x-litellm-call-id"],
+ "".join(
+ event.delta.text
+ for event in streamed.parse()
+ if event.type == "content_block_delta" and event.delta.type == "text_delta"
+ ),
+ )
+ client: Final = openai.OpenAI(base_url=self.base_url() + "/v1", api_key=self.key, max_retries=0)
+ if endpoint == "chat":
+ if not stream:
+ completion: Final = client.chat.completions.with_raw_response.create(
+ model=self.model, messages=[{"role": "user", "content": marker}]
+ )
+ return Sent(
+ completion.headers["x-litellm-call-id"], completion.parse().choices[0].message.content or ""
+ )
+ with client.chat.completions.with_streaming_response.create(
+ model=self.model, messages=[{"role": "user", "content": marker}], stream=True
+ ) as chunks:
+ return Sent(
+ chunks.headers["x-litellm-call-id"],
+ "".join(chunk.choices[0].delta.content or "" for chunk in chunks.parse() if chunk.choices),
+ )
+ if not stream:
+ created: Final = client.responses.with_raw_response.create(model=self.model, input=marker)
+ return Sent(created.headers["x-litellm-call-id"], created.parse().output_text)
+ with client.responses.with_streaming_response.create(model=self.model, input=marker, stream=True) as events:
+ return Sent(
+ events.headers["x-litellm-call-id"],
+ "".join(event.delta for event in events.parse() if event.type == "response.output_text.delta"),
+ )
+
+ async def async_sdk(self, endpoint: Endpoint, marker: str, stream: bool) -> Sent:
+ if endpoint == "messages":
+ messages: Final = anthropic.AsyncAnthropic(
+ base_url=self.base_url(), api_key=self.key, max_retries=0
+ ).messages
+ if not stream:
+ reply: Final = await messages.with_raw_response.create(
+ model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}]
+ )
+ block: Final = reply.parse().content[0]
+ assert isinstance(block, anthropic.types.TextBlock), block
+ return Sent(reply.headers["x-litellm-call-id"], block.text)
+ async with messages.with_streaming_response.create(
+ model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}], stream=True
+ ) as streamed:
+ pieces: Final = [
+ event.delta.text
+ async for event in await streamed.parse()
+ if event.type == "content_block_delta" and event.delta.type == "text_delta"
+ ]
+ return Sent(streamed.headers["x-litellm-call-id"], "".join(pieces))
+ client: Final = openai.AsyncOpenAI(base_url=self.base_url() + "/v1", api_key=self.key, max_retries=0)
+ if endpoint == "chat":
+ if not stream:
+ completion: Final = await client.chat.completions.with_raw_response.create(
+ model=self.model, messages=[{"role": "user", "content": marker}]
+ )
+ return Sent(
+ completion.headers["x-litellm-call-id"], completion.parse().choices[0].message.content or ""
+ )
+ async with client.chat.completions.with_streaming_response.create(
+ model=self.model, messages=[{"role": "user", "content": marker}], stream=True
+ ) as chunks:
+ deltas: Final = [
+ chunk.choices[0].delta.content or "" async for chunk in await chunks.parse() if chunk.choices
+ ]
+ return Sent(chunks.headers["x-litellm-call-id"], "".join(deltas))
+ if not stream:
+ created: Final = await client.responses.with_raw_response.create(model=self.model, input=marker)
+ return Sent(created.headers["x-litellm-call-id"], created.parse().output_text)
+ async with client.responses.with_streaming_response.create(
+ model=self.model, input=marker, stream=True
+ ) as events:
+ texts: Final = [
+ event.delta async for event in await events.parse() if event.type == "response.output_text.delta"
+ ]
+ return Sent(events.headers["x-litellm-call-id"], "".join(texts))
+
+ def send(self, endpoint: Endpoint, client: Client, marker: str, stream: bool) -> Sent:
+ if client == "raw":
+ return self.raw(endpoint, marker, stream)
+ if client == "sdk":
+ return self.sdk(endpoint, marker, stream)
+ return asyncio.run(self.async_sdk(endpoint, marker, stream))
+
+
+def _db_systems(spans: tuple[Span, ...]) -> set[str]:
+ return {
+ str(system)
+ for span in spans
+ if (system := span["attributes"].get("db.system.name") or span["attributes"].get("db.system")) is not None
+ }
+
+
+def _names(spans: tuple[Span, ...]) -> list[str]:
+ return sorted(span["name"] for span in spans)
+
+
+def _has_root(spans: tuple[Span, ...]) -> bool:
+ return any(span["kind"] == SERVER for span in spans)
+
+
+def _trace_of_call(sink: str, call_id: str, since: int) -> tuple[Span, ...]:
+ _, spans = recorded_spans(sink, since)
+ traces: Final = {span["trace_id"] for span in spans if span["attributes"].get("litellm.call_id") == call_id}
+ return tuple(span for span in spans if span["trace_id"] in traces)
+
+
+def _operator_trace(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
+ trace: Final = eventually(
+ lambda: _trace_of_call(rig.sinks.operator, sent.call_id, cursors.operator),
+ lambda spans: _has_root(spans) and "redis" in _db_systems(spans),
+ seconds=40,
+ )
+ assert len({span["trace_id"] for span in trace}) == 1, _names(trace)
+ return trace
+
+
+def _traced_raw(rig: Rig, endpoint: Endpoint, marker: str) -> tuple[str, Sent]:
+ trace_id: Final = uuid.uuid4().hex
+ return trace_id, rig.raw(endpoint, marker, stream=False, trace_id=trace_id)
+
+
+def _operator_trace_by_id(rig: Rig, trace_id: str, cursors: Cursors) -> tuple[Span, ...]:
+ return eventually(
+ lambda: spans_for_trace(recorded_spans(rig.sinks.operator, cursors.operator)[1], trace_id),
+ _has_root,
+ seconds=40,
+ )
+
+
+def _tenant_mirror(rig: Rig, operator: tuple[Span, ...], cursors: Cursors) -> tuple[Span, ...]:
+ kept: Final = frozenset(span["name"] for span in operator if not _db_systems((span,)))
+ return eventually(
+ lambda: spans_for_trace(recorded_spans(rig.sinks.tenant, cursors.tenant)[1], operator[0]["trace_id"]),
+ lambda spans: kept <= {span["name"] for span in spans},
+ seconds=40,
+ )
+
+
+def _assert_tenant_mirrors(rig: Rig, operator: tuple[Span, ...], cursors: Cursors) -> tuple[Span, ...]:
+ tenant: Final = _tenant_mirror(rig, operator, cursors)
+ assert _db_systems(tenant) == set(), f"datastore spans reached the tenant: {_names(tenant)}"
+ assert sum(1 for span in tenant if span["kind"] == SERVER) == 1, _names(tenant)
+ return tenant
+
+
+def _assert_withheld(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
+ tenant: Final = _assert_tenant_mirrors(rig, _operator_trace(rig, sent, cursors), cursors)
+ assert any("gen_ai.operation.name" in span["attributes"] for span in tenant), _names(tenant)
+ return tenant
+
+
+def _config(directory: Path, otel_audit_config: AuditConfigWriter, otel: Mapping[str, JsonValue], name: str) -> Path:
+ written: Final = otel_audit_config(directory, {})
+ loaded: Final = object_value(JSON.validate_python(yaml.safe_load(written.read_text())))
+ settings: Final = object_value(loaded["callback_settings"])
+ config: Final = {**loaded, "callback_settings": {**settings, "otel": {**object_value(settings["otel"]), **otel}}}
+ path: Final = directory / f"{name}.yaml"
+ path.write_text(yaml.safe_dump(config))
+ return path
+
+
+@contextmanager
+def _started(
+ provider: Wire,
+ sinks: SpanSinks,
+ config: Path,
+ directory: Path,
+ langfuse_vars: Mapping[str, JsonValue],
+ workers: int,
+) -> Generator[Rig]:
+ with (
+ gateway_from_environment() as gateway,
+ owned_proxy_process(
+ gateway,
+ directory,
+ {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"},
+ config=config,
+ remove_environment=("LITELLM_OTEL_EXCLUDED_SERVICES",),
+ workers=workers,
+ ) as owned,
+ owned.gateway.scenario() as scenario,
+ ):
+ model: Final = scenario.model(api_base=provider.url + "/v1")
+ team: Final = scenario.team()
+ attached: Final = owned.gateway.request(
+ "POST", f"/team/{team}/callback", {"callback_name": "langfuse_otel", "callback_vars": dict(langfuse_vars)}
+ )
+ assert attached.status_code == 200, attached.text
+ yield Rig(owned.gateway, owned, scenario, model, scenario.key(team_id=team), provider, sinks)
+
+
+@pytest.fixture(scope="module")
+def provider() -> Iterator[Wire]:
+ with wire_server(_upstream) as wire:
+ yield wire
+
+
+@pytest.fixture(scope="module")
+def rig(
+ provider: Wire,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path_factory: pytest.TempPathFactory,
+) -> Iterator[Rig]:
+ directory: Final = tmp_path_factory.mktemp("excluded-matrix")
+ config: Final = _config(directory, otel_audit_config, {"excluded_services": ["redis", "postgres"]}, "matrix")
+ with _started(provider, audit_sinks, config, directory, langfuse_vars, workers=2) as started:
+ yield started
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("stream", [False, True], ids=["unary", "stream"])
+@pytest.mark.parametrize("client", CLIENTS)
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_tenant_trace_keeps_request_spans_without_datastore_spans(
+ rig: Rig, endpoint: Endpoint, client: Client, stream: bool
+) -> None:
+ cursors: Final = rig.cursors()
+ marker: Final = _marker()
+ sent: Final = rig.send(endpoint, client, marker, stream)
+ assert sent.text == REPLY_TEXT, sent
+ assert rig.upstream_hits(marker) == 1
+ _assert_withheld(rig, sent, cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("endpoint", ["chat", "messages"])
+def test_cache_hit_twin_keeps_datastore_spans_off_the_tenant(rig: Rig, endpoint: Endpoint) -> None:
+ marker: Final = _marker()
+ first: Final = rig.raw(endpoint, marker, stream=False)
+ assert first.text == REPLY_TEXT, first
+ assert rig.upstream_hits(marker) == 1
+ cursors: Final = rig.cursors()
+ trace_id, hit = eventually(
+ lambda: _traced_raw(rig, endpoint, marker), lambda sent: rig.upstream_hits(marker) == 0, seconds=20
+ )
+ assert hit.text == REPLY_TEXT, hit
+ _assert_tenant_mirrors(rig, _operator_trace_by_id(rig, trace_id, cursors), cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_failed_upstream_call_keeps_datastore_spans_off_the_tenant(rig: Rig, endpoint: Endpoint) -> None:
+ cursors: Final = rig.cursors()
+ marker: Final = "excl-fail-" + uuid.uuid4().hex
+ trace_id: Final = uuid.uuid4().hex
+ path, body = _body(rig.model, endpoint, marker, stream=False)
+ failed: Final = rig.proxy.client.post(
+ path,
+ json=body,
+ headers={"Authorization": f"Bearer {rig.key}", "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"},
+ )
+ assert failed.status_code == 500, failed.text
+ assert rig.upstream_hits(marker) >= 1
+ operator: Final = eventually(
+ lambda: spans_for_trace(recorded_spans(rig.sinks.operator, cursors.operator)[1], trace_id),
+ lambda spans: _has_root(spans) and "redis" in _db_systems(spans),
+ seconds=40,
+ )
+ _assert_tenant_mirrors(rig, operator, cursors)
+
+
+@pytest.mark.timeout(120)
+def test_key_level_callback_vars_destination_is_filtered_too(rig: Rig, langfuse_vars: dict[str, JsonValue]) -> None:
+ key: Final = rig.scenario.key(
+ metadata={
+ "logging": [
+ {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": dict(langfuse_vars)}
+ ]
+ }
+ )
+ cursors: Final = rig.cursors()
+ marker: Final = _marker()
+ sent: Final = rig.raw("chat", marker, stream=False, key=key)
+ assert sent.text == REPLY_TEXT, sent
+ assert rig.upstream_hits(marker) == 1
+ _assert_withheld(rig, sent, cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("status", [403, 404])
+def test_rejecting_tenant_destination_leaves_serving_and_the_operator_trace_intact(rig: Rig, status: int) -> None:
+ configure_sink(rig.sinks.tenant, status=status)
+ try:
+ cursors: Final = rig.cursors()
+ marker: Final = _marker()
+ sent: Final = rig.raw("chat", marker, stream=True)
+ assert sent.text == REPLY_TEXT, sent
+ assert rig.upstream_hits(marker) == 1
+ _assert_withheld(rig, sent, cursors)
+ finally:
+ configure_sink(rig.sinks.tenant, status=200)
+ after: Final = rig.cursors()
+ _assert_withheld(rig, rig.raw("responses", _marker(), stream=False), after)
+
+
+def _burst(rig: Rig, count: int) -> tuple[Sent | str, ...]:
+ def one(index: int) -> Sent | str:
+ try:
+ return rig.raw(ENDPOINTS[index % 3], _marker(), stream=index % 2 == 0)
+ except (httpx.HTTPError, AssertionError) as error:
+ return repr(error)
+
+ with ThreadPoolExecutor(max_workers=10) as pool:
+ return tuple(pool.map(one, range(count)))
+
+
+def _served(results: tuple[Sent | str, ...]) -> tuple[Sent, ...]:
+ return tuple(result for result in results if isinstance(result, Sent))
+
+
+def _assert_operator_exactly_once(rig: Rig, served: tuple[Sent, ...], cursors: Cursors) -> set[str]:
+ wanted: Final = {sent.call_id for sent in served}
+
+ def roots() -> dict[str, int]:
+ _, spans = recorded_spans(rig.sinks.operator, cursors.operator)
+ traced: Final = {
+ span["trace_id"]: str(span["attributes"]["litellm.call_id"])
+ for span in spans
+ if span["attributes"].get("litellm.call_id") in wanted
+ }
+ counts: Final = {call: 0 for call in wanted}
+ for span in spans:
+ if span["kind"] == SERVER and span["trace_id"] in traced:
+ counts[traced[span["trace_id"]]] += 1
+ return counts
+
+ landed: Final = eventually(roots, lambda counts: all(count >= 1 for count in counts.values()), seconds=90)
+ assert landed == {call: 1 for call in wanted}, landed
+ _, spans = recorded_spans(rig.sinks.operator, cursors.operator)
+ return {span["trace_id"] for span in spans if span["attributes"].get("litellm.call_id") in wanted}
+
+
+def _assert_tenant_never_saw_datastore_spans(rig: Rig, cursors: Cursors, traces: set[str]) -> None:
+ tenant: Final = eventually(
+ lambda: recorded_spans(rig.sinks.tenant, cursors.tenant)[1],
+ lambda spans: traces <= {span["trace_id"] for span in spans if span["kind"] == SERVER},
+ seconds=90,
+ )
+ assert _db_systems(tenant) == set(), _names(tenant)
+
+
+@pytest.mark.timeout(300)
+def test_tenant_outage_during_a_mixed_burst_keeps_serving_and_never_leaks_datastore_spans(rig: Rig) -> None:
+ cursors: Final = rig.cursors()
+ configure_sink(rig.sinks.tenant, status=503)
+ try:
+ results: Final = _burst(rig, 30)
+ finally:
+ configure_sink(rig.sinks.tenant, status=200)
+ served: Final = _served(results)
+ assert len(served) == 30, [result for result in results if isinstance(result, str)]
+ assert all(sent.text == REPLY_TEXT for sent in served), served
+ traces: Final = _assert_operator_exactly_once(rig, served, cursors)
+ _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+ after: Final = rig.cursors()
+ _assert_withheld(rig, rig.raw("messages", _marker(), stream=True), after)
+
+
+@pytest.mark.timeout(300)
+def test_stalled_tenant_destination_during_a_burst_does_not_block_responses(rig: Rig) -> None:
+ cursors: Final = rig.cursors()
+ configure_sink(rig.sinks.tenant, paused=True)
+ try:
+ results: Final = _burst(rig, 20)
+ finally:
+ configure_sink(rig.sinks.tenant, paused=False)
+ served: Final = _served(results)
+ assert len(served) == 20, [result for result in results if isinstance(result, str)]
+ traces: Final = _assert_operator_exactly_once(rig, served, cursors)
+ _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+
+
+@pytest.mark.timeout(300)
+def test_killing_one_of_two_workers_mid_burst_keeps_the_filter_on_the_survivor(rig: Rig) -> None:
+ root: Final = psutil.Process(rig.owned.process.pid)
+ workers: Final = eventually(
+ lambda: tuple(child for child in root.children() if "resource_tracker" not in " ".join(child.cmdline())),
+ lambda found: len(found) == 2,
+ seconds=30,
+ )
+ cursors: Final = rig.cursors()
+
+ def one(index: int) -> Sent | str:
+ if index == 6:
+ os.kill(workers[0].pid, signal.SIGKILL)
+ try:
+ return rig.raw("chat", _marker(), stream=index % 2 == 0)
+ except (httpx.HTTPError, AssertionError) as error:
+ return repr(error)
+
+ with ThreadPoolExecutor(max_workers=6) as pool:
+ results: Final = tuple(pool.map(one, range(18)))
+ assert rig.owned.process.poll() is None, "Proxy root exited after a worker was killed"
+ failures: Final = tuple(result for result in results if isinstance(result, str))
+ assert all(failure.startswith(("ReadError(", "RemoteProtocolError(", "ConnectError(")) for failure in failures), (
+ failures
+ )
+ assert len(failures) <= 6, failures
+ settled: Final = tuple(result for index, result in enumerate(results) if index > 12 and isinstance(result, Sent))
+ traces: Final = _assert_operator_exactly_once(rig, settled, cursors)
+ _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+ after: Final = rig.cursors()
+ _assert_withheld(rig, rig.raw("chat", _marker(), stream=False), after)
+
+
+@dataclass(frozen=True, slots=True)
+class Setting:
+ otel: Mapping[str, JsonValue]
+ withholds_redis: bool
+ logs: str | None
+
+
+SETTINGS: Final[dict[str, Setting]] = {
+ "missing": Setting({}, False, None),
+ "null": Setting({"excluded_services": None}, False, None),
+ "empty_list": Setting({"excluded_services": []}, False, None),
+ "empty_string": Setting({"excluded_services": ""}, False, None),
+ "yaml_string": Setting({"excluded_services": "redis"}, True, None),
+ "duplicates": Setting({"excluded_services": ["redis", "redis"]}, True, None),
+ "case_and_space": Setting({"excluded_services": ["REDIS", " Postgres "]}, True, None),
+ "integer": Setting({"excluded_services": 7}, False, INVALID_VALUE_LOG),
+ "mapping": Setting({"excluded_services": {"redis": True}}, False, INVALID_VALUE_LOG),
+ "non_string_item": Setting({"excluded_services": [7, "redis"]}, True, INVALID_VALUE_LOG),
+ "oversized_name": Setting({"excluded_services": "x" * 5000}, False, INVALID_NAME_LOG),
+}
+
+
+@pytest.mark.timeout(180)
+@pytest.mark.parametrize("name", SETTINGS)
+def test_excluded_services_setting_shapes_boot_and_resolve(
+ name: str,
+ provider: Wire,
+ audit_sinks: SpanSinks,
+ otel_audit_config: AuditConfigWriter,
+ langfuse_vars: dict[str, JsonValue],
+ tmp_path: Path,
+) -> None:
+ setting: Final = SETTINGS[name]
+ config: Final = _config(tmp_path, otel_audit_config, setting.otel, name)
+ with _started(provider, audit_sinks, config, tmp_path, langfuse_vars, workers=1) as started:
+ cursors: Final = started.cursors()
+ marker: Final = _marker()
+ sent: Final = started.raw("chat", marker, stream=False)
+ assert sent.text == REPLY_TEXT, sent
+ assert started.upstream_hits(marker) == 1
+ operator: Final = _operator_trace(started, sent, cursors)
+ tenant: Final = _tenant_mirror(started, operator, cursors)
+ if setting.withholds_redis:
+ assert "redis" not in _db_systems(tenant), _names(tenant)
+ else:
+ eventually(
+ lambda: _db_systems(
+ spans_for_trace(recorded_spans(started.sinks.tenant, cursors.tenant)[1], tenant[0]["trace_id"])
+ ),
+ lambda systems: "redis" in systems,
+ seconds=30,
+ )
+ log: Final = started.owned.log.read_text()
+ if setting.logs is None:
+ assert INVALID_NAME_LOG not in log and INVALID_VALUE_LOG not in log, log[-2000:]
+ else:
+ assert setting.logs in log, log[-4000:]
diff --git a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py
index dcaff3c911a..86837f7f46c 100644
--- a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py
+++ b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py
@@ -11,6 +11,7 @@
"""
import asyncio
+import logging
import pytest
@@ -22,17 +23,18 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E4
)
from litellm.integrations.otel import LiteLLM, OpenTelemetryV2Config # noqa: E402
-from litellm.integrations.otel.plumbing import providers # noqa: E402
+from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402
from litellm.integrations.otel.model.baggage import ( # noqa: E402
BAGGAGE_PROMOTED_KEYS,
DEFAULT_BAGGAGE_METADATA_KEYS,
)
-from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402
+from litellm.integrations.otel.model.config import excluded_db_systems_from # noqa: E402
from litellm.integrations.otel.model.payloads import GuardrailSpanData # noqa: E402
from litellm.integrations.otel.model.spans import ( # noqa: E402
LITELLM_PROXY_REQUEST_SPAN_NAME,
SpanRole,
)
+from litellm.integrations.otel.plumbing import providers # noqa: E402
# --------------------------------------------------------------------------- #
# Area 1 — baggage allowlists configurable
@@ -74,13 +76,11 @@ def test_baggage_keys_from_config_yaml_kwargs():
def test_baggage_processor_allowlist_uses_config_keys():
- cfg = OpenTelemetryV2Config(
- exporter="in_memory", baggage_promoted_keys=[LiteLLM.TEAM_ID]
- )
+ cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_promoted_keys=[LiteLLM.TEAM_ID])
provider, exporter = providers.in_memory_provider(cfg)
- from litellm.integrations.otel.plumbing import context as ctx_mod
from litellm.integrations.otel.emitter import SpanEmitter
from litellm.integrations.otel.model.payloads import ServiceSpanData
+ from litellm.integrations.otel.plumbing import context as ctx_mod
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
ctx = ctx_mod.set_request_baggage({LiteLLM.TEAM_ID: "t1", LiteLLM.TEAM_ALIAS: "ta"})
@@ -90,6 +90,68 @@ def test_baggage_processor_allowlist_uses_config_keys():
assert LiteLLM.TEAM_ALIAS not in span.attributes # not in this allowlist
+@pytest.mark.parametrize(
+ "given,expected",
+ [
+ (["redis"], frozenset({"redis"})),
+ (["postgres"], frozenset({"postgresql"})),
+ (["postgresql"], frozenset({"postgresql"})),
+ (["batch_write_to_db"], frozenset({"postgresql"})),
+ (["redis_spend_update_queue"], frozenset({"redis"})),
+ (["redis", "postgres"], frozenset({"redis", "postgresql"})),
+ ],
+)
+def test_excluded_services_normalize_to_db_system_names(given, expected):
+ assert OpenTelemetryV2Config(excluded_services=given).excluded_services == expected
+
+
+def test_excluded_services_from_env_csv(monkeypatch):
+ monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis, postgres")
+ assert OpenTelemetryV2Config().excluded_services == frozenset({"redis", "postgresql"})
+
+
+def test_excluded_services_config_wins_over_env(monkeypatch):
+ monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis")
+ assert OpenTelemetryV2Config(excluded_services=["postgres"]).excluded_services == frozenset({"postgresql"})
+
+
+def test_excluded_services_drops_a_non_datastore_service_and_logs(caplog):
+ with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+ config = OpenTelemetryV2Config(excluded_services=["auth", "redis"])
+ assert config.excluded_services == frozenset({"redis"})
+ assert any("'auth' is not a datastore service; ignored" in record.message for record in caplog.records)
+
+
+def test_excluded_services_env_drops_a_bad_value_and_logs(monkeypatch, caplog):
+ monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "auth,postgres")
+ with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+ config = OpenTelemetryV2Config()
+ assert config.excluded_services == frozenset({"postgresql"})
+ assert any("'auth' is not a datastore service; ignored" in record.message for record in caplog.records)
+
+
+@pytest.mark.parametrize(
+ "given,expected,logged",
+ [
+ (None, frozenset(), None),
+ ("", frozenset(), None),
+ ([], frozenset(), None),
+ (["REDIS", " Postgres "], frozenset({"redis", "postgresql"}), None),
+ (7, frozenset(), "excluded_services must be a list or comma-separated string; 7 ignored"),
+ ({"redis": True}, frozenset(), "excluded_services must be a list or comma-separated string"),
+ ([7, "redis"], frozenset({"redis"}), "excluded_services must be a list of service names; 7 ignored"),
+ ],
+)
+def test_malformed_excluded_services_logs_and_still_builds_the_config(given, expected, logged, caplog):
+ with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+ config = OpenTelemetryV2Config(excluded_services=given)
+ resolved = excluded_db_systems_from(given)
+ assert config.excluded_services == expected
+ assert resolved == expected
+ messages = [record.message for record in caplog.records]
+ assert (logged is None and messages == []) or any(logged in message for message in messages), messages
+
+
# --------------------------------------------------------------------------- #
# Area 2 — pass-through LLM span parents to the ambient server span
# --------------------------------------------------------------------------- #
@@ -124,9 +186,7 @@ def test_passthrough_llm_span_parents_to_ambient_server_span():
later (possibly detached) success callback only closes the already-parented
span, so it never becomes a separate root trace."""
logger, exporter = _logger()
- server = logger._emitter.start_span(
- SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
- )
+ server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
kwargs = {
"standard_logging_object": _payload(),
"litellm_params": {"metadata": {}},
@@ -150,9 +210,7 @@ def test_llm_span_unaffected_by_phase_span_active_at_close():
successor to the old auth-failure-401 case where the LLM log nested under
``auth``: the span is now born after auth, parented to the request root."""
logger, exporter = _logger()
- server = logger._emitter.start_span(
- SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
- )
+ server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
kwargs = {
"standard_logging_object": _payload(),
"litellm_params": {"metadata": {}},
diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py
index 9cb3dbb9deb..5a7057203e4 100644
--- a/tests/unit/integrations/otel/test_otel_v2_destinations.py
+++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py
@@ -514,6 +514,53 @@ class TestFanOut:
for child in ("auth /v1/chat/completions", "chat gpt-4"):
assert by_name[child].parent.span_id == root.context.span_id
+ def test_excluded_services_drop_only_the_datastore_spans_at_the_tenant(self):
+ """The exclusion is per ``db.system.*`` value: a span naming an excluded
+ datastore never reaches the tenant, while every span of the request's
+ own work (root, auth, guardrail, model) still does, and the operator's
+ own exporter keeps the full tree."""
+ dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(operator_exporter))
+ provider.add_span_processor(
+ TenantFanOutSpanProcessor(
+ processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter),
+ excluded_db_systems=frozenset({"redis", "postgresql"}),
+ )
+ )
+ tracer = get_tracer(provider, "litellm")
+
+ def run():
+ set_request_destinations((LANGFUSE_DEST,))
+ with tracer.start_as_current_span("POST /v1/chat/completions"):
+ with tracer.start_as_current_span("auth /v1/chat/completions"):
+ pass
+ with tracer.start_as_current_span("execute_guardrail pii"):
+ pass
+ with tracer.start_as_current_span("redis async_get_cache") as redis_span:
+ redis_span.set_attribute("db.system.name", "redis")
+ with tracer.start_as_current_span("batch_write_to_db _PROXY_track_cost_callback") as spend_span:
+ spend_span.set_attribute("db.system", "postgresql")
+ with tracer.start_as_current_span("chat gpt-4"):
+ pass
+
+ in_fresh_context(run)
+
+ assert {s.name for s in dest_exporter.get_finished_spans()} == {
+ "POST /v1/chat/completions",
+ "auth /v1/chat/completions",
+ "execute_guardrail pii",
+ "chat gpt-4",
+ }
+ assert {s.name for s in operator_exporter.get_finished_spans()} == {
+ "POST /v1/chat/completions",
+ "auth /v1/chat/completions",
+ "execute_guardrail pii",
+ "redis async_get_cache",
+ "batch_write_to_db _PROXY_track_cost_callback",
+ "chat gpt-4",
+ }
+
def test_a_team_naming_two_backends_gets_the_trace_at_both(self):
"""The fan-out rides one provider, so it cannot skip a destination on the
grounds that some other backend owns it: nothing else would deliver it."""
@@ -1022,6 +1069,89 @@ class TestProviderWiring:
assert kinds(published).count("TenantFanOutSpanProcessor") == 1
assert "TenantFanOutSpanProcessor" not in kinds(other)
+ @staticmethod
+ def _fan_out_of(logger: OpenTelemetryV2) -> TenantFanOutSpanProcessor:
+ return next(
+ processor
+ for processor in logger._tracer_provider._active_span_processor._span_processors
+ if isinstance(processor, TenantFanOutSpanProcessor)
+ )
+
+ def test_callback_settings_excluded_services_win_over_the_published_preset_env_config(self, monkeypatch):
+ """A preset builds its config env-only, so the fan-out must read
+ ``callback_settings.otel.excluded_services`` itself rather than the
+ published logger's config, or the env value would win."""
+ monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+ preset = OpenTelemetryV2(
+ config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")], excluded_services=["redis"]),
+ callback_name="langfuse_otel",
+ )
+
+ publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+ assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+
+ def test_callback_settings_excluded_services_apply_even_when_other_otel_env_vars_are_malformed(self, monkeypatch):
+ """Reading the setting must not rebuild the whole settings model, or an unrelated bad env
+ value the operator overrode in config would stop publication before the fan-out is attached"""
+ preset = OpenTelemetryV2(
+ config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")]),
+ callback_name="langfuse_otel",
+ )
+ monkeypatch.setenv("LITELLM_OTEL_LEGACY_COMPAT", "not-a-bool")
+ monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+
+ publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+ assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+
+ def test_excluded_services_fall_back_to_the_published_logger_config_without_callback_settings(self, monkeypatch):
+ monkeypatch.setattr(litellm, "callback_settings", {"otel": {"exporter": "in_memory"}}, raising=False)
+ preset = OpenTelemetryV2(
+ config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")], excluded_services=["redis"]),
+ callback_name="langfuse_otel",
+ )
+
+ publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+ assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"redis"})
+
+ def test_otel_after_a_preset_reuses_it_and_still_takes_callback_settings_exclusions(self, monkeypatch):
+ """``callbacks: [langfuse_otel, otel]`` keeps one v2 logger, exactly as
+ before ``excluded_services`` existed, and the exclusion still comes from
+ ``callback_settings.otel`` rather than the preset's env-only config."""
+ from litellm.litellm_core_utils import litellm_logging as logging_module
+
+ logging_module._in_memory_loggers.clear()
+ monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+ monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk")
+ monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk")
+ monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis")
+ is_otel_v2_enabled.cache_clear()
+ monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+ try:
+
+ def init(name: str) -> CustomLogger | None:
+ return logging_module._init_custom_logger_compatible_class(
+ logging_integration=name, # pyright: ignore[reportArgumentType] # test passes a literal callback name
+ internal_usage_cache=None,
+ llm_router=None,
+ custom_logger_init_args={},
+ )
+
+ preset = init("langfuse_otel")
+ otel_cb = init("otel")
+
+ assert isinstance(preset, OpenTelemetryV2)
+ assert otel_cb is preset
+ v2_loggers = [cb for cb in logging_module._in_memory_loggers if isinstance(cb, OpenTelemetryV2)]
+ assert v2_loggers == [preset], v2_loggers
+ publish_global_otel_v2_provider(logging_module._in_memory_loggers, lambda _p: None, registered=preset)
+ assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+ finally:
+ logging_module._in_memory_loggers.clear()
+ is_otel_v2_enabled.cache_clear()
+
@pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"])
def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical):
monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive")
diff --git a/ui/litellm-dashboard/public/assets/agent-traces-preview.png b/ui/litellm-dashboard/public/assets/agent-traces-preview.png
new file mode 100644
index 00000000000..34569e26331
Binary files /dev/null and b/ui/litellm-dashboard/public/assets/agent-traces-preview.png differ
diff --git a/ui/litellm-dashboard/public/assets/logos/crewai-color.svg b/ui/litellm-dashboard/public/assets/logos/crewai-color.svg
new file mode 100644
index 00000000000..95cb17f9364
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/crewai-color.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/langchain.svg b/ui/litellm-dashboard/public/assets/logos/langchain.svg
new file mode 100644
index 00000000000..939b79989a7
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/langchain.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/langgraph-color.svg b/ui/litellm-dashboard/public/assets/logos/langgraph-color.svg
new file mode 100644
index 00000000000..14f16e3cd1d
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/langgraph-color.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/llamaindex-color.svg b/ui/litellm-dashboard/public/assets/logos/llamaindex-color.svg
new file mode 100644
index 00000000000..99be517874e
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/llamaindex-color.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/openai-agents.svg b/ui/litellm-dashboard/public/assets/logos/openai-agents.svg
new file mode 100644
index 00000000000..78caf4fa20f
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/openai-agents.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/opentelemetry.svg b/ui/litellm-dashboard/public/assets/logos/opentelemetry.svg
new file mode 100644
index 00000000000..606165cf788
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/opentelemetry.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/public/assets/logos/pydantic-ai-color.svg b/ui/litellm-dashboard/public/assets/logos/pydantic-ai-color.svg
new file mode 100644
index 00000000000..85827432f0c
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/pydantic-ai-color.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
index 3b52a2eac33..cc497677a1a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx
@@ -23,7 +23,9 @@ vi.mock("@/components/DashboardHeader", () => ({
}));
vi.mock("@/app/(dashboard)/components/SidebarProvider", () => ({
- default: () =>
,
+ default: ({ sidebarCollapsed }: { sidebarCollapsed: boolean }) => (
+
+ ),
}));
vi.mock("@/components/DebugWarningBanner", () => ({
@@ -112,6 +114,27 @@ describe("(dashboard) Layout", () => {
},
);
+ it("collapses the sidebar on Logs for a full-screen view and expands it again after leaving", async () => {
+ const dashboard = () => (
+
+
+
+
+
+ );
+ const { rerender } = render(dashboard());
+ pendingUiConfig.resolve();
+ expect(await screen.findByTestId("sidebar")).toHaveAttribute("data-collapsed", "false");
+
+ vi.mocked(usePathname).mockReturnValue("/ui/logs");
+ rerender(dashboard());
+ expect(screen.getByTestId("sidebar")).toHaveAttribute("data-collapsed", "true");
+
+ vi.mocked(usePathname).mockReturnValue("/ui/api-keys");
+ rerender(dashboard());
+ expect(screen.getByTestId("sidebar")).toHaveAttribute("data-collapsed", "false");
+ });
+
it("does not mount route content until getUiConfig has resolved", async () => {
render(
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
index 406a323fbfb..72f26919060 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx
@@ -99,11 +99,18 @@ export function AgentControlPlaneView() {
);
}
+const FULL_BLEED_SEGMENTS = new Set(["logs"]);
+
function DashboardShell({ children }: { children: React.ReactNode }) {
const { accessToken } = useAuth();
- const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
const { mode } = usePluginMode();
- const isPlayground = routeSegmentForPathname(usePathname()) === "playground";
+ const routeSegment = routeSegmentForPathname(usePathname());
+ const isPlayground = routeSegment === "playground";
+ const isFullBleed = FULL_BLEED_SEGMENTS.has(routeSegment);
+ // A manual toggle holds only for the route it was made on; full-bleed routes default to collapsed.
+ const [sidebarOverride, setSidebarOverride] = useState<{ segment: string; collapsed: boolean } | null>(null);
+ const sidebarCollapsed = sidebarOverride?.segment === routeSegment ? sidebarOverride.collapsed : isFullBleed;
+ const toggleSidebar = () => setSidebarOverride({ segment: routeSegment, collapsed: !sidebarCollapsed });
const isGateway = mode === "ai-gateway";
@@ -133,7 +140,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
// so the page can't be dragged past the end of the nav.
return (
-
setSidebarCollapsed((v) => !v)} />
+
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index 42dc5350b49..d0a5349364f 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -115,6 +115,7 @@ import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity
import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets";
import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab";
import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard";
+import type { SpanDetail, Trace, TracePage } from "./view_logs/TraceView/traceTypes";
import {
createApiClient,
deriveErrorMessage,
@@ -2101,6 +2102,33 @@ export const uiSpendLogsCall = async ({
}
};
+/**
+ * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not
+ * configured; callers can detect that through the thrown `ApiError`'s `status`.
+ */
+export const agentTraceListCall = async ({
+ accessToken,
+ startMs,
+ endMs,
+ cursor,
+}: {
+ accessToken: string;
+ startMs: number;
+ endMs: number;
+ cursor?: string | null;
+}): Promise
=> {
+ const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined };
+ return apiClient.get(`/v1/traces`, { accessToken, query });
+};
+
+export const agentTraceCall = async (accessToken: string, traceId: string): Promise =>
+ apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { accessToken });
+
+export const agentTraceSpanCall = async (accessToken: string, traceId: string, spanId: string): Promise =>
+ apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, {
+ accessToken,
+ });
+
export const adminSpendLogsCall = async (accessToken: string) => {
try {
const data = await apiClient.get(`/global/spend/logs`, { accessToken });
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx
new file mode 100644
index 00000000000..5b506cadbb0
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx
@@ -0,0 +1,42 @@
+"use client";
+
+import moment from "moment";
+import { useMemo, useState } from "react";
+
+import { AgentTracesSection } from "./AgentTracesSection";
+
+const DEFAULT_RANGE_HOURS = 24;
+const TIME_FORMAT = "YYYY-MM-DDTHH:mm";
+
+/** Agent Traces: agent runs (developer view). Lives as the "Agent Traces" tab of the Logs page. */
+export default function AgentTracesPage({ accessToken }: { accessToken: string }) {
+ const [rangeHours, setRangeHours] = useState(DEFAULT_RANGE_HOURS);
+ const [live, setLive] = useState(true);
+ const [anchor, setAnchor] = useState(() => moment());
+ const { startTime, endTime } = useMemo(
+ () => ({
+ startTime: anchor.clone().subtract(rangeHours, "hours").format(TIME_FORMAT),
+ endTime: anchor.format(TIME_FORMAT),
+ }),
+ [anchor, rangeHours],
+ );
+
+ const changeRange = (hours: number) => {
+ setRangeHours(hours);
+ setAnchor(moment());
+ };
+
+ return (
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx
new file mode 100644
index 00000000000..6d28e8a53e9
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx
@@ -0,0 +1,265 @@
+import { fireEvent, screen, within } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { ApiError } from "@/lib/http/client";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import traceList from "./__fixtures__/trace_list.json";
+import AgentTracesPage from "./AgentTracesPage";
+import { AgentTracesSection, filterRuns } from "./AgentTracesSection";
+import type { TracePage, TraceSummary } from "./traceTypes";
+
+vi.mock("../../networking", () => ({
+ agentTraceListCall: vi.fn(),
+ agentTraceCall: vi.fn(),
+ agentTraceSpanCall: vi.fn(),
+ getProxyBaseUrl: () => "http://localhost:4000",
+}));
+
+vi.mock("./TraceDrawer", () => ({
+ RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => (
+
+ run {traceId}
+
+
+ ),
+}));
+
+import { agentTraceListCall } from "../../networking";
+
+const runs = (traceList as TracePage).data as TraceSummary[];
+
+const renderSection = () =>
+ renderWithProviders(
+ ,
+ );
+
+// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone.
+const renderWindowed = () =>
+ renderWithProviders(
+ ,
+ );
+
+const bucketRunCounts = () =>
+ screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs")));
+
+describe("AgentTracesSection", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ beforeEach(() => {
+ vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({
+ left: 0,
+ width: 600,
+ top: 0,
+ height: 56,
+ right: 600,
+ bottom: 56,
+ x: 0,
+ y: 0,
+ toJSON: () => ({}),
+ } as DOMRect);
+ testQueryClient.clear();
+ vi.mocked(agentTraceListCall).mockReset();
+ });
+
+ it("renders the setup snippet when the proxy answers 501", async () => {
+ vi.mocked(agentTraceListCall).mockRejectedValue(
+ new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }),
+ );
+ renderSection();
+
+ const card = await screen.findByTestId("tracing-setup-card");
+ expect(card).toHaveTextContent("Tracing is not enabled");
+ expect(card).toHaveTextContent("store: clickhouse");
+ expect(card).toHaveTextContent("OTEL_EXPORTER_OTLP_ENDPOINT=");
+ expect(card).not.toHaveTextContent(/langsmith/i);
+ expect(card).toHaveTextContent('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"');
+ expect(card).toHaveTextContent("Let Claude Code or Codex set it up");
+ });
+
+ it("shows the waiting guide when tracing is on but no runs have arrived", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] });
+ renderSection();
+
+ const card = await screen.findByTestId("tracing-setup-card");
+ expect(card).toHaveTextContent("Waiting for traces");
+ expect(card).toHaveTextContent("No traces detected yet");
+ expect(card).not.toHaveTextContent("store: clickhouse");
+ });
+
+ it("treats a proxy without the trace routes (404) like tracing being off", async () => {
+ vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" }));
+ renderSection();
+
+ const card = await screen.findByTestId("tracing-setup-card");
+ expect(card).toHaveTextContent("Tracing is not enabled");
+ expect(card).toHaveTextContent("CLICKHOUSE_READER_URL");
+ });
+
+ it("lists every run with its input, counts and failed column", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderSection();
+
+ const rows = await screen.findAllByTestId("agent-trace-row");
+ expect(rows).toHaveLength(runs.length);
+ const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans"));
+ expect(lead).toBeDefined();
+ const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement;
+ expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument();
+ expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument();
+ expect(screen.queryByRole("columnheader", { name: "Cost" })).not.toBeInTheDocument();
+ });
+
+ it("filters by input text and by trace id", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderSection();
+ await screen.findAllByTestId("agent-trace-row");
+
+ const search = screen.getByLabelText("Search runs");
+ fireEvent.change(search, { target: { value: "acme-404" } });
+ expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1);
+
+ const lead = runs.find((r) => r.name === "research_lead") as TraceSummary;
+ fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } });
+ const rows = screen.getAllByTestId("agent-trace-row");
+ expect(rows).toHaveLength(1);
+ expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans");
+ });
+
+ it("status filter 'Failed' keeps only runs with errors", () => {
+ const failed = filterRuns(runs, "", "all", "error");
+ expect(failed.length).toBeGreaterThan(0);
+ expect(failed.every((r) => r.error_count > 0)).toBe(true);
+ const ok = filterRuns(runs, "", "all", "ok");
+ expect(ok.every((r) => r.error_count === 0)).toBe(true);
+ expect(failed.length + ok.length).toBe(runs.length);
+ });
+
+ it("opens the run in place and goes back to the list", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderSection();
+ const rows = await screen.findAllByTestId("agent-trace-row");
+
+ fireEvent.click(rows[0]);
+ expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`);
+ expect(screen.queryByTestId("runs-table")).not.toBeInTheDocument();
+
+ fireEvent.click(screen.getByText("back"));
+ expect(screen.getByTestId("runs-table")).toBeInTheDocument();
+ });
+
+ it("plots every loaded run on the timeline", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderWindowed();
+ await screen.findAllByTestId("agent-trace-row");
+
+ expect(screen.getByTestId("traces-timeline")).toBeInTheDocument();
+ const counts = bucketRunCounts();
+ expect(counts).toHaveLength(60);
+ expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length);
+ });
+
+ it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderWindowed();
+ await screen.findAllByTestId("agent-trace-row");
+ const area = screen.getByTestId("timeline-area");
+ const x = (bucket: number) => bucket * 10 + 5;
+ const drag = (target: HTMLElement, from: number, to: number) => {
+ fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 });
+ fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 });
+ fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 });
+ };
+ const rowCount = () => screen.queryAllByTestId("agent-trace-row").length;
+ const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : []));
+ const first = withRuns[0];
+ // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run.
+ expect(first).toBeGreaterThan(1);
+ expect(first).toBeLessThan(30);
+
+ drag(area, 0, 1);
+ expect(screen.getByTestId("timeline-selection")).toBeInTheDocument();
+ expect(rowCount()).toBe(0);
+
+ drag(screen.getByTestId("timeline-handle-hi"), 1, first);
+ expect(rowCount()).toBeGreaterThan(0);
+
+ drag(screen.getByTestId("timeline-selection"), 1, 1 - first);
+ expect(rowCount()).toBeGreaterThan(0);
+ drag(screen.getByTestId("timeline-selection"), 0, 59);
+ expect(rowCount()).toBe(0);
+
+ fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" });
+ expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument();
+ expect(rowCount()).toBe(runs.length);
+ });
+});
+
+describe("AgentTracesPage", () => {
+ beforeEach(() => {
+ testQueryClient.clear();
+ vi.mocked(agentTraceListCall).mockReset();
+ });
+
+ it("shows the actual range, switches presets from the popover, and toggles Live", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderWithProviders();
+ await screen.findByTestId("runs-table");
+
+ const trigger = screen.getByRole("button", { name: "Time range" });
+ expect(trigger).toHaveTextContent(/ to /);
+ expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d");
+
+ fireEvent.click(trigger);
+ fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" }));
+ expect(await screen.findByText("Total 7d")).toBeInTheDocument();
+ const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0];
+ expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000);
+
+ const live = screen.getByRole("button", { name: "Live" });
+ expect(live).toHaveAttribute("aria-pressed", "true");
+ fireEvent.click(live);
+ expect(live).toHaveAttribute("aria-pressed", "false");
+ expect(screen.getByRole("button", { name: "Reset zoom" })).toBeDisabled();
+ });
+
+ it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderWithProviders();
+ await screen.findByTestId("runs-table");
+
+ vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] });
+ fireEvent.click(screen.getByRole("button", { name: "Time range" }));
+ fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" }));
+
+ expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument();
+ expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument();
+ });
+
+ it("asks the proxy for the last 24 hours by default", async () => {
+ vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+ renderWithProviders();
+ await screen.findByTestId("runs-table");
+
+ const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0];
+ expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000);
+ expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000);
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx
new file mode 100644
index 00000000000..2bfa3c320a7
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx
@@ -0,0 +1,185 @@
+"use client";
+
+import moment from "moment";
+import { useMemo, useState } from "react";
+
+import { AgentTracesTable } from "./AgentTracesTable";
+import { ALL_SERVICES, RunsToolbar, type RunStatusFilter } from "./RunsToolbar";
+import { RunView } from "./TraceDrawer";
+import type { TraceSummary } from "./traceTypes";
+import { previewText } from "./traceUtils";
+import { TimeRangeControls } from "./TimeRangeControls";
+import { TracesTimeline, type TimeWindow } from "./TracesTimeline";
+import { TracingSetupCard } from "./TracingSetupCard";
+import { traceWindowStartMs, useAgentTraces } from "./useAgentTraces";
+
+/** Client-side search (input text or trace id) plus service / status filters over the loaded runs. */
+export function filterRuns(
+ runs: TraceSummary[],
+ query: string,
+ service: string,
+ status: RunStatusFilter,
+): TraceSummary[] {
+ const q = query.trim().toLowerCase();
+ return runs.filter((run) => {
+ const haystack = [run.trace_id, previewText(run.input_preview), run.name].map((s) => s.toLowerCase());
+ const matchesQuery = !q || haystack.some((text) => text.includes(q));
+ const matchesService = service === ALL_SERVICES || run.service === service;
+ const failed = run.error_count > 0;
+ const matchesStatus = status === "all" || (status === "error" ? failed : !failed);
+ return matchesQuery && matchesService && matchesStatus;
+ });
+}
+
+const filterByWindow = (runs: TraceSummary[], range: TimeWindow): TraceSummary[] =>
+ runs.filter((run) => {
+ const t = moment(run.start_time).valueOf();
+ return t >= range.startMs && t < range.endMs;
+ });
+
+export interface TimeControls {
+ rangeHours: number;
+ onRangeHoursChange: (hours: number) => void;
+ onLiveChange: (live: boolean) => void;
+}
+
+interface AgentTracesSectionProps {
+ accessToken: string;
+ isActive: boolean;
+ startTime: string;
+ endTime: string;
+ isCustomDate: boolean;
+ isLiveTail: boolean;
+ /** Page-owned time range + live state; when given, the toolbar shows the range / Live control group. */
+ timeControls?: TimeControls;
+ /** Called when a run opens / closes, so the page can hide its own header while a run fills the view. */
+ onRunOpenChange?: (open: boolean) => void;
+}
+
+/** The Runs view: filters, the runs table and footer — or one run, in place, once a row is clicked. */
+export function AgentTracesSection({
+ accessToken,
+ isActive,
+ startTime,
+ endTime,
+ isCustomDate,
+ isLiveTail,
+ timeControls,
+ onRunOpenChange,
+}: AgentTracesSectionProps) {
+ const [openTraceId, setOpenTraceId] = useState(null);
+ const [query, setQuery] = useState("");
+ const [service, setService] = useState(ALL_SERVICES);
+ const [status, setStatus] = useState("all");
+ const [showSetup, setShowSetup] = useState(false);
+ const [zoom, setZoom] = useState(null);
+ const [rangeChanged, setRangeChanged] = useState(false);
+ const traceQuery = { accessToken, startTime, endTime, isCustomDate, isLiveTail, enabled: isActive };
+ const traces = useAgentTraces(traceQuery);
+
+ const services = useMemo(() => Array.from(new Set(traces.traces.map((t) => t.service))).sort(), [traces.traces]);
+ // Relative ranges end "now" (the list query uses Date.now() too); round to the minute so the histogram is stable.
+ const endMs = isCustomDate ? moment(endTime).valueOf() : moment().endOf("minute").valueOf();
+ const range = useMemo(
+ () => ({ startMs: traceWindowStartMs(startTime, endTime, isCustomDate, endMs), endMs }),
+ [startTime, endTime, isCustomDate, endMs],
+ );
+ const filtered = useMemo(
+ () => filterRuns(traces.traces, query, service, status),
+ [traces.traces, query, service, status],
+ );
+ const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
+
+ const changeRange = (hours: number, apply: (hours: number) => void) => {
+ setZoom(null);
+ setRangeChanged(true);
+ apply(hours);
+ };
+
+ const openRun = (traceId: string | null) => {
+ setOpenTraceId(traceId);
+ onRunOpenChange?.(traceId !== null);
+ };
+
+ if (traces.notEnabledDetail !== null) return ;
+ // Onboarding only on the first, default view; an empty range the user picked keeps its controls.
+ const isEmpty = !traces.isLoading && !traces.error && traces.traces.length === 0;
+ if (isEmpty && !rangeChanged) return ;
+ if (showSetup) {
+ return (
+
+
+
+
+ );
+ }
+
+ if (openTraceId !== null) {
+ return openRun(null)} />;
+ }
+
+ return (
+
+
+
+ {timeControls && (
+ changeRange(hours, timeControls.onRangeHoursChange)}
+ live={isLiveTail}
+ onLiveChange={timeControls.onLiveChange}
+ zoomed={zoom !== null}
+ onResetZoom={() => setZoom(null)}
+ />
+ )}
+
+
+
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx
new file mode 100644
index 00000000000..e4faff1621a
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx
@@ -0,0 +1,148 @@
+"use client";
+
+import { ArrowDown, ChevronRight } from "lucide-react";
+
+import { Button } from "@/components/ui/button";
+
+import { StatusMark } from "./StatusMark";
+import type { TraceSummary } from "./traceTypes";
+import { fmtMs, previewText, traceDisplayName } from "./traceUtils";
+
+interface AgentTracesTableProps {
+ traces: TraceSummary[];
+ isLoading: boolean;
+ error: Error | null;
+ hasMore: boolean;
+ onLoadMore: () => void;
+ onOpenTrace: (traceId: string) => void;
+}
+
+/** Spend is only on summaries once the spend-enrichment PR lands; show Cost when it's there. */
+type SummaryWithSpend = TraceSummary & { spend?: number };
+
+const SECOND_MS = 1000;
+const MINUTE_S = 60;
+const HOUR_M = 60;
+const DAY_H = 24;
+
+export function relativeTime(iso: string, now: number = Date.now()): string {
+ const diffS = Math.round((now - new Date(iso).getTime()) / SECOND_MS);
+ if (diffS < 5) return "just now";
+ if (diffS < MINUTE_S) return `${diffS}s ago`;
+ const diffM = Math.round(diffS / MINUTE_S);
+ if (diffM < HOUR_M) return `${diffM}m ago`;
+ const diffH = Math.round(diffM / HOUR_M);
+ if (diffH < DAY_H) return `${diffH}h ago`;
+ return `${Math.round(diffH / DAY_H)}d ago`;
+}
+
+export const formatCost = (cost: number): string => {
+ if (cost === 0) return "$0.00";
+ if (cost < 0.01) return `$${cost.toFixed(4)}`;
+ return `$${cost.toFixed(2)}`;
+};
+
+const firstLine = (text: string): string => text.split("\n")[0] ?? text;
+
+const TH = "px-3 font-medium";
+const TH_NUM = "px-3 text-right font-medium";
+const TD_NUM = "px-3 text-right font-mono tabular-nums text-muted-foreground";
+
+/** Devtool-dense runs list: one row per agent run, newest first. */
+export function AgentTracesTable({
+ traces,
+ isLoading,
+ error,
+ hasMore,
+ onLoadMore,
+ onOpenTrace,
+}: AgentTracesTableProps) {
+ const showCost = traces.some((t) => typeof (t as SummaryWithSpend).spend === "number");
+ const isEmpty = !isLoading && !error && traces.length === 0;
+ return (
+
+
+
+
+ |
+
+ Time
+
+ |
+ Service |
+ Input |
+ Agents |
+ Steps |
+ Duration |
+ {showCost && Cost | }
+ Failed |
+ |
+
+
+
+ {traces.map((run) => (
+ onOpenTrace(run.trace_id)}
+ className="h-9 cursor-pointer border-b border-border/60 text-[12px] hover:bg-accent/50"
+ >
+ |
+ {relativeTime(run.start_time)}
+ |
+
+ {run.service}
+ |
+
+
+ 0 ? "error" : "ok"} subtle />
+
+ {firstLine(previewText(run.input_preview)) || traceDisplayName(run)}
+
+
+ {run.trace_id}
+
+
+ |
+ {run.agent_count.toLocaleString()} |
+ {run.span_count.toLocaleString()} |
+ {fmtMs(run.duration_ms)} |
+ {showCost && (
+
+ {formatCost((run as SummaryWithSpend).spend ?? 0)}
+ |
+ )}
+
+ {run.error_count > 0 ? (
+
+ ) : (
+ 0
+ )}
+ |
+
+
+ |
+
+ ))}
+
+
+ {isLoading &&
Loading runs…
}
+ {error && (
+
Could not load runs: {error.message}
+ )}
+ {isEmpty && (
+
No runs match these filters.
+ )}
+ {hasMore && (
+
+
+
+ )}
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AttributesDetail.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AttributesDetail.tsx
new file mode 100644
index 00000000000..9001b1470a2
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AttributesDetail.tsx
@@ -0,0 +1,33 @@
+"use client";
+
+import type { Span } from "./traceTypes";
+
+interface AttributesDetailProps {
+ traceId: string;
+ span: Span;
+ attributes: Record | undefined;
+ isLoading: boolean;
+}
+
+/** Raw OTEL attributes as a key / value grid, ids first. */
+export function AttributesDetail({ traceId, span, attributes, isLoading }: AttributesDetailProps) {
+ const entries: [string, string][] = [
+ ["trace_id", traceId],
+ ["span_id", span.span_id],
+ ["parent_span_id", span.parent_span_id ?? "—"],
+ ...Object.entries(attributes ?? {}).sort(([a], [b]) => a.localeCompare(b)),
+ ];
+ return (
+
+
+ {entries.map(([key, value]) => (
+
+ ))}
+
+ {isLoading &&
Loading attributes…
}
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/CopyButton.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/CopyButton.tsx
new file mode 100644
index 00000000000..60dda4ce72d
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/CopyButton.tsx
@@ -0,0 +1,63 @@
+"use client";
+
+import { Check, Copy } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+const COPIED_RESET_MS = 1600;
+
+interface CopyButtonProps {
+ value: string;
+ label?: string;
+ copiedLabel?: string;
+ iconOnly?: boolean;
+ className?: string;
+}
+
+/** Copy → check for a moment. `iconOnly` renders a bare icon button with a tooltip. */
+export function CopyButton({
+ value,
+ label = "Copy",
+ copiedLabel = "Copied",
+ iconOnly = false,
+ className,
+}: CopyButtonProps) {
+ const [copied, setCopied] = useState(false);
+
+ useEffect(() => {
+ if (!copied) return;
+ const timeout = window.setTimeout(() => setCopied(false), COPIED_RESET_MS);
+ return () => window.clearTimeout(timeout);
+ }, [copied]);
+
+ const button = (
+
+ );
+
+ if (!iconOnly) return button;
+ return (
+
+
+
+ {copied ? copiedLabel : label}
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx
new file mode 100644
index 00000000000..d60805e8f45
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx
@@ -0,0 +1,152 @@
+"use client";
+
+import { useQuery, type UseQueryOptions } from "@tanstack/react-query";
+import { AlertTriangle, Bot, CornerDownRight, Wrench } from "lucide-react";
+
+import { agentTraceSpanCall } from "../../networking";
+import type { ErrorSource } from "./traceTree";
+import type { Span, SpanDetail, TraceMessage } from "./traceTypes";
+import { errorSource, parseMessages, prettyPayload } from "./traceUtils";
+
+const ERROR_SOURCE_LABEL: Record = { tool: "Tool", model: "Model", litellm: "LiteLLM" };
+const TRACEBACK_MARKER = "Traceback (most recent call last):";
+
+/** LangSmith records `repr(exc)` + traceback with no separator; keep the exception line. */
+export const errorHeadline = (error: string): string =>
+ (error.split(TRACEBACK_MARKER, 1)[0].split("\n")[0] ?? "").trim() || error.trim();
+
+/** `ValueError('x not found')` → "ValueError"; plain text → "error". */
+const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(headline)?.[1] ?? "error";
+
+/** Shared lazy fetch of one span's full input / output / attributes. */
+export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null) {
+ const queryOptions: UseQueryOptions = {
+ queryKey: ["agentTraceSpan", traceId, spanId, accessToken],
+ queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string),
+ enabled: spanId !== null,
+ staleTime: Infinity,
+ };
+ return useQuery(queryOptions);
+}
+
+export function SectionLabel({ children }: { children: React.ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+export function TextBlock({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) {
+ return (
+
+ {label}
+
+ {value}
+
+
+ );
+}
+
+function RoleIcon({ role }: { role: string }) {
+ if (role === "assistant") return ;
+ if (role === "tool") return ;
+ return ;
+}
+
+export function MessageBlock({ message }: { message: TraceMessage }) {
+ return (
+
+
+
+ {message.role}
+ {message.name ? · {message.name} : null}
+
+ {(message.tool_calls ?? []).map((call, i) => (
+
+ {call.name}
+ (
+ {JSON.stringify(call.args)}
+ )
+
+ ))}
+ {message.content && (
+
+ {message.content}
+
+ )}
+
+ );
+}
+
+export function ErrorBlock({ span }: { span: Span }) {
+ const source = errorSource(span);
+ if (!source) return null;
+ const headline = errorHeadline(span.error ?? "") || "Span reported an error status.";
+ return (
+
+
+
+ {ERROR_SOURCE_LABEL[source]} · {errorReason(headline)}
+
+
+ {headline}
+
+
+ );
+}
+
+function Payload({ label, value, mono }: { label: string; value: string; mono: boolean }) {
+ const messages = parseMessages(value);
+ if (messages) {
+ return (
+ <>
+ {`${label}${messages.length > 1 ? ` · ${messages.length} messages` : ""}`}
+ {messages.map((message, i) => (
+
+ ))}
+ >
+ );
+ }
+ return ;
+}
+
+interface DetailContentProps {
+ accessToken: string;
+ traceId: string;
+ span: Span;
+}
+
+/** Content tab: the error first (if any), then what went in and what came out. */
+export function DetailContent({ accessToken, traceId, span }: DetailContentProps) {
+ const detailQuery = useSpanDetail(accessToken, traceId, span.span_id);
+ const detail = detailQuery.data;
+ const isTool = span.type === "tool";
+ const empty = detail && !detail.input && !detail.output;
+
+ return (
+
+
+ {detailQuery.isLoading &&
Loading span…
}
+ {detailQuery.isError && (
+
+ Could not load span: {detailQuery.error.message}
+
+ )}
+ {detail?.input ?
: null}
+ {detail?.output ?
: null}
+ {empty && span.status !== "error" && (
+
+ No content recorded for this span.
+
+ )}
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx
new file mode 100644
index 00000000000..a15584e6d41
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx
@@ -0,0 +1,191 @@
+import { screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import { DetailPane } from "./DetailPane";
+import type { GroupRowData, SpanRowData } from "./traceTree";
+import type { Span, SpanDetail, Trace } from "./traceTypes";
+
+vi.mock("../../networking", () => ({
+ agentTraceSpanCall: vi.fn(),
+ getProxyBaseUrl: () => "http://proxy.test/",
+}));
+
+import { agentTraceSpanCall } from "../../networking";
+
+const span = (overrides: Partial & Pick): Span => ({
+ parent_span_id: "root",
+ name: overrides.span_id,
+ type: "chain",
+ agent: "support_triage_agent",
+ start_offset_ms: 0,
+ duration_ms: 1300,
+ status: "ok",
+ error: null,
+ input_preview: "",
+ model: null,
+ input_tokens: 0,
+ output_tokens: 0,
+ litellm_request_id: null,
+ ...overrides,
+});
+
+const rootFields: SpanFields = { span_id: "root", parent_span_id: null, name: "support_triage_agent", type: "agent" };
+const llmFields: SpanFields = {
+ span_id: "llm1",
+ name: "ChatOpenAI",
+ type: "llm",
+ model: "claude-sonnet-4-5",
+ input_tokens: 659,
+ output_tokens: 60,
+ litellm_request_id: "chatcmpl-abc",
+};
+const failedToolFields: SpanFields = {
+ span_id: "tool1",
+ name: "get_customer_plan",
+ type: "tool",
+ status: "error",
+ error:
+ "ValueError('customer acme-404 not found in billing DB')Traceback (most recent call last):\n File \"x.py\", line 1",
+};
+const root = span(rootFields);
+const llm = span(llmFields);
+const failedTool = span(failedToolFields);
+
+const trace: Trace = {
+ summary: {
+ trace_id: "t1",
+ name: "support_triage_agent",
+ service: "research-agent",
+ input_preview: '[{"role": "user", "content": "Customer acme-404 says billing is wrong."}]',
+ start_time: "2026-09-30T06:43:52.928000+00:00",
+ duration_ms: 1310,
+ status: "ok",
+ span_count: 3,
+ agent_count: 1,
+ agent_invocations: 1,
+ llm_calls: 1,
+ tool_calls: 1,
+ error_count: 1,
+ input_tokens: 659,
+ output_tokens: 60,
+ models: ["claude-sonnet-4-5"],
+ } as Trace["summary"],
+ agents: [],
+ spans: [root, llm, failedTool],
+};
+
+const details: Record = {
+ llm1: {
+ span_id: "llm1",
+ input: JSON.stringify([
+ { role: "system", content: "You are a LiteLLM support agent." },
+ { role: "user", content: "Customer acme-404 says billing is wrong." },
+ ]),
+ output: JSON.stringify({
+ role: "assistant",
+ content: "",
+ tool_calls: [{ name: "get_customer_plan", args: { customer_id: "acme-404" } }],
+ }),
+ attributes: { "gen_ai.request.model": "claude-sonnet-4-5" },
+ },
+ tool1: { span_id: "tool1", input: '{"customer_id":"acme-404"}', output: "", attributes: {} },
+ root: {
+ span_id: "root",
+ input: JSON.stringify([{ role: "user", content: "Customer acme-404 says billing is wrong." }]),
+ output: JSON.stringify({ role: "assistant", content: "Customer acme-404 is on the Enterprise plan." }),
+ attributes: {},
+ },
+};
+
+const spanRow = (s: Span): SpanRowData => ({
+ kind: "span",
+ id: s.span_id,
+ span: s,
+ depth: 1,
+ hasChildren: false,
+ collapsed: false,
+});
+
+const renderPane = (row: SpanRowData | GroupRowData) =>
+ renderWithProviders();
+
+describe("DetailPane", () => {
+ beforeEach(() => {
+ testQueryClient.clear();
+ vi.mocked(agentTraceSpanCall).mockReset();
+ vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, spanId) => details[spanId]);
+ });
+
+ it("renders the span tabs and the fetched LLM conversation with its tool call", async () => {
+ renderPane(spanRow(llm));
+ expect(screen.getByRole("tab", { name: "Content" })).toBeInTheDocument();
+ expect(screen.getByRole("tab", { name: "Request" })).toBeInTheDocument();
+ expect(screen.getByRole("tab", { name: "Attributes" })).toBeInTheDocument();
+ expect(await screen.findByText("You are a LiteLLM support agent.")).toBeInTheDocument();
+ expect(screen.getByText("get_customer_plan")).toBeInTheDocument();
+ expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1");
+ });
+
+ it("shows a tool failure as 'Tool · ' with the exception line and no traceback", async () => {
+ renderPane(spanRow(failedTool));
+ const error = screen.getByRole("region", { name: "Error" });
+ expect(error).toHaveTextContent("Tool · ValueError");
+ expect(error).toHaveTextContent("ValueError('customer acme-404 not found in billing DB')");
+ expect(error).not.toHaveTextContent("Traceback");
+ // tool args render as pretty JSON under "Input"
+ expect(await screen.findByText(/"customer_id": "acme-404"/)).toBeInTheDocument();
+ });
+
+ it("shows the LiteLLM request facts on the Request tab", async () => {
+ const user = userEvent.setup();
+ renderPane(spanRow(llm));
+ await user.click(screen.getByRole("tab", { name: "Request" }));
+ expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument();
+ expect(screen.getByText("659")).toBeInTheDocument();
+ expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument();
+ });
+
+ it("summarizes a ×N group with its failure pattern", () => {
+ const members = Array.from({ length: 12 }, (_, i) => {
+ const timedOut: SpanFields = {
+ span_id: `f${i}`,
+ name: "lookup_benchmark",
+ type: "tool",
+ status: "error",
+ error: "TimeoutError('slow')",
+ };
+ return span(timedOut);
+ });
+ const groupRow: GroupRowData = {
+ kind: "group",
+ id: "grp",
+ depth: 1,
+ name: "lookup_benchmark",
+ type: "tool",
+ agent: "researcher",
+ members,
+ failedCount: 12,
+ p50Duration: 640,
+ isFailureGroup: true,
+ expanded: false,
+ };
+ renderPane(groupRow);
+ const pane = screen.getByRole("complementary", { name: "Group details" });
+ expect(pane).toHaveTextContent("lookup_benchmark ×12");
+ expect(pane).toHaveTextContent("Invocations12");
+ expect(pane).toHaveTextContent("Failed12");
+ expect(pane).toHaveTextContent("TimeoutError('slow')");
+ });
+
+ it("'Copy step' copies a curl for just this span as Markdown", async () => {
+ const user = userEvent.setup();
+ const writeText = vi.fn().mockResolvedValue(undefined);
+ Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true });
+ renderPane(spanRow(llm));
+ await user.click(screen.getByRole("button", { name: "Copy step" }));
+ await waitFor(() => expect(writeText).toHaveBeenCalled());
+ expect(writeText.mock.calls[0][0]).toContain("http://proxy.test/v1/traces/t1?format=md&span_id=llm1");
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx
new file mode 100644
index 00000000000..0a4dd3c5029
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx
@@ -0,0 +1,188 @@
+"use client";
+
+import { PanelRightClose } from "lucide-react";
+import { useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { cn } from "@/lib/cva.config";
+
+import { AttributesDetail } from "./AttributesDetail";
+import { CopyButton } from "./CopyButton";
+import { DetailContent, errorHeadline, useSpanDetail } from "./DetailContent";
+import { RequestDetail } from "./RequestDetail";
+import { agentHandoffText } from "./TraceDrawer";
+import type { GroupRowData, TreeRow } from "./traceTree";
+import type { Span, Trace } from "./traceTypes";
+import { fmtMs, fmtTok } from "./traceUtils";
+
+interface DetailPaneProps {
+ trace: Trace;
+ row: TreeRow | undefined;
+ accessToken: string;
+ onClose: () => void;
+}
+
+type Tab = "content" | "request" | "attributes";
+
+const TABS: { id: Tab; label: string }[] = [
+ { id: "content", label: "Content" },
+ { id: "request", label: "Request" },
+ { id: "attributes", label: "Attributes" },
+];
+
+function PaneHeader({ children, onClose }: { children: React.ReactNode; onClose: () => void }) {
+ return (
+
+ );
+}
+
+function PaneFooter({ children }: { children: React.ReactNode }) {
+ return {children}
;
+}
+
+function Meta({ label, value }: { label: string; value: string }) {
+ return (
+
+ {label}=
+ {value}
+
+ );
+}
+
+function SpanPane({
+ trace,
+ span,
+ accessToken,
+ onClose,
+}: {
+ trace: Trace;
+ span: Span;
+ accessToken: string;
+ onClose: () => void;
+}) {
+ const [tab, setTab] = useState("content");
+ const traceId = trace.summary.trace_id;
+ const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null);
+ const tokens = span.input_tokens + span.output_tokens;
+ return (
+
+ );
+}
+
+function GroupMetric({ label, value }: { label: string; value: string }) {
+ return (
+
+ );
+}
+
+/** ×N group: rollup of every invocation plus the first failure's message. */
+function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; onClose: () => void }) {
+ const tokens = row.members.reduce((sum, m) => sum + m.input_tokens + m.output_tokens, 0);
+ const firstFailure = row.members.find((m) => m.status === "error" && m.error);
+ return (
+
+ );
+}
+
+/** Right pane of the run view: switches on the selected tree row. */
+export function DetailPane({ trace, row, accessToken, onClose }: DetailPaneProps) {
+ if (!row || row.kind === "load-more") {
+ return (
+
+ Select a span to inspect it.
+
+ );
+ }
+ if (row.kind === "group") return ;
+ return ;
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DurationBar.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DurationBar.tsx
new file mode 100644
index 00000000000..205712f4eec
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DurationBar.tsx
@@ -0,0 +1,27 @@
+import { cn } from "@/lib/cva.config";
+
+interface DurationBarProps {
+ startMs: number;
+ durationMs: number;
+ totalMs: number;
+ error?: boolean;
+ className?: string;
+}
+
+/** Waterfall bar: a hairline track with the span's slice of the run's timeline. */
+export function DurationBar({ startMs, durationMs, totalMs, error = false, className }: DurationBarProps) {
+ const left = totalMs > 0 ? Math.max(0, Math.min(98, (startMs / totalMs) * 100)) : 0;
+ const width = totalMs > 0 ? Math.max(1.5, Math.min(100 - left, (durationMs / totalMs) * 100)) : 1.5;
+ return (
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/RequestDetail.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/RequestDetail.tsx
new file mode 100644
index 00000000000..156393e6b37
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/RequestDetail.tsx
@@ -0,0 +1,91 @@
+"use client";
+
+import { ArrowUpRight } from "lucide-react";
+import { useState } from "react";
+
+import { Button } from "@/components/ui/button";
+
+import { LogDetailsDrawer } from "../LogDetailsDrawer";
+import { CopyButton } from "./CopyButton";
+import type { Span } from "./traceTypes";
+import { fmtMs, fmtTok } from "./traceUtils";
+import { useSpanRequestLog } from "./useSpanRequestLog";
+
+interface RequestDetailProps {
+ span: Span;
+ accessToken: string;
+ /** The run's start, so the request log is looked up at the span's time, whatever range the tabs show. */
+ traceStartMs: number;
+}
+
+/** Request tab for LLM spans; "Open request log" opens the LiteLLM request drawer over the run. */
+export function RequestDetail({ span, accessToken, traceStartMs }: RequestDetailProps) {
+ const [drawerOpen, setDrawerOpen] = useState(false);
+ const spanStartMs = traceStartMs + span.start_offset_ms;
+ const logQuery = useSpanRequestLog(accessToken, span.litellm_request_id, spanStartMs, drawerOpen);
+ const lookupDone = drawerOpen && logQuery.isSuccess;
+ const logNotFound = lookupDone && logQuery.data === null;
+
+ if (span.type !== "llm") {
+ return (
+
+ This span is not a model request.
+
+ );
+ }
+
+ const rows: [string, string][] = [
+ ["Model", span.model ?? "—"],
+ ["Input tokens", fmtTok(span.input_tokens)],
+ ["Output tokens", fmtTok(span.output_tokens)],
+ ["Total tokens", fmtTok(span.input_tokens + span.output_tokens)],
+ ["Latency", fmtMs(span.duration_ms)],
+ ];
+
+ return (
+
+
+ {rows.map(([label, value]) => (
+
+
-
+ {label}
+
+ -
+ {value}
+
+
+ ))}
+
+
+
+ Request ID
+ {span.litellm_request_id && (
+
+ )}
+
+
+ {span.litellm_request_id ?? "Not linked to a LiteLLM request"}
+
+
+ {span.litellm_request_id && (
+
+ )}
+ {logNotFound && (
+
No request log found for this call.
+ )}
+
setDrawerOpen(false)}
+ logEntry={logQuery.data ?? null}
+ accessToken={accessToken}
+ />
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/RunsToolbar.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/RunsToolbar.tsx
new file mode 100644
index 00000000000..f09e729752c
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/RunsToolbar.tsx
@@ -0,0 +1,92 @@
+"use client";
+
+import { Search } from "lucide-react";
+
+import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+
+export type RunStatusFilter = "all" | "ok" | "error";
+
+export const ALL_SERVICES = "all";
+
+interface RunsToolbarProps {
+ query: string;
+ service: string;
+ status: RunStatusFilter;
+ services: string[];
+ onQueryChange: (value: string) => void;
+ onServiceChange: (value: string) => void;
+ onStatusChange: (value: RunStatusFilter) => void;
+ /** Extra controls (time range, live tail) rendered on the right. */
+ children?: React.ReactNode;
+}
+
+const STATUS_ITEMS: { value: RunStatusFilter; label: string }[] = [
+ { value: "all", label: "All status" },
+ { value: "ok", label: "Succeeded" },
+ { value: "error", label: "Failed" },
+];
+
+/** Search + service / status filters for the Runs table. Filtering is client-side over the loaded page. */
+export function RunsToolbar({
+ query,
+ service,
+ status,
+ services,
+ onQueryChange,
+ onServiceChange,
+ onStatusChange,
+ children,
+}: RunsToolbarProps) {
+ const serviceItems = [
+ { value: ALL_SERVICES, label: "All services" },
+ ...services.map((s) => ({ value: s, label: s })),
+ ];
+ return (
+
+
+
+ onQueryChange(e.target.value)}
+ placeholder="Search input or trace ID"
+ aria-label="Search runs"
+ className="h-7 pl-8 text-[12px]"
+ />
+
+
+
+ {children &&
{children}
}
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx
new file mode 100644
index 00000000000..52affffded6
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx
@@ -0,0 +1,267 @@
+"use client";
+
+import {
+ Bot,
+ BrainCircuit,
+ ChevronDown,
+ ChevronRight,
+ CircleDot,
+ CornerDownRight,
+ MoreHorizontal,
+ Network,
+ Wrench,
+} from "lucide-react";
+import { useEffect, useRef } from "react";
+
+import { Switch } from "@/components/ui/switch";
+import { cn } from "@/lib/cva.config";
+
+import { DurationBar } from "./DurationBar";
+import type { TreeRow } from "./traceTree";
+import type { SpanType } from "./traceTypes";
+import { fmtMs } from "./traceUtils";
+
+interface SpanTreeProps {
+ rows: TreeRow[];
+ spanCount: number;
+ totalMs: number;
+ selectedId: string;
+ hideFramework: boolean;
+ onSelect: (id: string) => void;
+ onToggleHideFramework: (checked: boolean) => void;
+ onToggleSpan: (id: string) => void;
+ onToggleGroup: (id: string) => void;
+ onLoadMore: (groupId: string) => void;
+}
+
+const ROW_GRID = "grid h-8 w-full grid-cols-[minmax(275px,1fr)_minmax(94px,27%)_64px] items-center px-2";
+const INDENT_PX = 15;
+
+const rowClass = (selected: boolean): string =>
+ cn(
+ ROW_GRID,
+ "border-b border-border/60 text-left outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-inset",
+ selected ? "bg-accent shadow-[inset_2px_0_0_var(--foreground)]" : "hover:bg-muted/60",
+ );
+
+/** Span list with a framework toggle, a timeline column and keyboard hints. */
+export function SpanTree({
+ rows,
+ spanCount,
+ totalMs,
+ selectedId,
+ hideFramework,
+ onSelect,
+ onToggleHideFramework,
+ onToggleSpan,
+ onToggleGroup,
+ onLoadMore,
+}: SpanTreeProps) {
+ const scrollRef = useRef(null);
+
+ useEffect(() => {
+ scrollRef.current
+ ?.querySelector(`[data-row-id="${CSS.escape(selectedId)}"]`)
+ ?.scrollIntoView?.({ block: "nearest" });
+ }, [selectedId]);
+
+ return (
+
+
+
+ {`${spanCount.toLocaleString()} spans`}
+
+
+
+
+ Span
+ Timeline
+ Time
+
+
+ {rows.map((row) => (
+
+ ))}
+
+
+
+ J/K move
+
+
+ ←/→ fold
+
+
+ Esc close
+
+
+
+ );
+}
+
+function Kbd({ children }: { children: React.ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+interface TreeRowItemProps {
+ row: TreeRow;
+ selected: boolean;
+ totalMs: number;
+ onSelect: (id: string) => void;
+ onToggleSpan: (id: string) => void;
+ onToggleGroup: (id: string) => void;
+ onLoadMore: (groupId: string) => void;
+}
+
+function TreeRowItem({ row, selected, totalMs, onSelect, onToggleSpan, onToggleGroup, onLoadMore }: TreeRowItemProps) {
+ if (row.kind === "load-more") {
+ return (
+
+ );
+ }
+
+ if (row.kind === "group") {
+ const start = Math.min(...row.members.map((m) => m.start_offset_ms));
+ const end = Math.max(...row.members.map((m) => m.start_offset_ms + m.duration_ms));
+ return (
+
+ );
+ }
+
+ const { span } = row;
+ const failed = span.status === "error";
+ return (
+
+ );
+}
+
+function TypeIcon({ type, error }: { type: SpanType; error: boolean }) {
+ const className = cn("size-3 shrink-0", error ? "text-destructive" : "text-muted-foreground");
+ if (type === "agent") return ;
+ if (type === "llm") return ;
+ if (type === "tool") return ;
+ if (type === "framework") return ;
+ return ;
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/StatusMark.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/StatusMark.tsx
new file mode 100644
index 00000000000..07ad6f68249
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/StatusMark.tsx
@@ -0,0 +1,27 @@
+import { AlertCircle, Check, Circle } from "lucide-react";
+
+interface StatusMarkProps {
+ status: "ok" | "error" | "unset";
+ count?: number;
+ subtle?: boolean;
+}
+
+/** Run / span status glyph: red alert (+ count) for errors, a quiet muted dot or check otherwise. */
+export function StatusMark({ status, count, subtle = false }: StatusMarkProps) {
+ if (status === "error") {
+ return (
+
+
+ {count !== undefined && {count}}
+
+ );
+ }
+ if (subtle)
+ return (
+
+ );
+ return ;
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx
new file mode 100644
index 00000000000..0d3cf77a1e5
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx
@@ -0,0 +1,101 @@
+"use client";
+
+import { ChevronDown, Pause, Play, RotateCcw } from "lucide-react";
+import moment from "moment";
+
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuRadioGroup,
+ DropdownMenuRadioItem,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import { cn } from "@/lib/cva.config";
+
+import type { TimeWindow } from "./TracesTimeline";
+
+export const RANGE_PRESETS = [
+ { hours: 1, label: "Last hour" },
+ { hours: 6, label: "Last 6 hours" },
+ { hours: 24, label: "Last 24 hours" },
+ { hours: 168, label: "Last 7 days" },
+ { hours: 720, label: "Last 30 days" },
+] as const;
+
+const RANGE_LABEL_FORMAT = "MMM D, h:mm A";
+
+export const rangeLabel = (range: TimeWindow): string =>
+ `${moment(range.startMs).format(RANGE_LABEL_FORMAT)} to ${moment(range.endMs).format(RANGE_LABEL_FORMAT)}`;
+
+const SEGMENT = "inline-flex h-7 items-center gap-1.5 px-2.5 text-[13px] outline-none focus-visible:bg-accent";
+
+interface TimeRangeControlsProps {
+ range: TimeWindow;
+ rangeHours: number;
+ onRangeHoursChange: (hours: number) => void;
+ live: boolean;
+ onLiveChange: (live: boolean) => void;
+ zoomed: boolean;
+ onResetZoom: () => void;
+}
+
+/** Joined control group: reset zoom, the actual time range (opens presets), and Live. */
+export function TimeRangeControls({
+ range,
+ rangeHours,
+ onRangeHoursChange,
+ live,
+ onLiveChange,
+ zoomed,
+ onResetZoom,
+}: TimeRangeControlsProps) {
+ return (
+
+
+
+
+
+ {rangeLabel(range)}
+
+
+
+ onRangeHoursChange(Number(value))}
+ >
+ {RANGE_PRESETS.map((preset) => (
+
+ {preset.label}
+
+ ))}
+
+
+
+
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx
new file mode 100644
index 00000000000..2698a0c6643
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx
@@ -0,0 +1,146 @@
+import { screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import researchTrace from "./__fixtures__/research_trace.json";
+import swarmTrace from "./__fixtures__/swarm_trace.json";
+import { agentHandoffText, initialRunSelection, RunView } from "./TraceDrawer";
+import type { Span } from "./traceTypes";
+import type { Trace } from "./traceTypes";
+import { traceDisplayName } from "./traceUtils";
+
+vi.mock("../../networking", () => ({
+ agentTraceCall: vi.fn(),
+ agentTraceSpanCall: vi.fn(),
+ getProxyBaseUrl: () => "http://proxy.test/",
+}));
+
+// DetailPane is built separately; render a stub that exposes which row is selected.
+vi.mock("./DetailPane", () => ({
+ DetailPane: ({ row, onClose }: { row?: { id: string }; onClose: () => void }) => (
+
+
+
+ ),
+}));
+
+vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn().mockResolvedValue(true) }));
+
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import { agentTraceCall } from "../../networking";
+
+const swarm = swarmTrace as Trace;
+const research = researchTrace as Trace;
+
+const renderRun = (trace: Trace) => {
+ vi.mocked(agentTraceCall).mockResolvedValue(trace);
+ return renderWithProviders();
+};
+
+const rootSpanId = (trace: Trace): string => trace.spans.find((s) => s.parent_span_id === null)?.span_id ?? "";
+
+describe("RunView", () => {
+ beforeEach(() => {
+ testQueryClient.clear();
+ vi.mocked(copyToClipboard).mockClear();
+ });
+
+ it("shows a one-line run header: agent name, trace id, duration and steps", async () => {
+ renderRun(research);
+
+ const header = await screen.findByRole("banner");
+ expect(screen.getByRole("heading", { level: 1 })).toHaveTextContent(traceDisplayName(research.summary));
+ expect(header).toHaveTextContent(research.summary.trace_id);
+ expect(header).toHaveTextContent("duration 40.20s");
+ expect(header).toHaveTextContent(`steps ${research.summary.span_count}`);
+ expect(header).not.toHaveTextContent("failed");
+ });
+
+ it("folds researcher ×12 in the span tree", async () => {
+ renderRun(swarm);
+
+ const tree = await screen.findByRole("tree", { name: "Spans in time order" });
+ expect(tree).toHaveTextContent("researcher×12");
+ expect(screen.getByRole("banner")).toHaveTextContent(`failed ${swarm.summary.error_count}`);
+ });
+
+ it("opens a failed run on its first failed span", async () => {
+ renderRun(swarm);
+
+ const pane = await screen.findByTestId("detail-pane");
+ const { selectedId } = initialRunSelection(swarm);
+ expect(selectedId).not.toBe(rootSpanId(swarm));
+ expect(pane).toHaveAttribute("data-row-id", selectedId);
+ expect(swarm.spans.find((s) => s.span_id === selectedId)?.status).toBe("error");
+ });
+
+ it("opens a healthy run on the root span", async () => {
+ renderRun(research);
+ expect(await screen.findByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research));
+ });
+
+ it("moves the selection with J / K and closes the detail pane with Esc", async () => {
+ const user = userEvent.setup();
+ renderRun(research);
+
+ const pane = await screen.findByTestId("detail-pane");
+ const root = rootSpanId(research);
+ expect(pane).toHaveAttribute("data-row-id", root);
+ await user.keyboard("j");
+ expect(screen.getByTestId("detail-pane").getAttribute("data-row-id")).not.toBe(root);
+ await user.keyboard("k");
+ expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", root);
+ await user.keyboard("{Escape}");
+ expect(screen.queryByTestId("detail-pane")).not.toBeInTheDocument();
+ });
+
+ it("keeps a way back to the runs table when a run fails to load", async () => {
+ const user = userEvent.setup();
+ const onBack = vi.fn();
+ vi.mocked(agentTraceCall).mockRejectedValue(new Error("trace exceeds the 1000 span read limit"));
+ renderWithProviders();
+
+ expect(await screen.findByText("trace exceeds the 1000 span read limit")).toBeInTheDocument();
+ await user.click(screen.getByRole("button", { name: /back to traces/i }));
+ expect(onBack).toHaveBeenCalledTimes(1);
+ });
+
+ it("copies a curl one-liner for Claude / Codex", async () => {
+ const user = userEvent.setup();
+ renderRun(research);
+
+ await user.click(await screen.findByRole("button", { name: /copy for agent/i }));
+ expect(copyToClipboard).toHaveBeenCalledWith(agentHandoffText(research.summary.trace_id), "Command copied");
+ expect(agentHandoffText("t1")).toContain('"http://proxy.test/v1/traces/t1?format=md"');
+ expect(agentHandoffText("t1", "s1")).toContain("&span_id=s1");
+ });
+});
+
+describe("initialRunSelection", () => {
+ const base = research.spans.find((s) => s.parent_span_id === null) as Span;
+ const child = (over: Partial): Span => {
+ const defaults: Partial = { parent_span_id: base.span_id, status: "ok" };
+ return { ...base, ...defaults, ...over };
+ };
+
+ it("lands on a visible failure, never on a framework span the tree hides", () => {
+ const hiddenFields: Partial = { span_id: "mw", type: "framework", status: "error", start_offset_ms: 1 };
+ const toolFields: Partial = { span_id: "tool", type: "tool", status: "error", start_offset_ms: 5 };
+ const hiddenFailure = child(hiddenFields);
+ const toolFailure = child(toolFields);
+ const trace = { ...research, spans: [base, hiddenFailure, toolFailure] };
+ expect(initialRunSelection(trace).selectedId).toBe("tool");
+ });
+
+ it("falls back to the nearest visible ancestor when only a hidden span failed", () => {
+ const agent = child({ span_id: "agent", type: "agent", name: "researcher" });
+ const hiddenFields: Partial = { span_id: "mw", parent_span_id: "agent", type: "framework", status: "error" };
+ const hiddenFailure = child(hiddenFields);
+ const trace = { ...research, spans: [base, agent, hiddenFailure] };
+ expect(initialRunSelection(trace).selectedId).toBe("agent");
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx
new file mode 100644
index 00000000000..19201533cb3
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx
@@ -0,0 +1,269 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { ArrowLeft, Check, Circle, Copy } from "lucide-react";
+import { useCallback, useEffect, useMemo, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import { agentTraceCall, getProxyBaseUrl } from "../../networking";
+import { DetailPane } from "./DetailPane";
+import { SpanTree } from "./SpanTree";
+import type { SpanTreeState, TreeRow } from "./traceTree";
+import type { Trace } from "./traceTypes";
+import {
+ buildTreeRows,
+ firstErrorSpan,
+ fmtMs,
+ GROUP_PAGE_SIZE,
+ isFrameworkSpan,
+ nearestVisibleSpanId,
+ revealSpanInState,
+ traceDisplayName,
+} from "./traceUtils";
+
+/** What "Copy for agent" puts on the clipboard: a one-liner Claude Code / Codex can run. */
+export const agentHandoffText = (traceId: string, spanId?: string | null): string => {
+ const url = `${getProxyBaseUrl().replace(/\/$/, "")}/v1/traces/${traceId}?format=md${spanId ? `&span_id=${spanId}` : ""}`;
+ const what = spanId ? "this step of a LiteLLM agent trace" : "this LiteLLM agent trace";
+ return `Read ${what} and explain what happened and why it failed:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`;
+};
+
+const INITIAL_STATE: SpanTreeState = {
+ hideFramework: true,
+ collapsedSpanIds: new Set(),
+ expandedGroupIds: new Set(),
+ groupRevealCounts: {},
+};
+
+/** First failed span if the run has errors (with its tree path opened), otherwise the root agent. */
+export function initialRunSelection(trace: Trace): { selectedId: string; state: SpanTreeState } {
+ const failed = firstErrorSpan(trace.spans);
+ if (!failed || failed.parent_span_id === null) {
+ const root = trace.spans.find((s) => s.parent_span_id === null);
+ return { selectedId: root?.span_id ?? "", state: INITIAL_STATE };
+ }
+ const visibleFailure = trace.spans
+ .filter((s) => s.status === "error" && s.parent_span_id !== null && !isFrameworkSpan(s))
+ .sort((a, b) => a.start_offset_ms - b.start_offset_ms)[0];
+ const selectedId = visibleFailure?.span_id ?? nearestVisibleSpanId(trace.spans, failed.span_id, true);
+ return { selectedId, state: revealSpanInState(trace.spans, INITIAL_STATE, selectedId) };
+}
+
+const toggle = (set: ReadonlySet, id: string): Set => {
+ const next = new Set(set);
+ if (next.has(id)) next.delete(id);
+ else next.add(id);
+ return next;
+};
+
+function CopyForAgent({ traceId }: { traceId: string }) {
+ const [copied, setCopied] = useState(false);
+ useEffect(() => {
+ if (!copied) return;
+ const timeout = window.setTimeout(() => setCopied(false), 1600);
+ return () => window.clearTimeout(timeout);
+ }, [copied]);
+ return (
+
+ );
+}
+
+function Stat({ label, value, error = false }: { label: string; value: string; error?: boolean }) {
+ return (
+
+ {label}
+ {value}
+
+ );
+}
+
+function RunHeader({ trace, onBack }: { trace: Trace; onBack: () => void }) {
+ const { summary } = trace;
+ const failed = summary.error_count > 0;
+ return (
+
+
+
+ {traceDisplayName(summary)}
+
+ {summary.trace_id}
+
+
+
+
+
+ {failed && }
+
+
+
+
+
+ );
+}
+
+/** Tree + detail pane for one loaded run, with J/K/arrow keyboard navigation. */
+function RunBody({ trace, accessToken }: { trace: Trace; accessToken: string }) {
+ const initial = useMemo(() => initialRunSelection(trace), [trace]);
+ const [state, setState] = useState(initial.state);
+ const [selectedId, setSelectedId] = useState(initial.selectedId);
+ const [detailOpen, setDetailOpen] = useState(true);
+
+ const rows = useMemo(() => buildTreeRows(trace.spans, state), [trace, state]);
+ const selectedRow: TreeRow | undefined = rows.find((row) => row.id === selectedId) ?? rows[0];
+
+ const select = useCallback((id: string) => {
+ setSelectedId(id);
+ setDetailOpen(true);
+ }, []);
+ const toggleSpan = useCallback(
+ (id: string) => setState((prev) => ({ ...prev, collapsedSpanIds: toggle(prev.collapsedSpanIds, id) })),
+ [],
+ );
+ const toggleGroup = useCallback(
+ (id: string) => setState((prev) => ({ ...prev, expandedGroupIds: toggle(prev.expandedGroupIds, id) })),
+ [],
+ );
+ const loadMore = useCallback(
+ (groupId: string) =>
+ setState((prev) => ({
+ ...prev,
+ groupRevealCounts: {
+ ...prev.groupRevealCounts,
+ [groupId]: (prev.groupRevealCounts[groupId] ?? GROUP_PAGE_SIZE) + GROUP_PAGE_SIZE,
+ },
+ })),
+ [],
+ );
+
+ useEffect(() => {
+ const onKeyDown = (event: KeyboardEvent) => {
+ if ((event.target as HTMLElement | null)?.matches("input, textarea, [role='combobox']")) return;
+ const index = rows.findIndex((row) => row.id === selectedRow?.id);
+ const row = rows[index];
+ if (event.key === "Escape" && detailOpen) {
+ event.preventDefault();
+ event.stopPropagation();
+ setDetailOpen(false);
+ return;
+ }
+ if (["j", "J", "ArrowDown"].includes(event.key)) {
+ event.preventDefault();
+ const next = rows[Math.min(rows.length - 1, index + 1)];
+ if (next) select(next.id);
+ } else if (["k", "K", "ArrowUp"].includes(event.key)) {
+ event.preventDefault();
+ const next = rows[Math.max(0, index - 1)];
+ if (next) select(next.id);
+ } else if (event.key === "ArrowLeft" && row) {
+ if (row.kind === "span" && row.hasChildren && !row.collapsed) toggleSpan(row.id);
+ if (row.kind === "group" && row.expanded) toggleGroup(row.id);
+ } else if (event.key === "ArrowRight" && row) {
+ if (row.kind === "span" && row.hasChildren && row.collapsed) toggleSpan(row.id);
+ if (row.kind === "group" && !row.expanded) toggleGroup(row.id);
+ }
+ };
+ window.addEventListener("keydown", onKeyDown, true);
+ return () => window.removeEventListener("keydown", onKeyDown, true);
+ }, [rows, selectedRow, detailOpen, select, toggleSpan, toggleGroup]);
+
+ return (
+
+ setState((prev) => ({ ...prev, hideFramework }))}
+ onToggleSpan={toggleSpan}
+ onToggleGroup={toggleGroup}
+ onLoadMore={loadMore}
+ />
+ {detailOpen && (
+ setDetailOpen(false)} />
+ )}
+
+ );
+}
+
+interface RunViewProps {
+ traceId: string;
+ accessToken: string;
+ onBack: () => void;
+}
+
+/** One agent run: header with totals and "Copy for agent", span tree on the left, span details on the right. */
+export function RunView({ traceId, accessToken, onBack }: RunViewProps) {
+ const traceQuery = useQuery({
+ queryKey: ["agentTrace", traceId, accessToken],
+ queryFn: () => agentTraceCall(accessToken, traceId),
+ staleTime: 30_000,
+ });
+ const trace = traceQuery.data;
+
+ if (traceQuery.isLoading) {
+ return (
+
+
+
+ );
+ }
+ if (traceQuery.isError || !trace) {
+ return (
+
+
+
Could not load trace
+
{traceQuery.error?.message ?? "Unknown error"}
+
+ );
+ }
+ return (
+
+
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.test.ts
new file mode 100644
index 00000000000..46c69559af6
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+
+import { bandForWindow, bucketRuns, dragUpdate, formatSpan } from "./TracesTimeline";
+import type { TraceSummary } from "./traceTypes";
+
+const HOUR = 3600 * 1000;
+const START = Date.UTC(2026, 8, 30, 0, 0, 0);
+const range = { startMs: START, endMs: START + 10 * HOUR };
+
+const run = (offsetMs: number, errorCount = 0): TraceSummary =>
+ ({
+ trace_id: `t${offsetMs}`,
+ start_time: new Date(START + offsetMs).toISOString(),
+ error_count: errorCount,
+ }) as TraceSummary;
+
+describe("bucketRuns", () => {
+ it("splits the window into equal buckets that tile it exactly", () => {
+ const buckets = bucketRuns([], range, 10);
+ expect(buckets).toHaveLength(10);
+ expect(buckets[0].startMs).toBe(range.startMs);
+ expect(buckets.at(-1)?.endMs).toBe(range.endMs);
+ const fourthBucket = { startMs: START + 3 * HOUR, endMs: START + 4 * HOUR, runs: 0, failed: 0 };
+ expect(buckets[3]).toMatchObject(fourthBucket);
+ });
+
+ it("puts each run in the bucket covering its start time", () => {
+ const buckets = bucketRuns([run(0), run(30 * 60 * 1000), run(2.5 * HOUR), run(10 * HOUR - 1)], range, 10);
+ expect(buckets.map((b) => b.runs)).toEqual([2, 0, 1, 0, 0, 0, 0, 0, 0, 1]);
+ });
+
+ it("drops runs that start before or at/after the window", () => {
+ const buckets = bucketRuns([run(-1), run(10 * HOUR), run(20 * HOUR), run(HOUR)], range, 10);
+ expect(buckets.reduce((sum, b) => sum + b.runs, 0)).toBe(1);
+ expect(buckets[1].runs).toBe(1);
+ });
+
+ it("counts runs with any errors as failed", () => {
+ const buckets = bucketRuns([run(HOUR, 3), run(HOUR + 1), run(HOUR + 2, 1), run(5 * HOUR)], range, 10);
+ expect(buckets[1]).toMatchObject({ runs: 3, failed: 2 });
+ expect(buckets[5]).toMatchObject({ runs: 1, failed: 0 });
+ });
+});
+
+describe("formatSpan", () => {
+ it("prints the largest two units, dropping zero parts", () => {
+ expect(formatSpan(45 * 60 * 1000)).toBe("45m");
+ expect(formatSpan(6 * HOUR + 12 * 60 * 1000)).toBe("6h 12m");
+ expect(formatSpan(24 * HOUR)).toBe("1d");
+ expect(formatSpan(152 * 24 * HOUR + 23 * HOUR)).toBe("152d 23h");
+ expect(formatSpan(-5)).toBe("0m");
+ });
+});
+
+describe("dragUpdate", () => {
+ const band = { lo: 10, hi: 14 };
+
+ it("selects between the press point and the pointer, in either direction", () => {
+ expect(dragUpdate({ mode: "select", origin: 20, band: { lo: 20, hi: 20 } }, 25)).toEqual({ lo: 20, hi: 25 });
+ expect(dragUpdate({ mode: "select", origin: 20, band: { lo: 20, hi: 20 } }, 12)).toEqual({ lo: 12, hi: 20 });
+ });
+
+ it("resizes one edge without letting it cross the other", () => {
+ expect(dragUpdate({ mode: "resize-lo", origin: 10, band }, 4)).toEqual({ lo: 4, hi: 14 });
+ expect(dragUpdate({ mode: "resize-lo", origin: 10, band }, 30)).toEqual({ lo: 14, hi: 14 });
+ expect(dragUpdate({ mode: "resize-hi", origin: 14, band }, 40)).toEqual({ lo: 10, hi: 40 });
+ expect(dragUpdate({ mode: "resize-hi", origin: 14, band }, 2)).toEqual({ lo: 10, hi: 10 });
+ });
+
+ it("pans the band keeping its width, clamped to the strip", () => {
+ expect(dragUpdate({ mode: "move", origin: 12, band }, 20)).toEqual({ lo: 18, hi: 22 });
+ expect(dragUpdate({ mode: "move", origin: 12, band }, -50)).toEqual({ lo: 0, hi: 4 });
+ expect(dragUpdate({ mode: "move", origin: 12, band }, 500, 60)).toEqual({ lo: 55, hi: 59 });
+ });
+});
+
+describe("bandForWindow", () => {
+ it("maps a selected window back to the buckets it covers", () => {
+ const buckets = bucketRuns([], range, 10);
+ expect(bandForWindow(buckets, { startMs: START + 2 * HOUR, endMs: START + 5 * HOUR })).toEqual({ lo: 2, hi: 4 });
+ expect(bandForWindow(buckets, null)).toBeNull();
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.tsx
new file mode 100644
index 00000000000..7539e5a4e83
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.tsx
@@ -0,0 +1,365 @@
+"use client";
+
+import moment from "moment";
+import { useEffect, useMemo, useRef, useState } from "react";
+
+import { cn } from "@/lib/cva.config";
+
+import type { TraceSummary } from "./traceTypes";
+
+const BUCKETS = 60;
+const TICKS = 6;
+const MINUTE_MS = 60 * 1000;
+const HOUR_MS = 60 * MINUTE_MS;
+const DAY_MS = 24 * HOUR_MS;
+const EDGE_FORMAT = "MMM DD, HH:mm";
+
+export interface TimeWindow {
+ startMs: number;
+ endMs: number;
+}
+
+export interface Bucket {
+ startMs: number;
+ endMs: number;
+ runs: number;
+ failed: number;
+}
+
+/** Run counts per equal-width time bucket across the window; runs outside it are dropped. */
+export function bucketRuns(runs: readonly TraceSummary[], range: TimeWindow, buckets = BUCKETS): Bucket[] {
+ const width = (range.endMs - range.startMs) / buckets;
+ const placed = runs.map((run) => ({
+ index: Math.floor((moment(run.start_time).valueOf() - range.startMs) / width),
+ failed: run.error_count > 0,
+ }));
+ return Array.from({ length: buckets }, (_, i) => {
+ const hits = placed.filter((p) => p.index === i);
+ return {
+ startMs: range.startMs + i * width,
+ endMs: range.startMs + (i + 1) * width,
+ runs: hits.length,
+ failed: hits.filter((p) => p.failed).length,
+ };
+ });
+}
+
+/** Compact window length, Logfire-style: "45m", "6h 12m", "7d", "152d 23h". */
+export function formatSpan(ms: number): string {
+ const totalMinutes = Math.max(0, Math.round(ms / MINUTE_MS));
+ const days = Math.floor(totalMinutes / (24 * 60));
+ const hours = Math.floor((totalMinutes % (24 * 60)) / 60);
+ const minutes = totalMinutes % 60;
+ if (days > 0) return hours > 0 ? `${days}d ${hours}h` : `${days}d`;
+ if (hours > 0) return minutes > 0 ? `${hours}h ${minutes}m` : `${hours}h`;
+ return `${minutes}m`;
+}
+
+const tickFormat = (range: TimeWindow): string => (range.endMs - range.startMs > 2 * DAY_MS ? EDGE_FORMAT : "HH:mm");
+
+const pct = (value: number): string => `${value * 100}%`;
+
+/** Keep the first / last tick label inside the strip; center the rest on their tick. */
+const tickShift = (t: number): string => {
+ if (t === 0) return "translateX(0)";
+ if (t === 1) return "translateX(-100%)";
+ return "translateX(-50%)";
+};
+
+interface TracesTimelineProps {
+ runs: readonly TraceSummary[];
+ range: TimeWindow;
+ selection: TimeWindow | null;
+ onSelect: (selection: TimeWindow | null) => void;
+}
+
+function BucketBar({
+ bucket,
+ max,
+ dimmed,
+ hovered,
+}: {
+ bucket: Bucket;
+ max: number;
+ dimmed: boolean;
+ hovered: boolean;
+}) {
+ return (
+
+ {bucket.runs > 0 && (
+
+ {bucket.failed > 0 && (
+
+ )}
+
+ )}
+
+ );
+}
+
+function BucketTooltip({ bucket, index }: { bucket: Bucket; index: number }) {
+ return (
+
+
+ {moment(bucket.startMs).format(EDGE_FORMAT)} to {moment(bucket.endMs).format("HH:mm")}
+
+
+ {bucket.runs} {bucket.runs === 1 ? "run" : "runs"}
+ {bucket.failed > 0 && `, ${bucket.failed} failed`}
+
+
drag to zoom
+
+ );
+}
+
+const MIN_DURATION_LABEL_PX = 120;
+
+/** Bucket-index band [lo, hi], inclusive. */
+export interface Band {
+ lo: number;
+ hi: number;
+}
+
+export type DragMode = "select" | "move" | "resize-lo" | "resize-hi";
+
+export interface DragState {
+ mode: DragMode;
+ origin: number;
+ band: Band;
+}
+
+/** The band a drag produces when the pointer is over bucket `at`; always within [0, buckets - 1]. */
+export function dragUpdate(drag: DragState, at: number, buckets = BUCKETS): Band {
+ const last = buckets - 1;
+ const clamp = (i: number) => Math.min(last, Math.max(0, i));
+ const { band, origin } = drag;
+ if (drag.mode === "select") return { lo: Math.min(origin, clamp(at)), hi: Math.max(origin, clamp(at)) };
+ if (drag.mode === "resize-lo") return { lo: Math.min(clamp(at), band.hi), hi: band.hi };
+ if (drag.mode === "resize-hi") return { lo: band.lo, hi: Math.max(clamp(at), band.lo) };
+ const width = band.hi - band.lo;
+ const lo = Math.min(last - width, Math.max(0, band.lo + (at - origin)));
+ return { lo, hi: lo + width };
+}
+
+/** Bucket band covered by a selected window, or null when nothing is selected. */
+export function bandForWindow(buckets: readonly Bucket[], selection: TimeWindow | null): Band | null {
+ if (selection === null) return null;
+ const inside = buckets.flatMap((b, i) => (b.startMs >= selection.startMs && b.endMs <= selection.endMs ? [i] : []));
+ return inside.length > 0 ? { lo: inside[0], hi: inside[inside.length - 1] } : null;
+}
+
+const edgeFormat = (range: TimeWindow): string => (range.endMs - range.startMs < DAY_MS ? "HH:mm" : EDGE_FORMAT);
+
+/** The selected window drawn as a flat bracket: resize handles on each edge, times outside, span centered. */
+function SelectionBracket({
+ band,
+ window,
+ format,
+ stripWidth,
+ onHandleDown,
+}: {
+ band: Band;
+ window: TimeWindow;
+ format: string;
+ stripWidth: number;
+ onHandleDown: (mode: DragMode) => (e: React.PointerEvent) => void;
+}) {
+ const leftFrac = band.lo / BUCKETS;
+ const rightFrac = (band.hi + 1) / BUCKETS;
+ const widthPx = (rightFrac - leftFrac) * stripWidth;
+ const handle = "absolute inset-y-0 w-[3px] cursor-ew-resize bg-info";
+ const edgeLabel =
+ "pointer-events-none absolute -top-4 font-mono text-[10px] whitespace-nowrap text-info tabular-nums";
+ return (
+ <>
+
+
+
+ {widthPx >= MIN_DURATION_LABEL_PX && (
+
+ {formatSpan(window.endMs - window.startMs)}
+
+ )}
+
+
+ {moment(window.startMs).format(format)}
+
+ 0.88 ? "-translate-x-full" : "pl-1")} style={{ left: pct(rightFrac) }}>
+ {moment(window.endMs).format(format)}
+
+ >
+ );
+}
+
+function TickAxis({ range }: { range: TimeWindow }) {
+ const format = tickFormat(range);
+ const ticks = Array.from({ length: TICKS }, (_, i) => i / (TICKS - 1));
+ return (
+
+ {ticks.map((t) => (
+
0 && t < 1 && "items-center",
+ )}
+ style={{ left: pct(t), transform: tickShift(t) }}
+ >
+
+
+ {moment(range.startMs + (range.endMs - range.startMs) * t).format(format)}
+
+
+ ))}
+
+ );
+}
+
+/** Histogram of runs over the window. Drag to select; drag the bracket or its edges to adjust; Esc clears. */
+export function TracesTimeline({ runs, range, selection, onSelect }: TracesTimelineProps) {
+ const buckets = useMemo(() => bucketRuns(runs, range), [runs, range]);
+ const max = Math.max(1, ...buckets.map((b) => b.runs));
+ const [hover, setHover] = useState(null);
+ const [drag, setDrag] = useState(null);
+ const [draft, setDraft] = useState(null);
+ const areaRef = useRef(null);
+ const [stripWidth, setStripWidth] = useState(0);
+
+ useEffect(() => {
+ const el = areaRef.current;
+ if (!el) return;
+ const measure = () => setStripWidth(el.getBoundingClientRect().width);
+ measure();
+ if (typeof ResizeObserver === "undefined") return;
+ const observer = new ResizeObserver(measure);
+ observer.observe(el);
+ return () => observer.disconnect();
+ }, []);
+
+ const committed = bandForWindow(buckets, selection);
+ const band = drag ? draft : committed;
+ const windowOf = (b: Band): TimeWindow => ({ startMs: buckets[b.lo].startMs, endMs: buckets[b.hi].endMs });
+
+ const indexAt = (clientX: number): number => {
+ const rect = areaRef.current?.getBoundingClientRect();
+ if (!rect || rect.width === 0) return 0;
+ return Math.min(BUCKETS - 1, Math.max(0, Math.floor(((clientX - rect.left) / rect.width) * BUCKETS)));
+ };
+ const begin = (mode: DragMode, e: React.PointerEvent) => {
+ e.preventDefault();
+ e.stopPropagation();
+ areaRef.current?.setPointerCapture?.(e.pointerId);
+ const at = indexAt(e.clientX);
+ const start = mode === "select" || committed === null ? { lo: at, hi: at } : committed;
+ setDrag({ mode: committed === null ? "select" : mode, origin: at, band: start });
+ setDraft(start);
+ };
+ const onHandleDown = (mode: DragMode) => (e: React.PointerEvent) => begin(mode, e);
+ const onMove = (e: React.PointerEvent) => {
+ const at = indexAt(e.clientX);
+ setHover(at);
+ if (drag) setDraft(dragUpdate(drag, at));
+ };
+ const onUp = (e: React.PointerEvent) => {
+ areaRef.current?.releasePointerCapture?.(e.pointerId);
+ if (!drag || !draft) return;
+ const finished = draft;
+ const wasSelect = drag.mode === "select";
+ setDrag(null);
+ setDraft(null);
+ if (wasSelect && finished.lo === finished.hi) {
+ onSelect(null);
+ return;
+ }
+ onSelect(windowOf(finished));
+ };
+ const onKeyDown = (e: React.KeyboardEvent) => {
+ if (e.key !== "Escape" || selection === null) return;
+ e.preventDefault();
+ onSelect(null);
+ };
+
+ const isDimmed = (i: number): boolean => band !== null && (i < band.lo || i > band.hi);
+ const labelFormat = edgeFormat(range);
+
+ return (
+
+
+
+ Total {formatSpan(range.endMs - range.startMs)}
+
+
+
begin("select", e)}
+ onPointerMove={onMove}
+ onPointerUp={onUp}
+ onPointerLeave={() => setHover(null)}
+ onDoubleClick={() => onSelect(null)}
+ >
+ {hover !== null && !drag && (
+
+ )}
+ {buckets.map((b, i) => (
+
+ ))}
+ {band && (
+
+ )}
+
+
+ {hover !== null && !drag &&
}
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx
new file mode 100644
index 00000000000..c7532d2c7b6
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx
@@ -0,0 +1,57 @@
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+
+import { codingAgentPrompt, tracingEnvSnippet, TracingSetupCard } from "./TracingSetupCard";
+
+vi.mock("../../networking", () => ({ getProxyBaseUrl: () => "http://proxy.test/" }));
+vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn().mockResolvedValue(true) }));
+
+describe("TracingSetupCard", () => {
+ it("shows the waiting state and OTEL-only setup when tracing is on", () => {
+ render();
+
+ const card = screen.getByTestId("tracing-setup-card");
+ expect(card).toHaveTextContent("Waiting for traces");
+ expect(card).toHaveTextContent("No traces detected yet. Follow our guide to start tracing your application.");
+ expect(card).toHaveTextContent("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test");
+ expect(card).not.toHaveTextContent(/langsmith/i);
+ expect(card).not.toHaveTextContent("store: clickhouse");
+ });
+
+ it("swaps the install command and prompt when another framework is picked", async () => {
+ const user = userEvent.setup();
+ render();
+
+ expect(screen.getByTestId("tracing-setup-card")).toHaveTextContent("openinference-instrumentation-langchain");
+ await user.click(screen.getByRole("radio", { name: "CrewAI" }));
+
+ const card = screen.getByTestId("tracing-setup-card");
+ expect(screen.getByRole("radio", { name: "CrewAI" })).toBeChecked();
+ expect(card).toHaveTextContent("pip install -U opentelemetry-distro");
+ expect(card).toHaveTextContent("crewai openinference-instrumentation-crewai");
+ expect(card).toHaveTextContent("Send this CrewAI project's OpenTelemetry traces to LiteLLM.");
+ expect(card).not.toHaveTextContent("openinference-instrumentation-langchain");
+ });
+
+ it("shows the proxy config step only when tracing is not enabled", () => {
+ render();
+ const card = screen.getByTestId("tracing-setup-card");
+ expect(card).toHaveTextContent("Tracing is not enabled");
+ expect(card).toHaveTextContent("store: clickhouse");
+ });
+});
+
+describe("setup snippets", () => {
+ it("points OTLP at the proxy base URL and reads the key from the environment", () => {
+ const env = tracingEnvSnippet("http://proxy.test");
+ expect(env).toContain("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test\n");
+ expect(env).not.toContain("/v1/traces");
+ expect(env).toContain("Bearer $LITELLM_API_KEY");
+
+ const prompt = codingAgentPrompt("http://proxy.test", { label: "LangChain", packages: "langchain" });
+ expect(prompt).toContain("base_url=http://proxy.test/v1");
+ expect(prompt).toContain("opentelemetry-distro opentelemetry-exporter-otlp-proto-http langchain");
+ expect(prompt).not.toMatch(/langsmith/i);
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx
new file mode 100644
index 00000000000..35bfb8c57d8
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx
@@ -0,0 +1,390 @@
+"use client";
+
+import { ArrowUpRight, Check, Copy, Loader2 } from "lucide-react";
+import { useState } from "react";
+
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import previewImg from "../../../../public/assets/agent-traces-preview.png";
+import crewaiLogo from "../../../../public/assets/logos/crewai-color.svg";
+import langchainLogo from "../../../../public/assets/logos/langchain.svg";
+import langgraphLogo from "../../../../public/assets/logos/langgraph-color.svg";
+import llamaindexLogo from "../../../../public/assets/logos/llamaindex-color.svg";
+import openaiAgentsLogo from "../../../../public/assets/logos/openai-agents.svg";
+import otelLogo from "../../../../public/assets/logos/opentelemetry.svg";
+import pydanticAiLogo from "../../../../public/assets/logos/pydantic-ai-color.svg";
+import { getProxyBaseUrl } from "../../networking";
+
+const COPIED_RESET_MS = 1500;
+const DOCS_URL = "https://docs.litellm.ai";
+const OTEL_BASE_PACKAGES = "opentelemetry-distro opentelemetry-exporter-otlp-proto-http";
+const RUN_SNIPPET = "opentelemetry-instrument python my_agent.py";
+
+type Installer = "pip" | "uv";
+
+interface FrameworkGuide {
+ id: string;
+ label: string;
+ logo: string;
+ packages: string;
+ quickstart: string;
+}
+
+const FRAMEWORKS: readonly FrameworkGuide[] = [
+ {
+ id: "langgraph",
+ label: "LangGraph / Deep Agents",
+ logo: langgraphLogo.src,
+ packages: "langgraph langchain-openai openinference-instrumentation-langchain",
+ quickstart: `from langchain.agents import create_agent
+from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = create_agent(model=llm, tools=[], name="my_agent")
+agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`,
+ },
+ {
+ id: "langchain",
+ label: "LangChain",
+ logo: langchainLogo.src,
+ packages: "langchain langchain-openai openinference-instrumentation-langchain",
+ quickstart: `from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+llm.invoke("What is LiteLLM?")`,
+ },
+ {
+ id: "openai-agents",
+ label: "OpenAI Agents SDK",
+ logo: openaiAgentsLogo.src,
+ packages: "openai-agents openinference-instrumentation-openai-agents",
+ quickstart: `from agents import Agent, OpenAIChatCompletionsModel, Runner
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(name="my_agent", model=OpenAIChatCompletionsModel(model="claude-sonnet-4-5", openai_client=client))
+print(Runner.run_sync(agent, "What is LiteLLM?").final_output)`,
+ },
+ {
+ id: "crewai",
+ label: "CrewAI",
+ logo: crewaiLogo.src,
+ packages: "crewai openinference-instrumentation-crewai",
+ quickstart: `from crewai import LLM, Agent, Crew, Task
+
+llm = LLM(model="openai/claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(role="Researcher", goal="Answer questions", backstory="", llm=llm)
+task = Task(description="What is LiteLLM?", expected_output="A short answer", agent=agent)
+Crew(agents=[agent], tasks=[task]).kickoff()`,
+ },
+ {
+ id: "pydantic-ai",
+ label: "Pydantic AI",
+ logo: pydanticAiLogo.src,
+ packages: "pydantic-ai openinference-instrumentation-pydantic-ai",
+ quickstart: `from pydantic_ai import Agent
+from pydantic_ai.models.openai import OpenAIModel
+from pydantic_ai.providers.openai import OpenAIProvider
+
+provider = OpenAIProvider(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(OpenAIModel("claude-sonnet-4-5", provider=provider), name="my_agent", instrument=True)
+print(agent.run_sync("What is LiteLLM?").output)`,
+ },
+ {
+ id: "llamaindex",
+ label: "LlamaIndex",
+ logo: llamaindexLogo.src,
+ packages: "llama-index llama-index-llms-openai-like openinference-instrumentation-llama-index",
+ quickstart: `from llama_index.llms.openai_like import OpenAILike
+
+llm = OpenAILike(model="claude-sonnet-4-5", api_base="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"], is_chat_model=True)
+print(llm.complete("What is LiteLLM?"))`,
+ },
+ {
+ id: "otel",
+ label: "OpenTelemetry",
+ logo: otelLogo.src,
+ packages: "",
+ quickstart: `# Any OTEL SDK works. Use the gen_ai.* semantic conventions:
+# gen_ai.operation.name, gen_ai.agent.name, gen_ai.response.id, gen_ai.usage.*
+from opentelemetry import trace
+
+tracer = trace.get_tracer("my-agent")
+attrs = {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "my_agent"}
+with tracer.start_as_current_span("my_agent", attributes=attrs):
+ ...`,
+ },
+];
+
+const HIGHLIGHTS = [
+ ["Input and output", "What the agent was asked and what it answered, at the top of every run."],
+ ["Every step, nested", "LLM calls, tool calls and subagents in one tree, with timing."],
+ ["Failures pinpointed", "See whether the tool, the model or LiteLLM broke."],
+ ["Hand off to Claude / Codex", "Copy one command and your coding agent debugs the run."],
+] as const;
+
+const installPackages = (guide: Pick): string =>
+ [OTEL_BASE_PACKAGES, guide.packages].filter(Boolean).join(" ");
+
+/** The endpoint is the proxy base URL: OTLP exporters append /v1/traces themselves. */
+export const tracingEnvSnippet = (proxyUrl: string): string =>
+ [
+ `export OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`,
+ 'export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"',
+ "export OTEL_SERVICE_NAME=my-agent",
+ ].join("\n");
+
+export const codingAgentPrompt = (proxyUrl: string, guide: Pick): string =>
+ [
+ `Send this ${guide.label} project's OpenTelemetry traces to LiteLLM.`,
+ "",
+ `1. Add these dependencies: ${installPackages(guide)}`,
+ "2. Set these env vars wherever the project loads config (.env, settings, deployment manifests):",
+ ` OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`,
+ ' OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"',
+ " OTEL_SERVICE_NAME=",
+ "3. Start the app through OTEL auto-instrumentation: opentelemetry-instrument .",
+ `4. Point every LLM client at LiteLLM: base_url=${proxyUrl}/v1, api key from LITELLM_API_KEY.`,
+ "5. Give each agent and subagent a name so runs are easy to read.",
+ "6. Run the agent once and confirm the run shows up in the LiteLLM UI under Logs > Agent Traces.",
+ "",
+ "Never hardcode the key. Read it from LITELLM_API_KEY.",
+ ].join("\n");
+
+export const PROXY_CONFIG_SNIPPET = [
+ "general_settings:",
+ " tracing:",
+ " store: clickhouse",
+ "",
+ "# env: CLICKHOUSE_URL (writer) and CLICKHOUSE_READER_URL (read-only user)",
+].join("\n");
+
+function CodeBlock({ code, tabs, wrap = false }: { code: string; tabs?: React.ReactNode; wrap?: boolean }) {
+ const [copied, setCopied] = useState(false);
+ const copy = async () => {
+ await copyToClipboard(code);
+ setCopied(true);
+ window.setTimeout(() => setCopied(false), COPIED_RESET_MS);
+ };
+ return (
+
+
+ {tabs}
+
+
+
+ {code}
+
+
+ );
+}
+
+function FileLabel({ children }: { children: React.ReactNode }) {
+ return {children};
+}
+
+function LineTabs({
+ value,
+ options,
+ onChange,
+}: {
+ value: T;
+ options: T[];
+ onChange: (v: T) => void;
+}) {
+ return (
+
+ {options.map((option) => (
+
+ ))}
+
+ );
+}
+
+function Step({ title, children }: { title: string; children: React.ReactNode }) {
+ return (
+
+ );
+}
+
+function SetupStatus({ detail, connected }: { detail: string | null; connected: boolean }) {
+ const pill = "inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[12px] text-foreground";
+ const hint = "text-[13px] text-muted-foreground";
+ if (connected) {
+ return (
+ <>
+
+ Receiving traces
+
+ Add another agent: point its OpenTelemetry exporter at LiteLLM.
+ >
+ );
+ }
+ if (detail === null) {
+ return (
+ <>
+
+ Waiting for traces…
+
+ No traces detected yet. Follow our guide to start tracing your application.
+ >
+ );
+ }
+ return (
+ <>
+ Tracing is not enabled
+ Turn on tracing in the proxy config, then point your agent at LiteLLM.
+ >
+ );
+}
+
+function WhatYoullSee() {
+ return (
+
+ What you'll see
+
+

+
+
+ {HIGHLIGHTS.map(([title, body]) => (
+
+ ))}
+
+
+ );
+}
+
+function FrameworkPicker({ value, onChange }: { value: string; onChange: (id: string) => void }) {
+ return (
+
+ {FRAMEWORKS.map((f) => (
+
+ ))}
+
+ );
+}
+
+/**
+ * Agent Traces onboarding: shown until the first trace arrives (and when tracing isn't enabled on the proxy).
+ * `detail` is the proxy's 501 message when tracing is off; null means tracing is on and we're waiting.
+ */
+export function TracingSetupCard({ detail, connected = false }: { detail: string | null; connected?: boolean }) {
+ const proxyUrl = getProxyBaseUrl().replace(/\/$/, "");
+ const [framework, setFramework] = useState(FRAMEWORKS[0].id);
+ const [installer, setInstaller] = useState("pip");
+ const guide = FRAMEWORKS.find((f) => f.id === framework) ?? FRAMEWORKS[0];
+ const packages = installPackages(guide);
+ const install = installer === "pip" ? `pip install -U ${packages}` : `uv add ${packages}`;
+
+ return (
+
+
+
+
+
+
+
+
+ Send your agent's OpenTelemetry traces to LiteLLM
+
+
+ Standard OTLP. Pick your framework, set three env vars, and run your agent as usual.
+
+
+
+ {detail !== null && (
+
+ config.yaml} />
+
+ )}
+
+
+
+ Paste this into your coding agent from the project root, or follow the steps below by hand.
+
+ Prompt} />
+
+
+
+ }
+ />
+
+
+
+ Shell} />
+
+
+
+ my_agent.py}
+ />
+
+ Shell} />
+
+
+
+
+
+ );
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json
new file mode 100644
index 00000000000..810bbb024a0
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json
@@ -0,0 +1,2056 @@
+{
+ "summary": {
+ "trace_id": "4bad42b84e9de3ba46fc870185f8f023",
+ "name": "deep_research_agent",
+ "service": "agent-demo",
+ "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?",
+ "start_time": "2026-09-30T04:36:29.377138Z",
+ "duration_ms": 51385.449,
+ "status": "ok",
+ "span_count": 126,
+ "agent_count": 2,
+ "llm_calls": 7,
+ "tool_calls": 26,
+ "error_count": 0,
+ "input_tokens": 30175,
+ "output_tokens": 2620,
+ "models": ["claude-sonnet-4-5"],
+ "agent_invocations": 2
+ },
+ "agents": [
+ {
+ "name": "deep_research_agent",
+ "parent_agent": null,
+ "invocations": 1,
+ "llm_calls": 2,
+ "tool_calls": 2,
+ "duration_ms": 51385.449
+ },
+ {
+ "name": "researcher",
+ "parent_agent": "deep_research_agent",
+ "invocations": 1,
+ "llm_calls": 5,
+ "tool_calls": 24,
+ "duration_ms": 35175.278
+ }
+ ],
+ "spans": [
+ {
+ "span_id": "5e79f3b5b504985e",
+ "parent_span_id": null,
+ "name": "deep_research_agent",
+ "type": "agent",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 0.0,
+ "duration_ms": 51385.449,
+ "status": "ok",
+ "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f6e5c97125fa4e7d",
+ "parent_span_id": "5e79f3b5b504985e",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 0.538,
+ "duration_ms": 0.089,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "83451f3235847f6c",
+ "parent_span_id": "5e79f3b5b504985e",
+ "name": "model",
+ "type": "chain",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 0.743,
+ "duration_ms": 9518.225,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cf04e1aa03f344fa",
+ "parent_span_id": "83451f3235847f6c",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 1.892,
+ "duration_ms": 9516.701,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e491eb7ac3b968ae",
+ "parent_span_id": "cf04e1aa03f344fa",
+ "name": "SubAgentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 2.113,
+ "duration_ms": 9516.359,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "56ebca946acf9ccb",
+ "parent_span_id": "e491eb7ac3b968ae",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 2.243,
+ "duration_ms": 9515.82,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1458b553bf3f622b",
+ "parent_span_id": "56ebca946acf9ccb",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 5.493,
+ "duration_ms": 9512.377,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1dfaf70fdd1184f2",
+ "parent_span_id": "1458b553bf3f622b",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 5.669,
+ "duration_ms": 9511.945,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8a6a1c31940d07af",
+ "parent_span_id": "1dfaf70fdd1184f2",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 6.07,
+ "duration_ms": 9510.777,
+ "status": "ok",
+ "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3332,
+ "output_tokens": 467,
+ "litellm_request_id": "chatcmpl-4077bb36-9380-4a3b-9481-245700cef09a",
+ "error": null
+ },
+ {
+ "span_id": "f6fdd164d528fee5",
+ "parent_span_id": "5e79f3b5b504985e",
+ "name": "tools",
+ "type": "chain",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9519.796,
+ "duration_ms": 2.662,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "33c7678eb38a8670",
+ "parent_span_id": "f6fdd164d528fee5",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9520.719,
+ "duration_ms": 1.361,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1526d46d48d29a09",
+ "parent_span_id": "33c7678eb38a8670",
+ "name": "write_file",
+ "type": "tool",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9521.356,
+ "duration_ms": 0.622,
+ "status": "ok",
+ "input_preview": "{\"file_path\": \"/tmp/research_todos.md\", \"content\": \"# Research Plan: ClickHouse vs Postgres for OTEL Spans (50k/sec)\\n\\n## Tasks\\n- [ ] Research ClickHouse and Postgres capabilities for high-volume ti\u2026",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2697122e295d91b6",
+ "parent_span_id": "5e79f3b5b504985e",
+ "name": "tools",
+ "type": "chain",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9522.666,
+ "duration_ms": 35177.678,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "56def7c7e192434a",
+ "parent_span_id": "2697122e295d91b6",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9523.385,
+ "duration_ms": 35176.632,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b2fb3a8f5a2fce01",
+ "parent_span_id": "56def7c7e192434a",
+ "name": "task",
+ "type": "tool",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 9523.758,
+ "duration_ms": 35176.06,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\": \"researcher\", \"description\": \"Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\\n\\nFocus on:\\n1. Write throughput c\u2026",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "81499b492fd93f85",
+ "parent_span_id": "b2fb3a8f5a2fce01",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 9524.284,
+ "duration_ms": 35175.278,
+ "status": "ok",
+ "input_preview": "Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\n\nFocus on:\n1. Write throughput capabilities - can each handle 50k spans/sec sustain\u2026",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "edabd9535a4b5eab",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 9525.39,
+ "duration_ms": 0.185,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "64a2c760897f3310",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 9525.814,
+ "duration_ms": 6071.296,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "beadbe4afce32dfa",
+ "parent_span_id": "64a2c760897f3310",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 9526.227,
+ "duration_ms": 6070.569,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6ff260ca90f57faf",
+ "parent_span_id": "beadbe4afce32dfa",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 9526.755,
+ "duration_ms": 6069.949,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9f92adb4d5271faf",
+ "parent_span_id": "6ff260ca90f57faf",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 9532.039,
+ "duration_ms": 6064.571,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b00edbc03c811428",
+ "parent_span_id": "9f92adb4d5271faf",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 9532.387,
+ "duration_ms": 6064.093,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3cebcc140afc30f9",
+ "parent_span_id": "b00edbc03c811428",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 9532.919,
+ "duration_ms": 6063.326,
+ "status": "ok",
+ "input_preview": "Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\n\nFocus on:\n1. Write throughput capabilities - can each handle 50k spans/sec sustain\u2026",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2986,
+ "output_tokens": 427,
+ "litellm_request_id": "chatcmpl-aaaa4b78-17dc-4364-8fa5-2d83369f1dbc",
+ "error": null
+ },
+ {
+ "span_id": "21e6ff34da1cdb18",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15598.294,
+ "duration_ms": 2.406,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4949beead378f935",
+ "parent_span_id": "21e6ff34da1cdb18",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15599.204,
+ "duration_ms": 1.048,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fe62f2ad03a0116c",
+ "parent_span_id": "4949beead378f935",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15599.584,
+ "duration_ms": 0.492,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse Postgres OpenTelemetry OTEL spans performance comparison\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "837c7d187c132fb3",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15600.875,
+ "duration_ms": 1.641,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "deccff55c4fd4e1c",
+ "parent_span_id": "837c7d187c132fb3",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15601.505,
+ "duration_ms": 0.71,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "83388a7eab791953",
+ "parent_span_id": "deccff55c4fd4e1c",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15601.783,
+ "duration_ms": 0.344,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse write throughput 50000 spans per second telemetry\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5a7c1909a7b6070a",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15602.648,
+ "duration_ms": 2.664,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fc789eb6dfefe6c3",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15602.886,
+ "duration_ms": 3.473,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8d63b242d67f1501",
+ "parent_span_id": "5a7c1909a7b6070a",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15603.582,
+ "duration_ms": 0.583,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "594d2d442764411c",
+ "parent_span_id": "8d63b242d67f1501",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15603.797,
+ "duration_ms": 0.284,
+ "status": "ok",
+ "input_preview": "{\"query\": \"Postgres time-series observability tracing performance\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "110d913a5ba8d617",
+ "parent_span_id": "fc789eb6dfefe6c3",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15604.55,
+ "duration_ms": 0.581,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b5b9908c8f4133c8",
+ "parent_span_id": "110d913a5ba8d617",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15604.775,
+ "duration_ms": 0.278,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse compression storage efficiency time-series data\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0c9973c7cc376699",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15605.598,
+ "duration_ms": 3.401,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ea13fd361da7da39",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15605.981,
+ "duration_ms": 3.366,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b0ef93a5f95ea00b",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15606.532,
+ "duration_ms": 3.926,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b4133562d7f3f1c4",
+ "parent_span_id": "0c9973c7cc376699",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15607.106,
+ "duration_ms": 0.695,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "994b1cf39376a0e3",
+ "parent_span_id": "b4133562d7f3f1c4",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15607.354,
+ "duration_ms": 0.347,
+ "status": "ok",
+ "input_preview": "{\"query\": \"OpenTelemetry trace data storage backend ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f884a82a7cac90da",
+ "parent_span_id": "ea13fd361da7da39",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15608.205,
+ "duration_ms": 0.598,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fcc7087fe59f84ce",
+ "parent_span_id": "f884a82a7cac90da",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15608.424,
+ "duration_ms": 0.297,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse columnar storage OLAP query performance aggregations\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "298c83b12c06a0a3",
+ "parent_span_id": "b0ef93a5f95ea00b",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15609.655,
+ "duration_ms": 0.533,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "602ed4ff76e7243f",
+ "parent_span_id": "298c83b12c06a0a3",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15609.865,
+ "duration_ms": 0.256,
+ "status": "ok",
+ "input_preview": "{\"query\": \"Postgres vs ClickHouse observability metrics traces\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aef73f54fb954970",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15610.576,
+ "duration_ms": 1.089,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0f257b3a9aa1a26a",
+ "parent_span_id": "aef73f54fb954970",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15610.952,
+ "duration_ms": 0.479,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8d20153626cba902",
+ "parent_span_id": "0f257b3a9aa1a26a",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15611.125,
+ "duration_ms": 0.234,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse insert performance batch writes sustained throughput\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "05114d7d3c246e6b",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15612.076,
+ "duration_ms": 4879.181,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "95e74cb656606e14",
+ "parent_span_id": "05114d7d3c246e6b",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15612.405,
+ "duration_ms": 4878.621,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d67fe2f9eb270591",
+ "parent_span_id": "95e74cb656606e14",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15612.747,
+ "duration_ms": 4878.212,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5205f67bffb4617b",
+ "parent_span_id": "d67fe2f9eb270591",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15614.444,
+ "duration_ms": 4876.442,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b60e34f522a4a74d",
+ "parent_span_id": "5205f67bffb4617b",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15614.655,
+ "duration_ms": 4876.123,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "319a549ffff70b5b",
+ "parent_span_id": "b60e34f522a4a74d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 15615.144,
+ "duration_ms": 4875.436,
+ "status": "ok",
+ "input_preview": "ClickHouse ingests 1M+ rows/s per node with batched inserts; use MergeTree ordered by (tenant, service, time) and a bloom filter index on TraceId.",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 4013,
+ "output_tokens": 297,
+ "litellm_request_id": "chatcmpl-09ff3e21-7ac8-452f-bc08-71651da0ffee",
+ "error": null
+ },
+ {
+ "span_id": "d9175dbeb8cf2bf4",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20492.092,
+ "duration_ms": 3.862,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9711d7caa64f983b",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20492.471,
+ "duration_ms": 3.823,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "81f6605116388677",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20492.78,
+ "duration_ms": 4.79,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "338e443ef6992f98",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20493.062,
+ "duration_ms": 6.429,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e286dc1a2580db1d",
+ "parent_span_id": "d9175dbeb8cf2bf4",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20493.764,
+ "duration_ms": 0.767,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9c0a8b4cd5e79898",
+ "parent_span_id": "e286dc1a2580db1d",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20494.085,
+ "duration_ms": 0.346,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse data compression ratio time-series storage\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d364b99422266c18",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20494.615,
+ "duration_ms": 5.279,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ead94fe51eb865ca",
+ "parent_span_id": "9711d7caa64f983b",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20495.151,
+ "duration_ms": 0.621,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2f556851946544ca",
+ "parent_span_id": "ead94fe51eb865ca",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20495.39,
+ "duration_ms": 0.304,
+ "status": "ok",
+ "input_preview": "{\"query\": \"trace reconstruction query performance TraceId filtering\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ca3e2f0e602c4f22",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20496.423,
+ "duration_ms": 4.49,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "545a060525cee8b9",
+ "parent_span_id": "81f6605116388677",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20496.822,
+ "duration_ms": 0.502,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "561c6db3f6b4a1b4",
+ "parent_span_id": "545a060525cee8b9",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20497.01,
+ "duration_ms": 0.245,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse operational complexity maintenance resource requirements\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "59eaeb0a2148a4fc",
+ "parent_span_id": "338e443ef6992f98",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20497.905,
+ "duration_ms": 0.477,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bf7f4ce484795c2d",
+ "parent_span_id": "59eaeb0a2148a4fc",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20498.081,
+ "duration_ms": 0.235,
+ "status": "ok",
+ "input_preview": "{\"query\": \"Postgres TimescaleDB hypertables time-series partitioning\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7be82d50e4acd54d",
+ "parent_span_id": "d364b99422266c18",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20498.729,
+ "duration_ms": 0.536,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cea1504e1f9ca159",
+ "parent_span_id": "7be82d50e4acd54d",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20498.933,
+ "duration_ms": 0.258,
+ "status": "ok",
+ "input_preview": "{\"query\": \"observability backend Jaeger Tempo Signoz storage engine\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f2dc8e9e8a205d6d",
+ "parent_span_id": "ca3e2f0e602c4f22",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20500.169,
+ "duration_ms": 0.483,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a3221021b0534e6f",
+ "parent_span_id": "f2dc8e9e8a205d6d",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 20500.348,
+ "duration_ms": 0.236,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse MergeTree TTL retention policy data lifecycle\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ea7b4372ee86b40b",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 20501.344,
+ "duration_ms": 4146.99,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "32e693f696ee8e09",
+ "parent_span_id": "ea7b4372ee86b40b",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20501.773,
+ "duration_ms": 4144.753,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6c2857333d95a7b5",
+ "parent_span_id": "32e693f696ee8e09",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20502.135,
+ "duration_ms": 4143.95,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dc46587f667ba574",
+ "parent_span_id": "6c2857333d95a7b5",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20503.637,
+ "duration_ms": 4142.004,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c1ad676c00916e76",
+ "parent_span_id": "dc46587f667ba574",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 20503.86,
+ "duration_ms": 4141.488,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ac941208ba750376",
+ "parent_span_id": "c1ad676c00916e76",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 20504.499,
+ "duration_ms": 4139.454,
+ "status": "ok",
+ "input_preview": "ClickHouse ingests 1M+ rows/s per node with batched inserts; use MergeTree ordered by (tenant, service, time) and a bloom filter index on TraceId.",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 4634,
+ "output_tokens": 246,
+ "litellm_request_id": "chatcmpl-67cda920-0fea-4e56-8e37-d1bb2061bb0c",
+ "error": null
+ },
+ {
+ "span_id": "c08547669a2510b2",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24652.82,
+ "duration_ms": 13.612,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bd63f327de3102e9",
+ "parent_span_id": "c08547669a2510b2",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24655.114,
+ "duration_ms": 2.154,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5145888189f26430",
+ "parent_span_id": "bd63f327de3102e9",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 24656.316,
+ "duration_ms": 0.713,
+ "status": "ok",
+ "input_preview": "{\"query\": \"columnar database analytical queries aggregation performance\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cf5c2a5726620c6d",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24657.803,
+ "duration_ms": 7.245,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b66b214e1eed463c",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24658.91,
+ "duration_ms": 9.157,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "54a17e39aa2da7bc",
+ "parent_span_id": "cf5c2a5726620c6d",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24661.804,
+ "duration_ms": 1.678,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "90d5ea4b66156997",
+ "parent_span_id": "54a17e39aa2da7bc",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 24662.798,
+ "duration_ms": 0.54,
+ "status": "ok",
+ "input_preview": "{\"query\": \"OLTP vs OLAP database workload write-heavy time-series\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2eeee520cb394aae",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24663.682,
+ "duration_ms": 5.76,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9b38b070edfc6711",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24664.352,
+ "duration_ms": 5.445,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "71e43febd60525c1",
+ "parent_span_id": "b66b214e1eed463c",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24665.558,
+ "duration_ms": 0.741,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9afc16768c8fd206",
+ "parent_span_id": "71e43febd60525c1",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 24665.846,
+ "duration_ms": 0.354,
+ "status": "ok",
+ "input_preview": "{\"query\": \"bloom filter index TraceId span lookup performance\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "27a60dcb575d4570",
+ "parent_span_id": "2eeee520cb394aae",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24667.07,
+ "duration_ms": 0.771,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "24aef47986e5ea47",
+ "parent_span_id": "27a60dcb575d4570",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 24667.372,
+ "duration_ms": 0.365,
+ "status": "ok",
+ "input_preview": "{\"query\": \"Postgres vacuum bloat write amplification high insert rate\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7d90328211921de8",
+ "parent_span_id": "9b38b070edfc6711",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24668.608,
+ "duration_ms": 0.63,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "15ff1df26a3bdd19",
+ "parent_span_id": "7d90328211921de8",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 24668.855,
+ "duration_ms": 0.3,
+ "status": "ok",
+ "input_preview": "{\"query\": \"batched inserts bulk loading streaming data ingestion\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "22935852b890a769",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 24670.265,
+ "duration_ms": 4373.43,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "46edf02452728f7b",
+ "parent_span_id": "22935852b890a769",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24670.767,
+ "duration_ms": 4372.593,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e405386748bc7fc6",
+ "parent_span_id": "46edf02452728f7b",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24671.171,
+ "duration_ms": 4372.11,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ea469e74ff5f8b33",
+ "parent_span_id": "e405386748bc7fc6",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24673.157,
+ "duration_ms": 4370.031,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e8d172adbef06ea4",
+ "parent_span_id": "ea469e74ff5f8b33",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 24673.434,
+ "duration_ms": 4369.627,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dd0a07cd84bbac0c",
+ "parent_span_id": "e8d172adbef06ea4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 24674.097,
+ "duration_ms": 4368.626,
+ "status": "ok",
+ "input_preview": "No results.",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 5104,
+ "output_tokens": 242,
+ "litellm_request_id": "chatcmpl-706f846d-b28a-40e4-8db8-c5bbd9f29eb2",
+ "error": null
+ },
+ {
+ "span_id": "809cf9e03fa89682",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29044.691,
+ "duration_ms": 4.366,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7059f01a979b7c48",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29045.222,
+ "duration_ms": 4.334,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "da423a9f87165ead",
+ "parent_span_id": "809cf9e03fa89682",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29045.774,
+ "duration_ms": 0.843,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bd0135c8a0b822ab",
+ "parent_span_id": "da423a9f87165ead",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 29046.098,
+ "duration_ms": 0.393,
+ "status": "ok",
+ "input_preview": "{\"query\": \"ClickHouse real-world production use cases metrics logging\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2078d337f17556b2",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29046.713,
+ "duration_ms": 4.328,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5e01965b501bce97",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29047.0,
+ "duration_ms": 6.192,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f9984abf3a97e304",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29047.506,
+ "duration_ms": 5.967,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7fffb63a4b4681fa",
+ "parent_span_id": "7059f01a979b7c48",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29048.086,
+ "duration_ms": 0.729,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "64a68e4bab8bfadd",
+ "parent_span_id": "7fffb63a4b4681fa",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 29048.364,
+ "duration_ms": 0.352,
+ "status": "ok",
+ "input_preview": "{\"query\": \"time-range filtering query optimization index performance\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e0a57007e35f8e0a",
+ "parent_span_id": "2078d337f17556b2",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29049.983,
+ "duration_ms": 0.742,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "32743c0d0d2e322d",
+ "parent_span_id": "e0a57007e35f8e0a",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 29050.268,
+ "duration_ms": 0.36,
+ "status": "ok",
+ "input_preview": "{\"query\": \"storage compression LZ4 ZSTD codec efficiency\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "250398aa6572e9c5",
+ "parent_span_id": "5e01965b501bce97",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29051.476,
+ "duration_ms": 0.604,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7da2a2672db0d368",
+ "parent_span_id": "250398aa6572e9c5",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 29051.705,
+ "duration_ms": 0.294,
+ "status": "ok",
+ "input_preview": "{\"query\": \"distributed tracing backend database selection criteria\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "433f566b47440ac3",
+ "parent_span_id": "f9984abf3a97e304",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29052.424,
+ "duration_ms": 0.583,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cba353fa39655c62",
+ "parent_span_id": "433f566b47440ac3",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 29052.646,
+ "duration_ms": 0.28,
+ "status": "ok",
+ "input_preview": "{\"query\": \"resource requirements memory CPU disk IOPS high throughput\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1c1ffab6777cfd82",
+ "parent_span_id": "81499b492fd93f85",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 29053.891,
+ "duration_ms": 15645.011,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "82c4159bd2b62f0d",
+ "parent_span_id": "1c1ffab6777cfd82",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29054.329,
+ "duration_ms": 15644.333,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5e6739c7831893cc",
+ "parent_span_id": "82c4159bd2b62f0d",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29054.676,
+ "duration_ms": 15643.931,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f3890ceb4757c0e3",
+ "parent_span_id": "5e6739c7831893cc",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29056.46,
+ "duration_ms": 15642.083,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b9af74d13c4cd463",
+ "parent_span_id": "f3890ceb4757c0e3",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 29056.707,
+ "duration_ms": 15641.734,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "163d3c4f27176d45",
+ "parent_span_id": "b9af74d13c4cd463",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 29057.391,
+ "duration_ms": 15640.826,
+ "status": "ok",
+ "input_preview": "No results.",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 5536,
+ "output_tokens": 700,
+ "litellm_request_id": "chatcmpl-3c20e808-9b16-4427-994f-5ce7e8b1b94d",
+ "error": null
+ },
+ {
+ "span_id": "4eacac6e510c033e",
+ "parent_span_id": "5e79f3b5b504985e",
+ "name": "model",
+ "type": "chain",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44700.902,
+ "duration_ms": 6684.266,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "42f62d678007a842",
+ "parent_span_id": "4eacac6e510c033e",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44701.401,
+ "duration_ms": 6683.636,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "98f3151c49a85f4f",
+ "parent_span_id": "42f62d678007a842",
+ "name": "SubAgentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44701.808,
+ "duration_ms": 6683.194,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6e9a065125a09f98",
+ "parent_span_id": "98f3151c49a85f4f",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44702.068,
+ "duration_ms": 6682.9,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7ed1526630e248a6",
+ "parent_span_id": "6e9a065125a09f98",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44704.008,
+ "duration_ms": 6680.917,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "100f2545e642b97b",
+ "parent_span_id": "7ed1526630e248a6",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44704.268,
+ "duration_ms": 6680.599,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ecf35e7f10d8428a",
+ "parent_span_id": "100f2545e642b97b",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "deep_research_agent",
+ "start_offset_ms": 44704.789,
+ "duration_ms": 6679.937,
+ "status": "ok",
+ "input_preview": "Based on my research, here's a comprehensive comparison of **ClickHouse vs Postgres** for storing OpenTelemetry spans at 50,000 spans/second:\n\n## **1. Write Throughput Capabilities**\n\n\u2022 **ClickHouse**\u2026",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 4570,
+ "output_tokens": 241,
+ "litellm_request_id": "chatcmpl-f26ccb45-ab1b-44c6-bd9e-41a02ca5f14d",
+ "error": null
+ }
+ ]
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json
new file mode 100644
index 00000000000..021755dcc70
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json
@@ -0,0 +1,3504 @@
+{
+ "summary": {
+ "trace_id": "e309a123963901e74c29cd2d3c86ff9e",
+ "name": "research_lead",
+ "service": "research-agent",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]",
+ "start_time": "2026-09-30T06:43:54.291000+00:00",
+ "duration_ms": 40198.10688,
+ "status": "ok",
+ "span_count": 216,
+ "agent_count": 3,
+ "llm_calls": 21,
+ "tool_calls": 25,
+ "error_count": 0,
+ "input_tokens": 69506,
+ "output_tokens": 2960,
+ "models": ["claude-sonnet-4-5"],
+ "agent_invocations": 3
+ },
+ "agents": [
+ {
+ "name": "research_lead",
+ "parent_agent": null,
+ "invocations": 1,
+ "llm_calls": 3,
+ "tool_calls": 5,
+ "duration_ms": 40198.10688
+ },
+ {
+ "name": "researcher",
+ "parent_agent": "research_lead",
+ "invocations": 4,
+ "llm_calls": 17,
+ "tool_calls": 20,
+ "duration_ms": 41634.618112
+ },
+ {
+ "name": "critic",
+ "parent_agent": "research_lead",
+ "invocations": 1,
+ "llm_calls": 1,
+ "tool_calls": 0,
+ "duration_ms": 6897.236224
+ }
+ ],
+ "spans": [
+ {
+ "span_id": "3586edf49d446541",
+ "parent_span_id": null,
+ "name": "research_lead",
+ "type": "agent",
+ "agent": "research_lead",
+ "start_offset_ms": 0.0,
+ "duration_ms": 40198.10688,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "21583099385b82ce",
+ "parent_span_id": "3586edf49d446541",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 0.457984,
+ "duration_ms": 0.14208,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4e01052d4b06c859",
+ "parent_span_id": "3586edf49d446541",
+ "name": "model",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 0.71808,
+ "duration_ms": 8289.062912,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"}],\"files\":{}}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "586bc4a78e64db27",
+ "parent_span_id": "4e01052d4b06c859",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 1.564928,
+ "duration_ms": 8288.058112,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9e1c1bda9a35de71",
+ "parent_span_id": "586bc4a78e64db27",
+ "name": "SubAgentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 1.81504,
+ "duration_ms": 8287.769856,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7664dbfe677a9c5e",
+ "parent_span_id": "9e1c1bda9a35de71",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 1.959936,
+ "duration_ms": 8287.579904,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "45c4bd55d643e5d3",
+ "parent_span_id": "7664dbfe677a9c5e",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 4.811008,
+ "duration_ms": 8284.677888,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ae58781d8e2e70eb",
+ "parent_span_id": "45c4bd55d643e5d3",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 4.976896,
+ "duration_ms": 8284.438016,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2c51e6ddab97ed21",
+ "parent_span_id": "ae58781d8e2e70eb",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "research_lead",
+ "start_offset_ms": 5.357056,
+ "duration_ms": 8283.9168,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3378,
+ "output_tokens": 514,
+ "litellm_request_id": "chatcmpl-028008eb-a34b-4132-97c2-526b707e961e",
+ "error": null
+ },
+ {
+ "span_id": "b632d56816eccdae",
+ "parent_span_id": "3586edf49d446541",
+ "name": "tools",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 8290.258176,
+ "duration_ms": 6625.607936,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTe",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dcf2045cfb34c569",
+ "parent_span_id": "b632d56816eccdae",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 8290.758912,
+ "duration_ms": 6624.966912,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "53e84f1d468bc2a6",
+ "parent_span_id": "dcf2045cfb34c569",
+ "name": "task",
+ "type": "tool",
+ "agent": "research_lead",
+ "start_offset_ms": 8290.994944,
+ "duration_ms": 6624.634112,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8febe34dc2549d84",
+ "parent_span_id": "53e84f1d468bc2a6",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 8291.454976,
+ "duration_ms": 6624.08704,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 5",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cddd94e6c89deef9",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8291.94496,
+ "duration_ms": 0.247808,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1f76f6dea3bfdd04",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 8292.339968,
+ "duration_ms": 3149.536,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 50k in",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "410375c7f58f1f43",
+ "parent_span_id": "1f76f6dea3bfdd04",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8292.577024,
+ "duration_ms": 3149.182976,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bae8d0d50862f54b",
+ "parent_span_id": "410375c7f58f1f43",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8292.935936,
+ "duration_ms": 3148.79104,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "311aefedd3c6be21",
+ "parent_span_id": "bae8d0d50862f54b",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8296.100096,
+ "duration_ms": 3145.58976,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0ad9d77b910af51d",
+ "parent_span_id": "311aefedd3c6be21",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8296.327936,
+ "duration_ms": 3145.309952,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7e5df9f2d820e2e6",
+ "parent_span_id": "0ad9d77b910af51d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 8296.672,
+ "duration_ms": 3144.837888,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of Cl",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2890,
+ "output_tokens": 175,
+ "litellm_request_id": "chatcmpl-7a706b7b-02ef-466c-92d7-41a78c9e4fd2",
+ "error": null
+ },
+ {
+ "span_id": "5ea0111d986e34ad",
+ "parent_span_id": "3586edf49d446541",
+ "name": "tools",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 8297.884928,
+ "duration_ms": 10856.022016,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5fcad8b07c383cca",
+ "parent_span_id": "5ea0111d986e34ad",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 8298.398976,
+ "duration_ms": 10855.315968,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c9d82ef1972cee03",
+ "parent_span_id": "5fcad8b07c383cca",
+ "name": "task",
+ "type": "tool",
+ "agent": "research_lead",
+ "start_offset_ms": 8298.578944,
+ "duration_ms": 10855.035904,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Re",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d0e6fec2e9ef645f",
+ "parent_span_id": "c9d82ef1972cee03",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 8298.937856,
+ "duration_ms": 10854.58304,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ae518ba9a91ed090",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8299.330816,
+ "duration_ms": 0.09216,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b99c05775f3efc9a",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 8299.545856,
+ "duration_ms": 1781.35808,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f0296e7c6e5e1cae",
+ "parent_span_id": "b99c05775f3efc9a",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8299.73504,
+ "duration_ms": 1780.889088,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "afe678420ec73b49",
+ "parent_span_id": "f0296e7c6e5e1cae",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8299.952896,
+ "duration_ms": 1780.59904,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "639fe351c3e68fb2",
+ "parent_span_id": "afe678420ec73b49",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8300.909824,
+ "duration_ms": 1779.555072,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "80cbae3d4b5b835d",
+ "parent_span_id": "639fe351c3e68fb2",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8301.693952,
+ "duration_ms": 1778.65216,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a1ecd455b3c8a492",
+ "parent_span_id": "80cbae3d4b5b835d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 8301.971968,
+ "duration_ms": 1778.086144,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2878,
+ "output_tokens": 72,
+ "litellm_request_id": "chatcmpl-64493c16-aed8-4abf-92cc-5aa6753a3ea4",
+ "error": null
+ },
+ {
+ "span_id": "4017c649a1b8a6de",
+ "parent_span_id": "3586edf49d446541",
+ "name": "tools",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 8303.038976,
+ "duration_ms": 11230.483968,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggreg",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8a0f7ae876128076",
+ "parent_span_id": "4017c649a1b8a6de",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 8303.544832,
+ "duration_ms": 11229.657344,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cb62778d79cec332",
+ "parent_span_id": "8a0f7ae876128076",
+ "name": "task",
+ "type": "tool",
+ "agent": "research_lead",
+ "start_offset_ms": 8303.700992,
+ "duration_ms": 11229.302016,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filte",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8d9dc2aaa6c8f3a3",
+ "parent_span_id": "cb62778d79cec332",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 8303.95776,
+ "duration_ms": 11228.84608,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "38938f77ba141238",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8334.684928,
+ "duration_ms": 0.130048,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "58fe37b65d9f7a86",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 8334.936064,
+ "duration_ms": 1766.2848,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d1433394ac5bdb00",
+ "parent_span_id": "58fe37b65d9f7a86",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8335.143936,
+ "duration_ms": 1765.91104,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f75d1be9ac1947d3",
+ "parent_span_id": "d1433394ac5bdb00",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8335.355904,
+ "duration_ms": 1765.654016,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d5c084f9ab6e23e3",
+ "parent_span_id": "f75d1be9ac1947d3",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8336.306944,
+ "duration_ms": 1764.64384,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5d718e2115861da6",
+ "parent_span_id": "d5c084f9ab6e23e3",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8336.446976,
+ "duration_ms": 1764.434944,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "eefd457b75d7c70a",
+ "parent_span_id": "5d718e2115861da6",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 8336.706048,
+ "duration_ms": 1764.03584,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2881,
+ "output_tokens": 70,
+ "litellm_request_id": "chatcmpl-2b5813a1-a4d0-4abe-afee-d60951430963",
+ "error": null
+ },
+ {
+ "span_id": "25aadc9977ff9c18",
+ "parent_span_id": "3586edf49d446541",
+ "name": "tools",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 8337.646848,
+ "duration_ms": 12928.301056,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volum",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "26c7d32fd6431206",
+ "parent_span_id": "25aadc9977ff9c18",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 8338.075904,
+ "duration_ms": 12927.710208,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fa754e1794d2f80e",
+ "parent_span_id": "26c7d32fd6431206",
+ "name": "task",
+ "type": "tool",
+ "agent": "research_lead",
+ "start_offset_ms": 8338.21184,
+ "duration_ms": 12927.47008,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry da",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c2f4c93af99a664c",
+ "parent_span_id": "fa754e1794d2f80e",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 8338.432,
+ "duration_ms": 12927.101952,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bu",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7958498c8902047d",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8338.735104,
+ "duration_ms": 0.070912,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "225f39859d0b1797",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 8338.904064,
+ "duration_ms": 3404.400896,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f091b4570ff57cfc",
+ "parent_span_id": "225f39859d0b1797",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8339.051008,
+ "duration_ms": 3404.1408,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e906eb238cb8e5d0",
+ "parent_span_id": "f091b4570ff57cfc",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8339.220992,
+ "duration_ms": 3403.942144,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "26181ec6f6aa4e5e",
+ "parent_span_id": "e906eb238cb8e5d0",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8340.04096,
+ "duration_ms": 3403.091968,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5057524339d490c5",
+ "parent_span_id": "26181ec6f6aa4e5e",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 8340.16512,
+ "duration_ms": 3402.922752,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "11290db27f2eeea0",
+ "parent_span_id": "5057524339d490c5",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 8340.397056,
+ "duration_ms": 3402.59584,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2875,
+ "output_tokens": 197,
+ "litellm_request_id": "chatcmpl-cd4dcc18-4665-43f0-b22c-4685fec2aa33",
+ "error": null
+ },
+ {
+ "span_id": "d9dad6ec7330b73f",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 10081.412096,
+ "duration_ms": 2.178816,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres write performance time-series data ingestion 50k inserts per second high-volume\"},\"id\":\"toolu_017EZMv245fwxq2BWyzieBhq\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "96abed5e22e54c83",
+ "parent_span_id": "d9dad6ec7330b73f",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10082.205952,
+ "duration_ms": 1.025024,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ca8488012a54b761",
+ "parent_span_id": "96abed5e22e54c83",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 10082.61504,
+ "duration_ms": 0.475904,
+ "status": "ok",
+ "input_preview": "{\"query\":\"Postgres write performance time-series data ingestion 50k inserts per second high-volume\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0cde1462b20eb64e",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 10083.95904,
+ "duration_ms": 1856.2368,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "09bcb03638055e6f",
+ "parent_span_id": "0cde1462b20eb64e",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10084.428032,
+ "duration_ms": 1855.6608,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5f6058ccf4810d98",
+ "parent_span_id": "09bcb03638055e6f",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10084.84992,
+ "duration_ms": 1855.212032,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3c01c335af52bfaa",
+ "parent_span_id": "5f6058ccf4810d98",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10086.747136,
+ "duration_ms": 1853.28384,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0d72911998165ec9",
+ "parent_span_id": "3c01c335af52bfaa",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10087.028992,
+ "duration_ms": 1852.96,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "82056db608f1c211",
+ "parent_span_id": "0d72911998165ec9",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 10087.582976,
+ "duration_ms": 1852.304896,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2985,
+ "output_tokens": 65,
+ "litellm_request_id": "chatcmpl-81dd6157-0aaf-4d19-835a-6dcee93884e8",
+ "error": null
+ },
+ {
+ "span_id": "d7401760f3f5a01e",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 10101.520896,
+ "duration_ms": 1.287936,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"OpenTelemetry span data access patterns query requirements trace aggregation time-range filtering\"},\"id\":\"toolu_01Sa3MPe67WdezvSNxYVCTZ7\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "85f208624a903f82",
+ "parent_span_id": "d7401760f3f5a01e",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10102.009856,
+ "duration_ms": 0.573184,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "72fcc3295390f481",
+ "parent_span_id": "85f208624a903f82",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 10102.22592,
+ "duration_ms": 0.272128,
+ "status": "ok",
+ "input_preview": "{\"query\":\"OpenTelemetry span data access patterns query requirements trace aggregation time-range filtering\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bd75dbd12044c1c3",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 10103.049984,
+ "duration_ms": 1676.137984,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "87a3f5bcdc02321a",
+ "parent_span_id": "bd75dbd12044c1c3",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10103.357952,
+ "duration_ms": 1675.725824,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fefc0d7b6a5d355d",
+ "parent_span_id": "87a3f5bcdc02321a",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10103.67872,
+ "duration_ms": 1675.377152,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bae2312368589d86",
+ "parent_span_id": "fefc0d7b6a5d355d",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10105.19808,
+ "duration_ms": 1673.82784,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "044e406836b59c6d",
+ "parent_span_id": "bae2312368589d86",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 10105.406976,
+ "duration_ms": 1673.577984,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "26fd0196ea969838",
+ "parent_span_id": "044e406836b59c6d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 10105.803776,
+ "duration_ms": 1673.087232,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2966,
+ "output_tokens": 59,
+ "litellm_request_id": "chatcmpl-320edff6-4f0b-4d67-b6e5-539c68ca2a82",
+ "error": null
+ },
+ {
+ "span_id": "7d66952eb3149ff8",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11442.199808,
+ "duration_ms": 1.049088,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse write performance time-series OpenTelemetry spans ingestion rate 50k inserts per second\"},\"id\":\"toolu_01Dg4aLbkY4pXiQMWew5maNs\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2accd598b191224f",
+ "parent_span_id": "7d66952eb3149ff8",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11442.521856,
+ "duration_ms": 0.385024,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "910bd68064a40985",
+ "parent_span_id": "2accd598b191224f",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11442.671872,
+ "duration_ms": 0.179968,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse write performance time-series OpenTelemetry spans ingestion rate 50k inserts per second\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3e3b69eab1b21370",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11443.020032,
+ "duration_ms": 1.074944,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse high volume writes benchmarks throughput inserts per second\"},\"id\":\"toolu_016UXr44rs5u7ZLpy7cSoVnd\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f68c7136c22a27c0",
+ "parent_span_id": "3e3b69eab1b21370",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11443.524864,
+ "duration_ms": 0.391168,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ec44997ed84dd458",
+ "parent_span_id": "f68c7136c22a27c0",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11443.681792,
+ "duration_ms": 0.182272,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse high volume writes benchmarks throughput inserts per second\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "89230f61ca3d25bb",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11444.18304,
+ "duration_ms": 0.926976,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse time-series data performance characteristics batch inserts\"},\"id\":\"toolu_01EmCdBSeJKEZt1A8vprJDcf\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b8323e5ea41e7d44",
+ "parent_span_id": "89230f61ca3d25bb",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11444.523008,
+ "duration_ms": 0.352,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a5c73b39062741a5",
+ "parent_span_id": "b8323e5ea41e7d44",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11444.66304,
+ "duration_ms": 0.160768,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse time-series data performance characteristics batch inserts\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c503d0b6702f171d",
+ "parent_span_id": "8febe34dc2549d84",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11445.348864,
+ "duration_ms": 3469.96608,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 50k in",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1aa2a868683855dc",
+ "parent_span_id": "c503d0b6702f171d",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11445.566976,
+ "duration_ms": 3469.622784,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6fa486e85cc4b885",
+ "parent_span_id": "1aa2a868683855dc",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11445.743872,
+ "duration_ms": 3469.415936,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a85cb07902b96d84",
+ "parent_span_id": "6fa486e85cc4b885",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11446.690048,
+ "duration_ms": 3468.435968,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b5f3affc5eac5183",
+ "parent_span_id": "a85cb07902b96d84",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11446.808064,
+ "duration_ms": 3468.271872,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "89af13298c71e8d3",
+ "parent_span_id": "b5f3affc5eac5183",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 11447.05792,
+ "duration_ms": 3467.904,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of Cl",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3279,
+ "output_tokens": 113,
+ "litellm_request_id": "chatcmpl-fa113ca8-9194-443e-b5b1-d009979ae7a8",
+ "error": null
+ },
+ {
+ "span_id": "f0e6ba4d76959b0f",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11743.6288,
+ "duration_ms": 1.161216,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse Postgres storage efficiency compression observability telemetry data retention\"},\"id\":\"toolu_01LpCf52iSMhb4hPvohUvKXo\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dbdf1fe23f0b7608",
+ "parent_span_id": "f0e6ba4d76959b0f",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11743.971072,
+ "duration_ms": 0.377856,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "89478eee5f16f7bc",
+ "parent_span_id": "dbdf1fe23f0b7608",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11744.114944,
+ "duration_ms": 0.179968,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse Postgres storage efficiency compression observability telemetry data retention\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c7a58fd2d402ca41",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11744.491008,
+ "duration_ms": 1.030912,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse compression storage footprint column-oriented telemetry\"},\"id\":\"toolu_01Ac1WiKauEA8EJTfYRx8xnu\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4e57e56745c075dc",
+ "parent_span_id": "c7a58fd2d402ca41",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11745.036032,
+ "duration_ms": 0.356864,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "332e55f64c2cf833",
+ "parent_span_id": "4e57e56745c075dc",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11745.176832,
+ "duration_ms": 0.168192,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse compression storage footprint column-oriented telemetry\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ae7ebc9a1d85d06f",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11745.652992,
+ "duration_ms": 0.724736,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres ClickHouse data retention TTL observability metrics logs\"},\"id\":\"toolu_019cQbrLgYaBHvDPNiHQKk8q\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "30534e0e14e0976d",
+ "parent_span_id": "ae7ebc9a1d85d06f",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11745.918976,
+ "duration_ms": 0.348928,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7135e0cf258b90c2",
+ "parent_span_id": "30534e0e14e0976d",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11746.05184,
+ "duration_ms": 0.167168,
+ "status": "ok",
+ "input_preview": "{\"query\":\"Postgres ClickHouse data retention TTL observability metrics logs\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ce0587c4a969adab",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11746.582016,
+ "duration_ms": 2849.200896,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d4d8a5bfedbb5dff",
+ "parent_span_id": "ce0587c4a969adab",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11746.767872,
+ "duration_ms": 2848.839936,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ffed87b10acccc61",
+ "parent_span_id": "d4d8a5bfedbb5dff",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11746.927872,
+ "duration_ms": 2848.633088,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "08d0d119b42f65cf",
+ "parent_span_id": "ffed87b10acccc61",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11747.862016,
+ "duration_ms": 2847.648,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "32947733f303bef9",
+ "parent_span_id": "08d0d119b42f65cf",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11747.977984,
+ "duration_ms": 2847.454976,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a5139c89995770e0",
+ "parent_span_id": "32947733f303bef9",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 11748.226048,
+ "duration_ms": 2847.051776,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3302,
+ "output_tokens": 162,
+ "litellm_request_id": "chatcmpl-ba8dcfcc-6ce3-48d4-88a1-46e07f2e3a5b",
+ "error": null
+ },
+ {
+ "span_id": "ba1b8bea84bb3419",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11779.385088,
+ "duration_ms": 0.870912,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"span data query patterns trace search filtering\"},\"id\":\"toolu_017Hv7XgHkS9cYQRvEGwK6zF\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "74635fe5c444a65f",
+ "parent_span_id": "ba1b8bea84bb3419",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11779.719936,
+ "duration_ms": 0.380928,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "941f5d334364eba3",
+ "parent_span_id": "74635fe5c444a65f",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11779.874048,
+ "duration_ms": 0.171776,
+ "status": "ok",
+ "input_preview": "{\"query\":\"span data query patterns trace search filtering\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e664750f1fd77b25",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11780.430848,
+ "duration_ms": 1400.856064,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "71fc36d2515d599f",
+ "parent_span_id": "e664750f1fd77b25",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11780.673792,
+ "duration_ms": 1400.436224,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1b100724c30e4584",
+ "parent_span_id": "71fc36d2515d599f",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11780.86784,
+ "duration_ms": 1400.175104,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "248a4bb6d8182204",
+ "parent_span_id": "1b100724c30e4584",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11781.968128,
+ "duration_ms": 1399.019776,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aa46be42752e8a7d",
+ "parent_span_id": "248a4bb6d8182204",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11782.093824,
+ "duration_ms": 1398.818048,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aa10c9eeb130add5",
+ "parent_span_id": "aa46be42752e8a7d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 11782.33984,
+ "duration_ms": 1398.417152,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3039,
+ "output_tokens": 61,
+ "litellm_request_id": "chatcmpl-f24284ae-daf1-4848-b58c-16939577b86a",
+ "error": null
+ },
+ {
+ "span_id": "ff9c60db21b2487f",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11940.40192,
+ "duration_ms": 0.845824,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"PostgreSQL insert performance throughput limitations batch inserts time-series\"},\"id\":\"toolu_01UUhyrPeiB41ruKWhD5h7Lt\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b581e9dae9a8e65a",
+ "parent_span_id": "ff9c60db21b2487f",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11940.723968,
+ "duration_ms": 0.363008,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "99b3c54206ab172e",
+ "parent_span_id": "b581e9dae9a8e65a",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 11940.865792,
+ "duration_ms": 0.168192,
+ "status": "ok",
+ "input_preview": "{\"query\":\"PostgreSQL insert performance throughput limitations batch inserts time-series\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "674fbd4c084f1ae9",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 11941.427968,
+ "duration_ms": 1514.968064,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1b77ec92eb9e3df7",
+ "parent_span_id": "674fbd4c084f1ae9",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11941.666048,
+ "duration_ms": 1514.493952,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a6b0961929376ce4",
+ "parent_span_id": "1b77ec92eb9e3df7",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11941.878016,
+ "duration_ms": 1514.230016,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2bccf2bcd2554900",
+ "parent_span_id": "a6b0961929376ce4",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11942.840832,
+ "duration_ms": 1513.211136,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "47c68249883b82f6",
+ "parent_span_id": "2bccf2bcd2554900",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 11942.966784,
+ "duration_ms": 1513.001216,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f871e3307724bf1b",
+ "parent_span_id": "47c68249883b82f6",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 11943.206912,
+ "duration_ms": 1512.521984,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3084,
+ "output_tokens": 70,
+ "litellm_request_id": "chatcmpl-31172160-15f3-4a0b-bb95-f054c1f49c96",
+ "error": null
+ },
+ {
+ "span_id": "9ce1da40c0fb0cff",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 13181.638912,
+ "duration_ms": 1.458944,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"OpenTelemetry traces spans storage query\"},\"id\":\"toolu_01DLFewyFmJArEjsbHzEi9UK\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "44411aefd212169b",
+ "parent_span_id": "9ce1da40c0fb0cff",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13182.208,
+ "duration_ms": 0.637952,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f870964607787ed0",
+ "parent_span_id": "44411aefd212169b",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 13182.466048,
+ "duration_ms": 0.29184,
+ "status": "ok",
+ "input_preview": "{\"query\":\"OpenTelemetry traces spans storage query\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ba0687166794762c",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 13183.383808,
+ "duration_ms": 1531.270144,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2465b30b775cc0f9",
+ "parent_span_id": "ba0687166794762c",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13183.76704,
+ "duration_ms": 1530.689792,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d417013a93db586b",
+ "parent_span_id": "2465b30b775cc0f9",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13184.08704,
+ "duration_ms": 1530.317056,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a47137f00b51744c",
+ "parent_span_id": "d417013a93db586b",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13185.6,
+ "duration_ms": 1528.73984,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "33c55af3901c059f",
+ "parent_span_id": "a47137f00b51744c",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13185.821952,
+ "duration_ms": 1528.440832,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2bb5a3398293f7cc",
+ "parent_span_id": "33c55af3901c059f",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 13186.283008,
+ "duration_ms": 1527.795968,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3114,
+ "output_tokens": 58,
+ "litellm_request_id": "chatcmpl-46c76f65-1f14-43fd-a270-102c535f4306",
+ "error": null
+ },
+ {
+ "span_id": "620a349bde8eeae1",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 13456.779776,
+ "duration_ms": 1.46432,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres high throughput write workload bottlenecks WAL autovacuum\"},\"id\":\"toolu_01MaMP4TQi5nppnzwyLFRj63\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "41f7c2f062e8a80b",
+ "parent_span_id": "620a349bde8eeae1",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13457.309952,
+ "duration_ms": 0.667904,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f458cf58f874d59a",
+ "parent_span_id": "41f7c2f062e8a80b",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 13457.581824,
+ "duration_ms": 0.304128,
+ "status": "ok",
+ "input_preview": "{\"query\":\"Postgres high throughput write workload bottlenecks WAL autovacuum\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0bee3eae4b1a5a9b",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 13458.531072,
+ "duration_ms": 1662.05696,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ca15856d85c17d15",
+ "parent_span_id": "0bee3eae4b1a5a9b",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13458.91072,
+ "duration_ms": 1661.483264,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b62d3dd6fafc6bd2",
+ "parent_span_id": "ca15856d85c17d15",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13459.238912,
+ "duration_ms": 1661.106944,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "578ab81ae856aec8",
+ "parent_span_id": "b62d3dd6fafc6bd2",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13460.74496,
+ "duration_ms": 1659.54688,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6ad32c56b2a6e93c",
+ "parent_span_id": "578ab81ae856aec8",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 13460.962048,
+ "duration_ms": 1659.25504,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "87a70110c1b9ba4a",
+ "parent_span_id": "6ad32c56b2a6e93c",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 13461.531904,
+ "duration_ms": 1658.518016,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3188,
+ "output_tokens": 68,
+ "litellm_request_id": "chatcmpl-a8cf94e1-5991-44b7-bedf-099ebf7865a6",
+ "error": null
+ },
+ {
+ "span_id": "5ec46fcb9874c3f1",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14596.340992,
+ "duration_ms": 2.127104,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse storage compression ratio LZ4 ZSTD column storage\"},\"id\":\"toolu_01PPB1nanXw7YTgoyJ4A2UUT\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e96cd58675e125a2",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14596.64512,
+ "duration_ms": 3.00288,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse TTL data retention policy automatic deletion\"},\"id\":\"toolu_013HjnuDx4nHSW6MztW7Tg7q\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dfcf3f95f7122c63",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14596.90496,
+ "duration_ms": 3.590144,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres storage size disk space observability time-series data\"},\"id\":\"toolu_01NGvQyJtnkBpwya3cY992A9\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "912d3b24265d6516",
+ "parent_span_id": "5ec46fcb9874c3f1",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14597.38112,
+ "duration_ms": 0.86784,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2da674074a3fb829",
+ "parent_span_id": "912d3b24265d6516",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 14597.650944,
+ "duration_ms": 0.504064,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse storage compression ratio LZ4 ZSTD column storage\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "94864c972b7fbfa7",
+ "parent_span_id": "e96cd58675e125a2",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14598.811136,
+ "duration_ms": 0.539904,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "534787a0806803cb",
+ "parent_span_id": "94864c972b7fbfa7",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 14599.01312,
+ "duration_ms": 0.265984,
+ "status": "ok",
+ "input_preview": "{\"query\":\"ClickHouse TTL data retention policy automatic deletion\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e31ef2e09e031c06",
+ "parent_span_id": "dfcf3f95f7122c63",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14599.888896,
+ "duration_ms": 0.410112,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d416a275d48fd41b",
+ "parent_span_id": "e31ef2e09e031c06",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 14600.044032,
+ "duration_ms": 0.197888,
+ "status": "ok",
+ "input_preview": "{\"query\":\"Postgres storage size disk space observability time-series data\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3dc6317b14b8b60b",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14600.783872,
+ "duration_ms": 2737.964032,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c03864e0e6e0498b",
+ "parent_span_id": "3dc6317b14b8b60b",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14601.085952,
+ "duration_ms": 2737.357056,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e4f30d6cb14fd53b",
+ "parent_span_id": "c03864e0e6e0498b",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14601.357824,
+ "duration_ms": 2737.00992,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "78906777d7f40bef",
+ "parent_span_id": "e4f30d6cb14fd53b",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14602.676992,
+ "duration_ms": 2735.609088,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "453af7c9bf4368ba",
+ "parent_span_id": "78906777d7f40bef",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14602.868992,
+ "duration_ms": 2735.300096,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "93c56b7c2d751e44",
+ "parent_span_id": "453af7c9bf4368ba",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 14603.279872,
+ "duration_ms": 2734.586112,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3638,
+ "output_tokens": 103,
+ "litellm_request_id": "chatcmpl-5f57c315-3c80-41f2-9ecb-2dca0f03737b",
+ "error": null
+ },
+ {
+ "span_id": "e9e68bf9e4c120ec",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14715.046144,
+ "duration_ms": 1.582848,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"trace aggregation time series data\"},\"id\":\"toolu_01USMJKAZMEvh5zpkAYBL4pd\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8758a1659a817971",
+ "parent_span_id": "e9e68bf9e4c120ec",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14715.649024,
+ "duration_ms": 0.72704,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "486548b06249d120",
+ "parent_span_id": "8758a1659a817971",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 14715.938048,
+ "duration_ms": 0.336896,
+ "status": "ok",
+ "input_preview": "{\"query\":\"trace aggregation time series data\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "32756d86f68a95f8",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 14716.905984,
+ "duration_ms": 1941.266176,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "82913eb816195416",
+ "parent_span_id": "32756d86f68a95f8",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14717.276928,
+ "duration_ms": 1940.743936,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6952c979b8c040e2",
+ "parent_span_id": "82913eb816195416",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14717.60896,
+ "duration_ms": 1940.370944,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "472f7491e326c68c",
+ "parent_span_id": "6952c979b8c040e2",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14719.175936,
+ "duration_ms": 1938.759168,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0ce2ac29e47d07fb",
+ "parent_span_id": "472f7491e326c68c",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 14719.395072,
+ "duration_ms": 1938.475008,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f6db3994aabaf84e",
+ "parent_span_id": "0ce2ac29e47d07fb",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 14720.00896,
+ "duration_ms": 1937.707008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3186,
+ "output_tokens": 57,
+ "litellm_request_id": "chatcmpl-6d7b2d42-6476-46c2-b07f-b2cff372cc51",
+ "error": null
+ },
+ {
+ "span_id": "378ba6e4ded8b83f",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15120.956928,
+ "duration_ms": 1.57696,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"TimescaleDB Postgres time series performance optimization partitioning hypertables\"},\"id\":\"toolu_01JU1EhmJ5jQgDAvS9DhG8Yk\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "22e75b96f36ac970",
+ "parent_span_id": "378ba6e4ded8b83f",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15121.536768,
+ "duration_ms": 0.732416,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "32f6c2e979c945ad",
+ "parent_span_id": "22e75b96f36ac970",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 15121.831936,
+ "duration_ms": 0.33792,
+ "status": "ok",
+ "input_preview": "{\"query\":\"TimescaleDB Postgres time series performance optimization partitioning hypertables\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "76e48aa2e94e4608",
+ "parent_span_id": "d0e6fec2e9ef645f",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 15122.853888,
+ "duration_ms": 4030.482176,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5b4eb8c646e5ea38",
+ "parent_span_id": "76e48aa2e94e4608",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15123.289088,
+ "duration_ms": 4029.922048,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ecbf621d117d41ef",
+ "parent_span_id": "5b4eb8c646e5ea38",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15123.646976,
+ "duration_ms": 4029.532928,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8d411a01f0f40e17",
+ "parent_span_id": "ecbf621d117d41ef",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15125.184,
+ "duration_ms": 4027.95904,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "82c68ac8749083a6",
+ "parent_span_id": "8d411a01f0f40e17",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 15125.409024,
+ "duration_ms": 4027.684096,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ca5f50b3b8879759",
+ "parent_span_id": "82c68ac8749083a6",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 15125.856,
+ "duration_ms": 4027.105024,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3290,
+ "output_tokens": 124,
+ "litellm_request_id": "chatcmpl-c86e1321-87e9-4599-a46e-e23dd0005e78",
+ "error": null
+ },
+ {
+ "span_id": "2122f08271342ae9",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 16658.482944,
+ "duration_ms": 1.293056,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"span attributes filtering time range\"},\"id\":\"toolu_01AdKnUtJyMFY6vE1Q44ZmF7\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c0c1df069970a064",
+ "parent_span_id": "2122f08271342ae9",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 16658.987008,
+ "duration_ms": 0.587008,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1a0cea88f0e7fcab",
+ "parent_span_id": "c0c1df069970a064",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 16659.219968,
+ "duration_ms": 0.271104,
+ "status": "ok",
+ "input_preview": "{\"query\":\"span attributes filtering time range\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a7abcb9eb548822e",
+ "parent_span_id": "8d9dc2aaa6c8f3a3",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 16659.99104,
+ "duration_ms": 2872.426752,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4905e77b56824800",
+ "parent_span_id": "a7abcb9eb548822e",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 16660.3072,
+ "duration_ms": 2871.8528,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8c5744fb3a7b0b0b",
+ "parent_span_id": "4905e77b56824800",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 16660.57088,
+ "duration_ms": 2871.527168,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d6c5eb192e2e391f",
+ "parent_span_id": "8c5744fb3a7b0b0b",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 16661.803776,
+ "duration_ms": 2870.225408,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ce347268a1f62ffb",
+ "parent_span_id": "d6c5eb192e2e391f",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 16662.00192,
+ "duration_ms": 2869.926144,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a2c15c6ccb466632",
+ "parent_span_id": "ce347268a1f62ffb",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 16662.38208,
+ "duration_ms": 2869.247744,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3257,
+ "output_tokens": 124,
+ "litellm_request_id": "chatcmpl-37698b76-1b34-4414-adf1-fb9cdc40765e",
+ "error": null
+ },
+ {
+ "span_id": "2e507cfbc4fe5c42",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 17339.544064,
+ "duration_ms": 3.478016,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"column-oriented database compression benefits storage savings\"},\"id\":\"toolu_01FshP24x5QcUUgJ8J4V6Mff\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a313420c8b250d41",
+ "parent_span_id": "2e507cfbc4fe5c42",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17340.464128,
+ "duration_ms": 0.952832,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "35761025656d39a1",
+ "parent_span_id": "a313420c8b250d41",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 17340.865024,
+ "duration_ms": 0.422144,
+ "status": "ok",
+ "input_preview": "{\"query\":\"column-oriented database compression benefits storage savings\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f85b480c8efa8f56",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 17341.547776,
+ "duration_ms": 1.922304,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"time-series database storage efficiency retention management\"},\"id\":\"toolu_01U9JwQstrUzHJFWYA6PJah1\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1b4facb81cfb56fe",
+ "parent_span_id": "f85b480c8efa8f56",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17342.16576,
+ "duration_ms": 0.634368,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "48d97fb6a419e1d6",
+ "parent_span_id": "1b4facb81cfb56fe",
+ "name": "search_docs",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 17342.4128,
+ "duration_ms": 0.300288,
+ "status": "ok",
+ "input_preview": "{\"query\":\"time-series database storage efficiency retention management\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2d1062cce334c01b",
+ "parent_span_id": "c2f4c93af99a664c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 17343.940864,
+ "duration_ms": 3921.300224,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6a88e16cbd043f5e",
+ "parent_span_id": "2d1062cce334c01b",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17344.491008,
+ "duration_ms": 3920.632064,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d94f78d8de8ab127",
+ "parent_span_id": "6a88e16cbd043f5e",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17344.94976,
+ "duration_ms": 3920.13824,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e35e83849f3a70e9",
+ "parent_span_id": "d94f78d8de8ab127",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17347.136,
+ "duration_ms": 3917.87392,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "738d993abf7b7974",
+ "parent_span_id": "e35e83849f3a70e9",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "researcher",
+ "start_offset_ms": 17347.393792,
+ "duration_ms": 3917.497344,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a1e9092a218f684e",
+ "parent_span_id": "738d993abf7b7974",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 17347.923968,
+ "duration_ms": 3916.752128,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 3808,
+ "output_tokens": 148,
+ "litellm_request_id": "chatcmpl-d9fb74a4-58ad-4ae3-9d0b-21b06c6b5145",
+ "error": null
+ },
+ {
+ "span_id": "7d729d7fe5527e09",
+ "parent_span_id": "3586edf49d446541",
+ "name": "model",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 21266.221056,
+ "duration_ms": 8096.979968,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"},{\"content\":\"I'll help you deter",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e1c1e76ef490f593",
+ "parent_span_id": "7d729d7fe5527e09",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 21266.488832,
+ "duration_ms": 8096.516352,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5414fd0da858774d",
+ "parent_span_id": "e1c1e76ef490f593",
+ "name": "SubAgentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 21266.683904,
+ "duration_ms": 8096.268032,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dda7d2699a1778df",
+ "parent_span_id": "5414fd0da858774d",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 21266.811904,
+ "duration_ms": 8096.086016,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d7159bb909d27803",
+ "parent_span_id": "dda7d2699a1778df",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 21267.846912,
+ "duration_ms": 8094.988288,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "036bad1c98591c3b",
+ "parent_span_id": "d7159bb909d27803",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 21267.974912,
+ "duration_ms": 8094.769152,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a7c9fedd896e4ddc",
+ "parent_span_id": "036bad1c98591c3b",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "research_lead",
+ "start_offset_ms": 21268.27392,
+ "duration_ms": 8094.284032,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 4492,
+ "output_tokens": 388,
+ "litellm_request_id": "chatcmpl-6ca81bcb-a605-4067-87df-8b2ba13d34be",
+ "error": null
+ },
+ {
+ "span_id": "96021b36c22e3de5",
+ "parent_span_id": "3586edf49d446541",
+ "name": "tools",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 29363.598848,
+ "duration_ms": 6899.43424,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"critic\",\"description\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/se",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ec41b5953e07ddf6",
+ "parent_span_id": "96021b36c22e3de5",
+ "name": "FilesystemMiddleware.wrap_tool_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 29364.22016,
+ "duration_ms": 6898.466816,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aea43e4e80c09950",
+ "parent_span_id": "ec41b5953e07ddf6",
+ "name": "task",
+ "type": "tool",
+ "agent": "research_lead",
+ "start_offset_ms": 29364.556032,
+ "duration_ms": 6897.913856,
+ "status": "ok",
+ "input_preview": "{\"subagent_type\":\"critic\",\"description\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inser",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "db4e1f0a6675441c",
+ "parent_span_id": "aea43e4e80c09950",
+ "name": "critic",
+ "type": "agent",
+ "agent": "critic",
+ "start_offset_ms": 29365.08288,
+ "duration_ms": 6897.236224,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inserts, while s",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aecf500a40176e62",
+ "parent_span_id": "db4e1f0a6675441c",
+ "name": "PatchToolCallsMiddleware.before_agent",
+ "type": "framework",
+ "agent": "critic",
+ "start_offset_ms": 29365.89312,
+ "duration_ms": 0.16768,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8776ee8a5ec58f7a",
+ "parent_span_id": "db4e1f0a6675441c",
+ "name": "model",
+ "type": "chain",
+ "agent": "critic",
+ "start_offset_ms": 29366.297856,
+ "duration_ms": 6895.67616,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inserts, while standa",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "100d096f30b12ba7",
+ "parent_span_id": "8776ee8a5ec58f7a",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "critic",
+ "start_offset_ms": 29366.657024,
+ "duration_ms": 6895.030784,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d562a157f40ce8ec",
+ "parent_span_id": "100d096f30b12ba7",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "critic",
+ "start_offset_ms": 29367.105024,
+ "duration_ms": 6894.512896,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3539d8541097d392",
+ "parent_span_id": "d562a157f40ce8ec",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "critic",
+ "start_offset_ms": 29373.202944,
+ "duration_ms": 6888.335104,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2b1e5b780d6e0868",
+ "parent_span_id": "3539d8541097d392",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "critic",
+ "start_offset_ms": 29373.569024,
+ "duration_ms": 6887.84896,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7b0fd995de99bb3a",
+ "parent_span_id": "2b1e5b780d6e0868",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "critic",
+ "start_offset_ms": 29374.034944,
+ "duration_ms": 6887.087872,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Critique the draft in <=2 bullets. Do not use tools.\"}, {\"role\": \"user\", \"content\": \"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 2926,
+ "output_tokens": 162,
+ "litellm_request_id": "chatcmpl-51dc714f-2a85-4f89-aa60-d30e5ea3aff3",
+ "error": null
+ },
+ {
+ "span_id": "8693e8571891da82",
+ "parent_span_id": "3586edf49d446541",
+ "name": "model",
+ "type": "chain",
+ "agent": "research_lead",
+ "start_offset_ms": 36263.529984,
+ "duration_ms": 3934.251008,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"},{\"content\":\"I'll help you deter",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e8a056a3f5fa8064",
+ "parent_span_id": "8693e8571891da82",
+ "name": "FilesystemMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 36264.162816,
+ "duration_ms": 3933.501952,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ff113c9de01aa964",
+ "parent_span_id": "e8a056a3f5fa8064",
+ "name": "SubAgentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 36264.601856,
+ "duration_ms": 3933.031168,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bcdae244a30669fe",
+ "parent_span_id": "ff113c9de01aa964",
+ "name": "SummarizationMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 36264.86016,
+ "duration_ms": 3932.74496,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "17958e38044e83b1",
+ "parent_span_id": "bcdae244a30669fe",
+ "name": "AnthropicPromptCachingMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 36266.764032,
+ "duration_ms": 3930.806016,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8e18dafd8aaf0af4",
+ "parent_span_id": "17958e38044e83b1",
+ "name": "UnsupportedContentMiddleware.wrap_model_call",
+ "type": "framework",
+ "agent": "research_lead",
+ "start_offset_ms": 36267.02208,
+ "duration_ms": 3930.494976,
+ "status": "ok",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4942936a6ea8b578",
+ "parent_span_id": "8e18dafd8aaf0af4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "research_lead",
+ "start_offset_ms": 36267.64288,
+ "duration_ms": 3929.739008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 5050,
+ "output_tokens": 170,
+ "litellm_request_id": "chatcmpl-7531fb51-2940-4e7a-9cf4-ff071e171f6b",
+ "error": null
+ }
+ ]
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json
new file mode 100644
index 00000000000..83cb6c4ba79
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json
@@ -0,0 +1,3368 @@
+{
+ "summary": {
+ "trace_id": "0602f23f8fa5d3a2c1a12521739ca866",
+ "name": "orchestrator",
+ "service": "swarm-orchestrator",
+ "input_preview": "",
+ "start_time": "2026-09-30T06:48:02.550000+00:00",
+ "duration_ms": 17145.690112,
+ "status": "ok",
+ "span_count": 207,
+ "agent_count": 4,
+ "llm_calls": 53,
+ "tool_calls": 45,
+ "error_count": 23,
+ "input_tokens": 42351,
+ "output_tokens": 4356,
+ "models": ["claude-haiku-4-5", "claude-sonnet-4-5"],
+ "agent_invocations": 4
+ },
+ "agents": [
+ {
+ "name": "orchestrator",
+ "parent_agent": null,
+ "invocations": 1,
+ "llm_calls": 0,
+ "tool_calls": 0,
+ "duration_ms": 17145.690112
+ },
+ {
+ "name": "researcher",
+ "parent_agent": "orchestrator",
+ "invocations": 12,
+ "llm_calls": 40,
+ "tool_calls": 35,
+ "duration_ms": 43891.366656
+ },
+ {
+ "name": "fact_checker",
+ "parent_agent": "researcher",
+ "invocations": 2,
+ "llm_calls": 12,
+ "tool_calls": 10,
+ "duration_ms": 11556.862976
+ },
+ {
+ "name": "critic",
+ "parent_agent": "orchestrator",
+ "invocations": 1,
+ "llm_calls": 1,
+ "tool_calls": 0,
+ "duration_ms": 5460.011008
+ }
+ ],
+ "spans": [
+ {
+ "span_id": "b7b1053f6c95fd70",
+ "parent_span_id": null,
+ "name": "orchestrator",
+ "type": "agent",
+ "agent": "orchestrator",
+ "start_offset_ms": 0.0,
+ "duration_ms": 17145.690112,
+ "status": "ok",
+ "input_preview": "",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ffb1aafc123f7cbf",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 54.492928,
+ "duration_ms": 8857.8112,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare ingest throughput (question 0).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f977e58a340dbc6b",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 55.320064,
+ "duration_ms": 1161.488896,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "03f611cf9bfc9ea4",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 55.501056,
+ "duration_ms": 3226.066944,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9d124d6bce3185dd",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 55.905024,
+ "duration_ms": 1136.644096,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6fbea322f87ea515",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 56.007168,
+ "duration_ms": 3131.008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "2714afb89a510e4a",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 56.350976,
+ "duration_ms": 271.669248,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d23937c848659551",
+ "parent_span_id": "2714afb89a510e4a",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 56.684032,
+ "duration_ms": 192.15616,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare backups (question 7).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"ae6337ae-8401-4264-a955-836b6b09c7a3\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f060fb4662d99991",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 56.779264,
+ "duration_ms": 1167.341824,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare storage cost (question 1).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6ba3ba633024d9e4",
+ "parent_span_id": "f060fb4662d99991",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 57.080064,
+ "duration_ms": 1166.923008,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 1).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"01a2d57c-f807-42eb-bda0-ed3b3399a41b\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "93e72d093e644a1d",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 57.244928,
+ "duration_ms": 1178.156288,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4bc4c26013a33115",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 57.316096,
+ "duration_ms": 11627.85792,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare ingest throughput (question 10).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ebd769f68fb432b3",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 57.57312,
+ "duration_ms": 1486.716928,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "71903d779d129c68",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 57.648128,
+ "duration_ms": 2114.355968,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b33e0f6b6f8a6ef0",
+ "parent_span_id": "71903d779d129c68",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 57.89312,
+ "duration_ms": 188.8128,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a1e8b8b06038a09e",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 57.960192,
+ "duration_ms": 3083.454976,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8db5b2b7871b58a6",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 58.30016,
+ "duration_ms": 1105.966848,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "41fa112865f7b1f5",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 58.381056,
+ "duration_ms": 1185.950976,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "be4ac7802a570aa4",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 58.62912,
+ "duration_ms": 193.360896,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "042d555505c3a65a",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 58.695168,
+ "duration_ms": 3089.047808,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3e4cf4753e6183c1",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 58.941184,
+ "duration_ms": 1278.447872,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "49397f8d7856d2ba",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 59.060224,
+ "duration_ms": 3135.371776,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "991c19abc6a2001c",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 59.379968,
+ "duration_ms": 1142.242048,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d9786287209602ae",
+ "parent_span_id": "f977e58a340dbc6b",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 60.77312,
+ "duration_ms": 1155.918848,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 676,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-f4447165-878b-4bf3-ae07-0a43b98482e9",
+ "error": null
+ },
+ {
+ "span_id": "8a556ee09900bc3c",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "researcher",
+ "type": "agent",
+ "agent": "researcher",
+ "start_offset_ms": 60.95104,
+ "duration_ms": 3001.430016,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5afec084c27fe7e2",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 61.202176,
+ "duration_ms": 1148.244992,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8a62792e0140b33c",
+ "parent_span_id": "5afec084c27fe7e2",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 61.479168,
+ "duration_ms": 1147.847936,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 604,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-74c4de8a-56f5-41f9-8b61-ed64736ac886",
+ "error": null
+ },
+ {
+ "span_id": "61f05341d2eddb8a",
+ "parent_span_id": "9d124d6bce3185dd",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 61.836032,
+ "duration_ms": 1130.572032,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 607,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-e987a212-12de-404f-bf30-db1db933c1bc",
+ "error": null
+ },
+ {
+ "span_id": "c076a3e9e3b197ae",
+ "parent_span_id": "d23937c848659551",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 62.268928,
+ "duration_ms": 186.465024,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-e7cb41a6-6020-4cdc-9d8e-1516de91bc11",
+ "error": null
+ },
+ {
+ "span_id": "c076a3e9e3b197ae",
+ "parent_span_id": "d23937c848659551",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 62.268928,
+ "duration_ms": 186.465024,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-e7cb41a6-6020-4cdc-9d8e-1516de91bc11",
+ "error": null
+ },
+ {
+ "span_id": "638912f9c31b4936",
+ "parent_span_id": "ebd769f68fb432b3",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 62.682112,
+ "duration_ms": 1481.43616,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 676,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-5891711d-025f-4e93-b2f7-240a3f6b0c84",
+ "error": null
+ },
+ {
+ "span_id": "c124c17c8d44876e",
+ "parent_span_id": "be4ac7802a570aa4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 65.745152,
+ "duration_ms": 186.13376,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-613f5748-76a7-4b30-ac6a-4003a58f9490",
+ "error": null
+ },
+ {
+ "span_id": "c124c17c8d44876e",
+ "parent_span_id": "be4ac7802a570aa4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 65.745152,
+ "duration_ms": 186.13376,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-613f5748-76a7-4b30-ac6a-4003a58f9490",
+ "error": null
+ },
+ {
+ "span_id": "791584567230ec00",
+ "parent_span_id": "6ba3ba633024d9e4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 69.241088,
+ "duration_ms": 1154.654976,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 1).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 70,
+ "litellm_request_id": "chatcmpl-bb6a3f5d-8947-4b51-8bb0-4c8997e05bc1",
+ "error": null
+ },
+ {
+ "span_id": "99016550b1f4b1d8",
+ "parent_span_id": "8db5b2b7871b58a6",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 70.228224,
+ "duration_ms": 1093.913856,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-96f92920-92c9-4848-bf5c-30633117837c",
+ "error": null
+ },
+ {
+ "span_id": "663415cd473e52bd",
+ "parent_span_id": "93e72d093e644a1d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 71.22816,
+ "duration_ms": 1164.059904,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 604,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-efb08407-99c5-41e5-affc-e7d3f7298f41",
+ "error": null
+ },
+ {
+ "span_id": "f0ac4348db8afae0",
+ "parent_span_id": "991c19abc6a2001c",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 72.411136,
+ "duration_ms": 1129.085952,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 604,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-ec0d2784-5f4c-4b9f-b88e-50e3ca221f41",
+ "error": null
+ },
+ {
+ "span_id": "cdd663e40c465dd9",
+ "parent_span_id": "3e4cf4753e6183c1",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 73.424128,
+ "duration_ms": 1263.8208,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 605,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-e72cb04b-a5c7-4975-84de-cb4f9ea55573",
+ "error": null
+ },
+ {
+ "span_id": "f08c5c831bf4bd72",
+ "parent_span_id": "b33e0f6b6f8a6ef0",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 74.630144,
+ "duration_ms": 171.833856,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 606,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-0e56e75e-8d08-4423-a7e5-003a81ef6046",
+ "error": null
+ },
+ {
+ "span_id": "f08c5c831bf4bd72",
+ "parent_span_id": "b33e0f6b6f8a6ef0",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 74.630144,
+ "duration_ms": 171.833856,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 606,
+ "output_tokens": 120,
+ "litellm_request_id": "chatcmpl-0e56e75e-8d08-4423-a7e5-003a81ef6046",
+ "error": null
+ },
+ {
+ "span_id": "202780b67dcbe6ec",
+ "parent_span_id": "71903d779d129c68",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 247.129088,
+ "duration_ms": 1.22112,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"question 2\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01QpRUG3ZDA2iCbyd3R1bvu7\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a822744d88aa4631",
+ "parent_span_id": "202780b67dcbe6ec",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 247.52512,
+ "duration_ms": 0.240896,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"question 2\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4f4840b705cf0258",
+ "parent_span_id": "71903d779d129c68",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 248.105984,
+ "duration_ms": 9.818112,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01Y5NpMECPckqZiivRNbis8y\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f81d2982c71d49ab",
+ "parent_span_id": "4f4840b705cf0258",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 249.108992,
+ "duration_ms": 5.188096,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8da3c27f759a9a70",
+ "parent_span_id": "2714afb89a510e4a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 249.403136,
+ "duration_ms": 0.691968,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01NsgVkiKqRHCMy4TCqCAdwo\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1ae5b620a479299e",
+ "parent_span_id": "8da3c27f759a9a70",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 249.64992,
+ "duration_ms": 0.270336,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"backups\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0e78db2488523ae2",
+ "parent_span_id": "2714afb89a510e4a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 250.228224,
+ "duration_ms": 4.8448,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\"},\"id\":\"toolu_01VbdVXDtw8UTMViJ7gokomj\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6e60f7c6e87a7d8e",
+ "parent_span_id": "0e78db2488523ae2",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 250.409984,
+ "duration_ms": 4.124928,
+ "status": "error",
+ "input_preview": "{\"topic\":\"backups\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ec9dd8a7195a53c4",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 252.23808,
+ "duration_ms": 0.653056,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"replication\",\"store\":\"ClickHouse\"},\"id\":\"toolu_018LjjywXXsaTX4fFcyoWDMc\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3aaf7251e50f38b0",
+ "parent_span_id": "ec9dd8a7195a53c4",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 252.502016,
+ "duration_ms": 0.173056,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"replication\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0f63b2ed7ede23d9",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 253.079296,
+ "duration_ms": 1.87264,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01AFJaxyTcM2JitwHJCbMX9x\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "008eaf94b0a32842",
+ "parent_span_id": "0f63b2ed7ede23d9",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 253.319936,
+ "duration_ms": 1.388032,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "31780bf2416808de",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 255.284992,
+ "duration_ms": 66.174208,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"},{\"content\":\"I'll look up the replication benchmarks for both ClickHouse an",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "58ca5d41dba8391d",
+ "parent_span_id": "31780bf2416808de",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 255.536128,
+ "duration_ms": 65.780736,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 831,
+ "output_tokens": 86,
+ "litellm_request_id": "chatcmpl-95594677-f719-46bb-b5ac-4120df774a00",
+ "error": null
+ },
+ {
+ "span_id": "58ca5d41dba8391d",
+ "parent_span_id": "31780bf2416808de",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 255.536128,
+ "duration_ms": 65.780736,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 831,
+ "output_tokens": 86,
+ "litellm_request_id": "chatcmpl-95594677-f719-46bb-b5ac-4120df774a00",
+ "error": null
+ },
+ {
+ "span_id": "7c54980e8bad2655",
+ "parent_span_id": "2714afb89a510e4a",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 256.829952,
+ "duration_ms": 61.04704,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare backups (question 7).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"ae6337ae-8401-4264-a955-836b6b09c7a3\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "315fe6ce2f65b09c",
+ "parent_span_id": "7c54980e8bad2655",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 257.037312,
+ "duration_ms": 60.630784,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}, {\"role\": \"assistant\", \"content\": \"\", \"tool_",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 833,
+ "output_tokens": 84,
+ "litellm_request_id": "chatcmpl-24edbb47-a7f5-403c-80c0-88ec1eac0dd7",
+ "error": null
+ },
+ {
+ "span_id": "315fe6ce2f65b09c",
+ "parent_span_id": "7c54980e8bad2655",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 257.037312,
+ "duration_ms": 60.630784,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}, {\"role\": \"assistant\", \"content\": \"\", \"tool_",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 833,
+ "output_tokens": 84,
+ "litellm_request_id": "chatcmpl-24edbb47-a7f5-403c-80c0-88ec1eac0dd7",
+ "error": null
+ },
+ {
+ "span_id": "f3645483ea551b89",
+ "parent_span_id": "71903d779d129c68",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 258.110976,
+ "duration_ms": 872.749056,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"},{\"content\":\"I'll look up the query latency benchmarks for both ClickHous",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1ea79fa9c6374836",
+ "parent_span_id": "f3645483ea551b89",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 258.290176,
+ "duration_ms": 872.399104,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 835,
+ "output_tokens": 82,
+ "litellm_request_id": "chatcmpl-c4972857-6f99-4aba-9840-9a8eb8919371",
+ "error": null
+ },
+ {
+ "span_id": "c7afe4c441a0fd73",
+ "parent_span_id": "2714afb89a510e4a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 318.098176,
+ "duration_ms": 6.99904,
+ "status": "error",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\",\"store\":\"Postgres\"},\"id\":\"toolu_01G9NCc3ydqBTtP2oL5Dm6Sa\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "285a73be5eeb1854",
+ "parent_span_id": "c7afe4c441a0fd73",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 318.509056,
+ "duration_ms": 0.802048,
+ "status": "error",
+ "input_preview": "{\"topic\":\"backups\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "efd995d7612509bf",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 321.657088,
+ "duration_ms": 0.692992,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"replication\",\"store\":\"Postgres\"},\"id\":\"toolu_019ZnuDWZ4iesRCHsqAJbDR5\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "829d07185a7ad5ae",
+ "parent_span_id": "efd995d7612509bf",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 321.92512,
+ "duration_ms": 0.188928,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"replication\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4a3842bd69cae949",
+ "parent_span_id": "41fa112865f7b1f5",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 322.51904,
+ "duration_ms": 921.635072,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"},{\"content\":\"I'll look up the replication benchmarks for both ClickHouse an",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8645585126f22d7b",
+ "parent_span_id": "4a3842bd69cae949",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 322.769152,
+ "duration_ms": 921.25696,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 941,
+ "output_tokens": 49,
+ "litellm_request_id": "chatcmpl-0990e3ad-175e-46df-bf19-68177c9e426a",
+ "error": null
+ },
+ {
+ "span_id": "cb2c4553753e56f5",
+ "parent_span_id": "71903d779d129c68",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1131.075072,
+ "duration_ms": 0.651008,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"question 2\",\"store\":\"Postgres\"},\"id\":\"toolu_01G3oiAoW5NPsSL2eJCagf1U\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8b32aaa41ba4f1df",
+ "parent_span_id": "cb2c4553753e56f5",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1131.366912,
+ "duration_ms": 0.198144,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"question 2\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "915aa192d636e089",
+ "parent_span_id": "71903d779d129c68",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1131.901184,
+ "duration_ms": 1039.888128,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"},{\"content\":\"I'll look up the query latency benchmarks for both ClickHous",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "be4cad07067533b9",
+ "parent_span_id": "915aa192d636e089",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1132.164096,
+ "duration_ms": 1039.467008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 942,
+ "output_tokens": 48,
+ "litellm_request_id": "chatcmpl-86ca494e-bc4a-4c54-ad78-b90745af11b0",
+ "error": null
+ },
+ {
+ "span_id": "09fab5aee6d32ab2",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1164.51712,
+ "duration_ms": 0.731904,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"storage cost\",\"store\":\"ClickHouse\"},\"id\":\"toolu_018MuiQt9fmjtqVz2RXCHZhy\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b43b5e1d2cd57e76",
+ "parent_span_id": "09fab5aee6d32ab2",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1164.958976,
+ "duration_ms": 0.16896,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"storage cost\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c97b01c60c5d1217",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1165.370112,
+ "duration_ms": 0.771072,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01HD8LARpprgDitPW6nfTZZe\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8a3858ffa83cf12f",
+ "parent_span_id": "c97b01c60c5d1217",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1165.632,
+ "duration_ms": 0.303104,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "53b54096466355c5",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1166.361088,
+ "duration_ms": 1261.767936,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"},{\"content\":\"I'll look up the storage cost benchmark for both ClickHouse ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "336aad6fad874893",
+ "parent_span_id": "53b54096466355c5",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1166.622208,
+ "duration_ms": 1261.346816,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 830,
+ "output_tokens": 84,
+ "litellm_request_id": "chatcmpl-e2f2e18d-7b17-4c6f-8bd0-5acffbf7e3ef",
+ "error": null
+ },
+ {
+ "span_id": "319e1d5201f7ecf8",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1192.827136,
+ "duration_ms": 0.879872,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"multi-tenancy\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01KWSWhiKjDWaLmoVLm9NXB6\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "33dc6bb0071e9f68",
+ "parent_span_id": "319e1d5201f7ecf8",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1193.089024,
+ "duration_ms": 0.297984,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"multi-tenancy\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e8c538553372ff0e",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1193.542144,
+ "duration_ms": 0.775936,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01Bof3pVkxMr4giscKi69THD\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8f066d65f6e6ba90",
+ "parent_span_id": "e8c538553372ff0e",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1193.885184,
+ "duration_ms": 0.267008,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1fed547ebbc3c231",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1194.514944,
+ "duration_ms": 1037.73824,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"},{\"content\":\"I'll look up the multi-tenancy benchmarks for both ClickHous",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "72f543f3390d665f",
+ "parent_span_id": "1fed547ebbc3c231",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1194.735104,
+ "duration_ms": 1037.377024,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 839,
+ "output_tokens": 85,
+ "litellm_request_id": "chatcmpl-1c04a496-4e08-4531-8ae1-3751e17e97bf",
+ "error": null
+ },
+ {
+ "span_id": "ef050fdc5fcbc5f6",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1201.884928,
+ "duration_ms": 0.799232,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"retention (question 3)\"},\"id\":\"toolu_014st8iy9P8SuxXiZMkh32tw\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8104b956bb5589ab",
+ "parent_span_id": "ef050fdc5fcbc5f6",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1202.150144,
+ "duration_ms": 0.17792,
+ "status": "ok",
+ "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"retention (question 3)\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e94883ee4a0748f4",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1202.44608,
+ "duration_ms": 0.976128,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01PhBp6nuzjb7wbTvVD6r53e\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "86ce9dc920022f5c",
+ "parent_span_id": "e94883ee4a0748f4",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1202.829056,
+ "duration_ms": 0.418048,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "98a82b26135e44b8",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1203.599104,
+ "duration_ms": 1051.271936,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"},{\"content\":\"I'll look up the retention benchmark (question 3) for both Click",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "57be1131bf87528d",
+ "parent_span_id": "98a82b26135e44b8",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1203.835904,
+ "duration_ms": 1050.923264,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}, {\"role\": \"assistant\", \"content\": \"I'll lo",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 839,
+ "output_tokens": 87,
+ "litellm_request_id": "chatcmpl-d78f00eb-0f7e-4b29-a1c1-91a709f357de",
+ "error": null
+ },
+ {
+ "span_id": "e2a9435a1c14cea3",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1209.68704,
+ "duration_ms": 0.747008,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"joins\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01RvNzdrWuJkVnodMpMZSLAz\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4f93b6871b48ecef",
+ "parent_span_id": "e2a9435a1c14cea3",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1209.918976,
+ "duration_ms": 0.168192,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"joins\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "cf5525d2d0f3040e",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1210.278912,
+ "duration_ms": 0.760064,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01K4qPmXFz6qj3LLDsvPhcNy\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "059af97bec295cf0",
+ "parent_span_id": "cf5525d2d0f3040e",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1210.6112,
+ "duration_ms": 0.24704,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7c3f6bcdea7df18c",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1211.208192,
+ "duration_ms": 968.805888,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"},{\"content\":\"I'll look up the benchmark data for joins on both ClickHouse and Pos",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "66575f940e1d8d9f",
+ "parent_span_id": "7c3f6bcdea7df18c",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1212.324096,
+ "duration_ms": 967.556096,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}, {\"role\": \"assistant\", \"content\": \"I'll look u",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 828,
+ "output_tokens": 83,
+ "litellm_request_id": "chatcmpl-563f2658-f904-44af-875d-9a9643ea50ee",
+ "error": null
+ },
+ {
+ "span_id": "ee0f8aad942106ae",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1217.080064,
+ "duration_ms": 0.70784,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput question 0\"},\"id\":\"toolu_01UaLUswR7HVY93frdKhzHX4\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "95db7339488d7721",
+ "parent_span_id": "ee0f8aad942106ae",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1217.318144,
+ "duration_ms": 0.175872,
+ "status": "ok",
+ "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput question 0\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "398d8a15feeb0db9",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1217.599232,
+ "duration_ms": 0.760832,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\"},\"id\":\"toolu_012iYU68ZdNkQXxbA7ka5GvP\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "98e38544cc4ed2a8",
+ "parent_span_id": "398d8a15feeb0db9",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1217.953024,
+ "duration_ms": 0.250112,
+ "status": "error",
+ "input_preview": "{\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6f3d60ffdf7ef9ce",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1218.537216,
+ "duration_ms": 1044.436736,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0e34ca6259e3e8c0",
+ "parent_span_id": "6f3d60ffdf7ef9ce",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1218.773248,
+ "duration_ms": 1044.088832,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 914,
+ "output_tokens": 89,
+ "litellm_request_id": "chatcmpl-46e407e8-e0c7-45ad-96bc-8f61270a6640",
+ "error": null
+ },
+ {
+ "span_id": "e6f3479907471837",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1235.660032,
+ "duration_ms": 0.740096,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"compression\"},\"id\":\"toolu_01R6ayWaGRufB8hhTzz2eCBk\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "53dd0c2ef37cc475",
+ "parent_span_id": "e6f3479907471837",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1235.916288,
+ "duration_ms": 0.177664,
+ "status": "ok",
+ "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"compression\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "37bc9ac680a894b3",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1236.209152,
+ "duration_ms": 0.708864,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01MVe4SLPi34HqRLCu3Cj9rH\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "84306979d3b3f910",
+ "parent_span_id": "37bc9ac680a894b3",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1236.538112,
+ "duration_ms": 0.229888,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a8dded09bd8531ad",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1237.088,
+ "duration_ms": 972.432128,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"},{\"content\":\"I'll look up the compression benchmarks for both ClickHouse an",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0ebb8092f3f97294",
+ "parent_span_id": "a8dded09bd8531ad",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1237.296128,
+ "duration_ms": 972.062976,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}, {\"role\": \"assistant\", \"content\": \"I'll ",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 827,
+ "output_tokens": 79,
+ "litellm_request_id": "chatcmpl-b710911c-5c59-4010-a305-eaa1f2567f31",
+ "error": null
+ },
+ {
+ "span_id": "c204916b6acf6662",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1337.686016,
+ "duration_ms": 0.606208,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"schema changes\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01BSaAngjGswgo1rfiMqBR38\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "62ff82b718533251",
+ "parent_span_id": "c204916b6acf6662",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1337.97504,
+ "duration_ms": 0.193024,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"schema changes\",\"store\":\"ClickHouse\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7f81614e4ddc7a77",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1338.407936,
+ "duration_ms": 0.702208,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01TY1tyXwoQU12aQXrddEs4v\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5e82032b7c3fb5e3",
+ "parent_span_id": "7f81614e4ddc7a77",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1338.639104,
+ "duration_ms": 0.268032,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4fa3cdf571fc30fb",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1339.326208,
+ "duration_ms": 907.485952,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"},{\"content\":\"I'll look up the benchmark data for schema changes on both ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fa68e281815fde27",
+ "parent_span_id": "4fa3cdf571fc30fb",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1339.57504,
+ "duration_ms": 907.12704,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}, {\"role\": \"assistant\", \"content\": \"I'",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 832,
+ "output_tokens": 80,
+ "litellm_request_id": "chatcmpl-afe091b4-18b5-49c6-8092-4160d185abf2",
+ "error": null
+ },
+ {
+ "span_id": "0f3c8c6d1b95b8f7",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1544.611072,
+ "duration_ms": 0.68992,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput\"},\"id\":\"toolu_017XDB7FAud42UVNwNFAYaDo\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "845d9db17364829c",
+ "parent_span_id": "0f3c8c6d1b95b8f7",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1544.907008,
+ "duration_ms": 0.197888,
+ "status": "ok",
+ "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d84e2d5d80a1e5cd",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1545.382912,
+ "duration_ms": 0.96128,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_016U8j45zbsYtH3UYf3DL51N\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "dfe4e1fd79458a7d",
+ "parent_span_id": "d84e2d5d80a1e5cd",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 1545.853184,
+ "duration_ms": 0.299008,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1f1988c9693fede7",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 1546.571008,
+ "duration_ms": 980.190976,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e46d723f65245490",
+ "parent_span_id": "1f1988c9693fede7",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 1546.854144,
+ "duration_ms": 979.710976,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 912,
+ "output_tokens": 86,
+ "litellm_request_id": "chatcmpl-44ba88cc-2a57-4159-baf0-4ff9963f0c5c",
+ "error": null
+ },
+ {
+ "span_id": "eb1eea92dffec096",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2180.201984,
+ "duration_ms": 0.648192,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"joins\",\"store\":\"Postgres\"},\"id\":\"toolu_01JHhKgKTUs2WS7qtKM3wCj1\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3ca66f2ec7e25124",
+ "parent_span_id": "eb1eea92dffec096",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2180.49408,
+ "duration_ms": 0.21504,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"joins\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1012810e08a39c3a",
+ "parent_span_id": "8a556ee09900bc3c",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2180.987136,
+ "duration_ms": 881.167872,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"},{\"content\":\"I'll look up the benchmark data for joins on both ClickHouse and Pos",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "debcd76428458d09",
+ "parent_span_id": "1012810e08a39c3a",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2181.210112,
+ "duration_ms": 880.774912,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}, {\"role\": \"assistant\", \"content\": \"I'll look u",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 934,
+ "output_tokens": 48,
+ "litellm_request_id": "chatcmpl-7a5abd7c-f49a-4f64-8d45-b2c839c1cbde",
+ "error": null
+ },
+ {
+ "span_id": "33f806f197bb8f31",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2209.719296,
+ "duration_ms": 0.584704,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"compression\"},\"id\":\"toolu_01HHBqgQqcHMd8evwgJuam2z\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "1b360c8d6e6c787a",
+ "parent_span_id": "33f806f197bb8f31",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2209.995008,
+ "duration_ms": 0.182272,
+ "status": "ok",
+ "input_preview": "{\"store\":\"Postgres\",\"topic\":\"compression\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "09d645b80fbb604e",
+ "parent_span_id": "6fbea322f87ea515",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2210.438144,
+ "duration_ms": 976.382976,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"},{\"content\":\"I'll look up the compression benchmarks for both ClickHouse an",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "13b096b9dae2d701",
+ "parent_span_id": "09d645b80fbb604e",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2210.657024,
+ "duration_ms": 976.033024,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}, {\"role\": \"assistant\", \"content\": \"I'll ",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 929,
+ "output_tokens": 46,
+ "litellm_request_id": "chatcmpl-5c9e8dbb-ecc2-429f-a9c0-a32a6dd21561",
+ "error": null
+ },
+ {
+ "span_id": "9668389035ccda64",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2232.43904,
+ "duration_ms": 0.559104,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"multi-tenancy\",\"store\":\"Postgres\"},\"id\":\"toolu_01JAfdCsAttQJNk9XYxypyYa\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c3d2f25141a98eb8",
+ "parent_span_id": "9668389035ccda64",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2232.690176,
+ "duration_ms": 0.180992,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"multi-tenancy\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "67b2045e76a73cc7",
+ "parent_span_id": "03f611cf9bfc9ea4",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2233.128192,
+ "duration_ms": 1048.211968,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"},{\"content\":\"I'll look up the multi-tenancy benchmarks for both ClickHous",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c55bf581d431c15d",
+ "parent_span_id": "67b2045e76a73cc7",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2233.336064,
+ "duration_ms": 1047.846912,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 950,
+ "output_tokens": 66,
+ "litellm_request_id": "chatcmpl-4c7b01ab-f44c-4b29-9a90-b3501daee307",
+ "error": null
+ },
+ {
+ "span_id": "40823f352c0bc917",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2246.971136,
+ "duration_ms": 0.545024,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"schema changes\",\"store\":\"Postgres\"},\"id\":\"toolu_01UebHCJ4T4tV9bvRQLxLKi5\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "298b6cf2f3be5dce",
+ "parent_span_id": "40823f352c0bc917",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2247.197184,
+ "duration_ms": 0.189952,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"schema changes\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b7c7d3bb5226d077",
+ "parent_span_id": "042d555505c3a65a",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2247.642112,
+ "duration_ms": 899.931904,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"},{\"content\":\"I'll look up the benchmark data for schema changes on both ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b7c18743204be618",
+ "parent_span_id": "b7c7d3bb5226d077",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2247.833088,
+ "duration_ms": 899.610112,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}, {\"role\": \"assistant\", \"content\": \"I'",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 936,
+ "output_tokens": 47,
+ "litellm_request_id": "chatcmpl-c7e73498-e07d-4ee7-804a-8c8fcb518108",
+ "error": null
+ },
+ {
+ "span_id": "3c16f5aa4a7be5a3",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2255.033088,
+ "duration_ms": 0.540928,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"retention (question 3)\"},\"id\":\"toolu_015RETCBXbiEpLSJrMxs9Zmo\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "105b73f1d31ee52a",
+ "parent_span_id": "3c16f5aa4a7be5a3",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2255.257088,
+ "duration_ms": 0.188928,
+ "status": "ok",
+ "input_preview": "{\"store\":\"Postgres\",\"topic\":\"retention (question 3)\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fadbe6d96a1873f4",
+ "parent_span_id": "49397f8d7856d2ba",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2255.70432,
+ "duration_ms": 938.539776,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"},{\"content\":\"I'll look up the retention benchmark (question 3) for both Click",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0fba6a6a6f83d0b8",
+ "parent_span_id": "fadbe6d96a1873f4",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2255.902976,
+ "duration_ms": 938.221056,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}, {\"role\": \"assistant\", \"content\": \"I'll lo",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 953,
+ "output_tokens": 50,
+ "litellm_request_id": "chatcmpl-079b3671-6e0e-4296-abf7-e9e3e1c27fcf",
+ "error": null
+ },
+ {
+ "span_id": "6110ac9f90c2752e",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2263.136256,
+ "duration_ms": 0.522752,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"ingest throughput question 0\"},\"id\":\"toolu_01TBx89BvBRuExmz2svZ1yif\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a25f4e268076e364",
+ "parent_span_id": "6110ac9f90c2752e",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2263.355136,
+ "duration_ms": 0.163072,
+ "status": "ok",
+ "input_preview": "{\"store\":\"Postgres\",\"topic\":\"ingest throughput question 0\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "63ab0a445ddf082b",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2263.825152,
+ "duration_ms": 1080.289792,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f0ca43516ddabae8",
+ "parent_span_id": "63ab0a445ddf082b",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2264.07808,
+ "duration_ms": 1079.866112,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1032,
+ "output_tokens": 102,
+ "litellm_request_id": "chatcmpl-a6a337fb-ed87-4945-8d38-c2618dbf7bb5",
+ "error": null
+ },
+ {
+ "span_id": "3e6908531fde1417",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2428.333056,
+ "duration_ms": 0.584192,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"storage cost\",\"store\":\"Postgres\"},\"id\":\"toolu_0179gVEsfBWz2fJApaowwUsR\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8604b658e1d09660",
+ "parent_span_id": "3e6908531fde1417",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2428.614144,
+ "duration_ms": 0.176896,
+ "status": "ok",
+ "input_preview": "{\"topic\":\"storage cost\",\"store\":\"Postgres\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5e866cac131c336b",
+ "parent_span_id": "a1e8b8b06038a09e",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2429.062144,
+ "duration_ms": 712.163072,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"},{\"content\":\"I'll look up the storage cost benchmark for both ClickHouse ",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "0b2947532de8748d",
+ "parent_span_id": "5e866cac131c336b",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2429.29408,
+ "duration_ms": 711.782144,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}, {\"role\": \"assistant\", \"content\": \"I'l",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 938,
+ "output_tokens": 23,
+ "litellm_request_id": "chatcmpl-c4edc774-323f-45e1-bc3a-8d0eb97d16f0",
+ "error": null
+ },
+ {
+ "span_id": "b8bca8de1d96cc40",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2526.972928,
+ "duration_ms": 0.666368,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"ingest throughput\"},\"id\":\"toolu_011QUPLx4iP9BABp9m4hKz5x\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "659942f79f12abd7",
+ "parent_span_id": "b8bca8de1d96cc40",
+ "name": "lookup_benchmark",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 2527.28192,
+ "duration_ms": 0.211968,
+ "status": "ok",
+ "input_preview": "{\"store\":\"Postgres\",\"topic\":\"ingest throughput\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "f25fc8653bc1fb53",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 2527.796992,
+ "duration_ms": 1241.76,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "48939b5fea74fe64",
+ "parent_span_id": "f25fc8653bc1fb53",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 2528.058112,
+ "duration_ms": 1241.334784,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1024,
+ "output_tokens": 90,
+ "litellm_request_id": "chatcmpl-35af2db3-6d09-48c8-8833-6343af4ba2fd",
+ "error": null
+ },
+ {
+ "span_id": "9c182a974b27b339",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 3344.328192,
+ "duration_ms": 4698.794752,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"check_fact\",\"args\":{\"claim\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"},\"id\":\"toolu_01RYBo12yuVNeAMGPKeeTU8W\",\"type\":\"tool",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "16d2a4ddd9cd75eb",
+ "parent_span_id": "9c182a974b27b339",
+ "name": "check_fact",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 3344.649984,
+ "duration_ms": 4698.307072,
+ "status": "ok",
+ "input_preview": "{\"claim\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "51a224d31e19f7f3",
+ "parent_span_id": "16d2a4ddd9cd75eb",
+ "name": "fact_checker",
+ "type": "agent",
+ "agent": "fact_checker",
+ "start_offset_ms": 3344.890112,
+ "duration_ms": 4697.988096,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "263738613df3c778",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 3345.30816,
+ "duration_ms": 786.53312,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "26162e38f6e4fb66",
+ "parent_span_id": "263738613df3c778",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 3346.041088,
+ "duration_ms": 785.647104,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 627,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-edbca34e-9078-48c7-abd7-8a9f4582870f",
+ "error": null
+ },
+ {
+ "span_id": "bf30867ce7910d16",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "tools",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 3769.766144,
+ "duration_ms": 6859.677952,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"check_fact\",\"args\":{\"claim\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"},\"id\":\"toolu_0156JMekNsuByLWS9LgD467H\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5c98a6ea86360aa7",
+ "parent_span_id": "bf30867ce7910d16",
+ "name": "check_fact",
+ "type": "tool",
+ "agent": "researcher",
+ "start_offset_ms": 3770.066176,
+ "duration_ms": 6859.212032,
+ "status": "ok",
+ "input_preview": "{\"claim\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8feaefa0bc4965be",
+ "parent_span_id": "5c98a6ea86360aa7",
+ "name": "fact_checker",
+ "type": "agent",
+ "agent": "fact_checker",
+ "start_offset_ms": 3770.310144,
+ "duration_ms": 6858.87488,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "b746d9d381022c15",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 3770.621952,
+ "duration_ms": 796.114176,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e4b9cea76a0d8d31",
+ "parent_span_id": "b746d9d381022c15",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 3770.832128,
+ "duration_ms": 795.727872,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}]",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 615,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-855924db-53e0-4e18-8f06-8d2cb0b58007",
+ "error": null
+ },
+ {
+ "span_id": "1c17cffbd43b34d7",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 4132.04096,
+ "duration_ms": 0.795136,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01PmKqDzGSF7Ro3VkLMxRv7v\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "89f28493ea2cdb94",
+ "parent_span_id": "1c17cffbd43b34d7",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 4132.351232,
+ "duration_ms": 0.283904,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "eb7c697a8563ac30",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 4132.987136,
+ "duration_ms": 901.867008,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5feef5ad987b1f20",
+ "parent_span_id": "eb7c697a8563ac30",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 4133.216256,
+ "duration_ms": 901.469952,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 707,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-657fe258-62c2-44cd-b2fc-ac49db6fabe3",
+ "error": null
+ },
+ {
+ "span_id": "449f25412adbd28f",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 4566.943232,
+ "duration_ms": 0.877824,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_017eXQyypsE9G9zXsj9Y8ajF\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "aef1a1f15b5d7f5b",
+ "parent_span_id": "449f25412adbd28f",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 4567.258112,
+ "duration_ms": 0.318976,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "46818b4693a3f76d",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 4568.013056,
+ "duration_ms": 865.807104,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "7e5425f54c786eb7",
+ "parent_span_id": "46818b4693a3f76d",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 4568.477184,
+ "duration_ms": 865.179904,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 695,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-3c2957fe-732d-4f5a-ba69-070abdcc38c0",
+ "error": null
+ },
+ {
+ "span_id": "ab19a3a60dea94a5",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5035.05408,
+ "duration_ms": 0.785152,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01ATAjLVTdnytZoCuos7HQs7\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5f53e6faca69b7de",
+ "parent_span_id": "ab19a3a60dea94a5",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 5035.332096,
+ "duration_ms": 0.305152,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4391ec180f69212e",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5036.003072,
+ "duration_ms": 856.84224,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "9b352033ec99891b",
+ "parent_span_id": "4391ec180f69212e",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 5036.255232,
+ "duration_ms": 856.406016,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 795,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-5911e772-31c2-4b17-b468-6a227c48ee6a",
+ "error": null
+ },
+ {
+ "span_id": "68a537f3863b70da",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5434.029056,
+ "duration_ms": 0.772096,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01GJGayEbSUaraG6jQsLqHHV\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "720f818c5e48ed15",
+ "parent_span_id": "68a537f3863b70da",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 5434.324992,
+ "duration_ms": 0.281088,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "8fa0feeeb5905661",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5434.936064,
+ "duration_ms": 897.363968,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "ed9378cc67e1d2bc",
+ "parent_span_id": "8fa0feeeb5905661",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 5435.160064,
+ "duration_ms": 896.971008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 783,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-e0aa7109-6024-4604-8a3b-48158cf4129f",
+ "error": null
+ },
+ {
+ "span_id": "b8568ad437cc7c6f",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5893.080064,
+ "duration_ms": 0.772864,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_019xqED1EHc6Z6VwWY2zuPVv\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e89ae09beec1b46c",
+ "parent_span_id": "b8568ad437cc7c6f",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 5893.383936,
+ "duration_ms": 0.272128,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3572e33ffdea5492",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 5894.023168,
+ "duration_ms": 934.276864,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "c19ded5674612356",
+ "parent_span_id": "3572e33ffdea5492",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 5894.546176,
+ "duration_ms": 933.578752,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 883,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-8740212c-796e-4bc0-b334-2f2be82819f5",
+ "error": null
+ },
+ {
+ "span_id": "5fb9c34e4ba07882",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 6332.523264,
+ "duration_ms": 0.802816,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01MYBaPSYtDjeSm3vZxhNFZ5\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "e199fcc910071a4e",
+ "parent_span_id": "5fb9c34e4ba07882",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 6332.82304,
+ "duration_ms": 0.286208,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "3b186a2923d5b458",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 6333.495296,
+ "duration_ms": 1013.369856,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "fab16e12209104f4",
+ "parent_span_id": "3b186a2923d5b458",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 6333.748224,
+ "duration_ms": 1012.94592,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 883,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-ee1c82af-b979-47c0-a2fb-94e4f9b3b6ea",
+ "error": null
+ },
+ {
+ "span_id": "aa4b97926e556afe",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 6828.529152,
+ "duration_ms": 0.78208,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01T67jNUGCh7eg6wiaHKmgHH\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "43f9afdb2a488daf",
+ "parent_span_id": "aa4b97926e556afe",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 6828.822272,
+ "duration_ms": 0.28672,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a0a80112bb486b0f",
+ "parent_span_id": "51a224d31e19f7f3",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 6829.462016,
+ "duration_ms": 1213.248,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "4d1d711bb127ba1d",
+ "parent_span_id": "a0a80112bb486b0f",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 6829.707264,
+ "duration_ms": 1212.832,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 974,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-1ef874c3-55cd-4c60-8fc1-c1bccd2a4e88",
+ "error": null
+ },
+ {
+ "span_id": "aeb2460c46a7b885",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 7347.08608,
+ "duration_ms": 0.802048,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_018V9AomXZBTgkUVnsJJUmSf\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d57d0138262e6459",
+ "parent_span_id": "aeb2460c46a7b885",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 7347.385088,
+ "duration_ms": 0.29184,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "61eef30374dafead",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 7348.064256,
+ "duration_ms": 1277.523968,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "994b738b34f44318",
+ "parent_span_id": "61eef30374dafead",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 7349.096192,
+ "duration_ms": 1276.313856,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 969,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-1585a7bc-85cf-4948-ace7-8685141196e0",
+ "error": null
+ },
+ {
+ "span_id": "99305ec77a4b0db8",
+ "parent_span_id": "ffb1aafc123f7cbf",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 8043.282176,
+ "duration_ms": 868.797952,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "bac938a998d22f5f",
+ "parent_span_id": "99305ec77a4b0db8",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 8043.563264,
+ "duration_ms": 868.342784,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1206,
+ "output_tokens": 49,
+ "litellm_request_id": "chatcmpl-733eeb0e-1466-486e-b7a8-c659a3e13fd4",
+ "error": null
+ },
+ {
+ "span_id": "76dd59068fe39775",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 8625.82016,
+ "duration_ms": 0.784896,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01DHE5bKweeAAYdUqyUEksxG\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "14dbb727670638dc",
+ "parent_span_id": "76dd59068fe39775",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 8626.10816,
+ "duration_ms": 0.290816,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "820968a1a530b332",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 8626.76224,
+ "duration_ms": 920.998656,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "d980027f6fef58d6",
+ "parent_span_id": "820968a1a530b332",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 8627.012096,
+ "duration_ms": 920.585984,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1060,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-1bc15649-313e-487d-bf02-ea93f63cc126",
+ "error": null
+ },
+ {
+ "span_id": "6b5359ea58dbc7df",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "tools",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 9548.065024,
+ "duration_ms": 0.884992,
+ "status": "ok",
+ "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01EMWtXbGHXpq6EL7jEbPsQX\",\"type\":\"tool_call\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "14c958250e70669e",
+ "parent_span_id": "6b5359ea58dbc7df",
+ "name": "verify_claim",
+ "type": "tool",
+ "agent": "fact_checker",
+ "start_offset_ms": 9548.434944,
+ "duration_ms": 0.303104,
+ "status": "error",
+ "input_preview": "{}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "46e1d0320d1c63e2",
+ "parent_span_id": "8feaefa0bc4965be",
+ "name": "model",
+ "type": "chain",
+ "agent": "fact_checker",
+ "start_offset_ms": 9549.12,
+ "duration_ms": 1079.88608,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "a9c4ba6ba55a93ef",
+ "parent_span_id": "46e1d0320d1c63e2",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "fact_checker",
+ "start_offset_ms": 9549.393152,
+ "duration_ms": 1079.435776,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1149,
+ "output_tokens": 60,
+ "litellm_request_id": "chatcmpl-ada1126b-e136-4cb4-92bd-b6d1419fdf09",
+ "error": null
+ },
+ {
+ "span_id": "725ee1f23444ee13",
+ "parent_span_id": "4bc4c26013a33115",
+ "name": "model",
+ "type": "chain",
+ "agent": "researcher",
+ "start_offset_ms": 10629.611008,
+ "duration_ms": 1055.321856,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "6b5c44b97608a665",
+ "parent_span_id": "725ee1f23444ee13",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "researcher",
+ "start_offset_ms": 10629.89312,
+ "duration_ms": 1054.866176,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question",
+ "model": "claude-haiku-4-5",
+ "input_tokens": 1186,
+ "output_tokens": 48,
+ "litellm_request_id": "chatcmpl-dc15e33e-0526-4017-8359-1ed0a0b5bfb7",
+ "error": null
+ },
+ {
+ "span_id": "ec33160969513265",
+ "parent_span_id": "b7b1053f6c95fd70",
+ "name": "critic",
+ "type": "agent",
+ "agent": "critic",
+ "start_offset_ms": 11685.611008,
+ "duration_ms": 5460.011008,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"12 findings, 1 failed. ClickHouse wins on ingest and cost.\"}]",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "5c43936b0d83e971",
+ "parent_span_id": "ec33160969513265",
+ "name": "model",
+ "type": "chain",
+ "agent": "critic",
+ "start_offset_ms": 11686.049024,
+ "duration_ms": 5459.417088,
+ "status": "ok",
+ "input_preview": "{\"messages\":[{\"content\":\"12 findings, 1 failed. ClickHouse wins on ingest and cost.\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9b909661-205a-4ad5-a595-358342a07c35\"}]}",
+ "model": null,
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "litellm_request_id": null,
+ "error": null
+ },
+ {
+ "span_id": "82692e8e9ec9105c",
+ "parent_span_id": "5c43936b0d83e971",
+ "name": "ChatOpenAI",
+ "type": "llm",
+ "agent": "critic",
+ "start_offset_ms": 11686.348032,
+ "duration_ms": 5458.984192,
+ "status": "ok",
+ "input_preview": "[{\"role\": \"system\", \"content\": \"Critique the summary in 2 bullets.\"}, {\"role\": \"user\", \"content\": \"12 findings, 1 failed. ClickHouse wins on ingest and cost.\"}]",
+ "model": "claude-sonnet-4-5",
+ "input_tokens": 38,
+ "output_tokens": 125,
+ "litellm_request_id": "chatcmpl-0f035e16-9296-469a-b925-d00aed5a1fed",
+ "error": null
+ }
+ ]
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json
new file mode 100644
index 00000000000..ce0c22434c1
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json
@@ -0,0 +1,59 @@
+{
+ "data": [
+ {
+ "trace_id": "e309a123963901e74c29cd2d3c86ff9e",
+ "name": "research_lead",
+ "service": "research-agent",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]",
+ "start_time": "2026-09-30T06:43:54.291000+00:00",
+ "duration_ms": 40198.0,
+ "status": "ok",
+ "span_count": 216,
+ "agent_count": 6,
+ "llm_calls": 21,
+ "tool_calls": 25,
+ "error_count": 0,
+ "input_tokens": 69506,
+ "output_tokens": 2960,
+ "models": ["claude-sonnet-4-5"],
+ "agent_invocations": 6
+ },
+ {
+ "trace_id": "f78f6df35480060fafadac887e234241",
+ "name": "support_triage_agent",
+ "service": "research-agent",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-404 says billing is wrong. What plan are they on?\"}]",
+ "start_time": "2026-09-30T06:43:52.928000+00:00",
+ "duration_ms": 1315.0,
+ "status": "ok",
+ "span_count": 5,
+ "agent_count": 1,
+ "llm_calls": 1,
+ "tool_calls": 1,
+ "error_count": 2,
+ "input_tokens": 659,
+ "output_tokens": 60,
+ "models": ["claude-sonnet-4-5"],
+ "agent_invocations": 1
+ },
+ {
+ "trace_id": "71498cec128bbea430f01de04b972f36",
+ "name": "support_triage_agent",
+ "service": "research-agent",
+ "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-42 gets 429s after upgrading. What plan are they on and what should they check?\"}]",
+ "start_time": "2026-09-30T06:43:47.373000+00:00",
+ "duration_ms": 5551.0,
+ "status": "ok",
+ "span_count": 9,
+ "agent_count": 1,
+ "llm_calls": 2,
+ "tool_calls": 2,
+ "error_count": 0,
+ "input_tokens": 1552,
+ "output_tokens": 202,
+ "models": ["claude-sonnet-4-5"],
+ "agent_invocations": 1
+ }
+ ],
+ "next_cursor": null
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTree.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTree.ts
new file mode 100644
index 00000000000..2f0e66833a4
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTree.ts
@@ -0,0 +1,48 @@
+/**
+ * Shared contract for the agent run view (span tree + detail pane).
+ * Rows are produced by `buildTreeRows` in traceUtils.ts and rendered by SpanTree / DetailPane.
+ */
+import type { Span, SpanType } from "./traceTypes";
+
+export type ErrorSource = "model" | "tool" | "litellm";
+
+export interface SpanRowData {
+ kind: "span";
+ id: string;
+ span: Span;
+ depth: number;
+ hasChildren: boolean;
+ collapsed: boolean;
+}
+
+export interface GroupRowData {
+ kind: "group";
+ id: string;
+ depth: number;
+ name: string;
+ type: SpanType;
+ agent: string;
+ members: Span[];
+ failedCount: number;
+ p50Duration: number;
+ /** Every member failed (e.g. a tool that failed ×12). */
+ isFailureGroup: boolean;
+ expanded: boolean;
+}
+
+export interface LoadMoreRowData {
+ kind: "load-more";
+ id: string;
+ depth: number;
+ groupId: string;
+ remaining: number;
+}
+
+export type TreeRow = SpanRowData | GroupRowData | LoadMoreRowData;
+
+export interface SpanTreeState {
+ hideFramework: boolean;
+ collapsedSpanIds: ReadonlySet;
+ expandedGroupIds: ReadonlySet;
+ groupRevealCounts: Readonly>;
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts
new file mode 100644
index 00000000000..f2611b17ad5
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts
@@ -0,0 +1,89 @@
+/**
+ * Agent tracing types. Mirrors `litellm/tracing/types.py` exactly.
+ *
+ * A trace is one agent run made of spans (agent / llm / tool / chain / framework).
+ */
+
+export type SpanType = "agent" | "llm" | "tool" | "chain" | "framework";
+export type SpanStatus = "ok" | "error" | "unset";
+
+export interface Span {
+ span_id: string;
+ parent_span_id: string | null;
+ name: string;
+ type: SpanType;
+ /** The agent this span runs inside, e.g. "researcher". */
+ agent: string;
+ /** Relative to trace start. */
+ start_offset_ms: number;
+ duration_ms: number;
+ status: SpanStatus;
+ /** Exception message when status is "error". */
+ error?: string | null;
+ input_preview: string;
+ model: string | null;
+ input_tokens: number;
+ output_tokens: number;
+ litellm_request_id: string | null;
+}
+
+/** One distinct agent in a trace. 200 invocations of `researcher` = one node. */
+export interface AgentNode {
+ name: string;
+ parent_agent: string | null;
+ invocations: number;
+ llm_calls: number;
+ tool_calls: number;
+ duration_ms: number;
+}
+
+export interface TraceSummary {
+ trace_id: string;
+ name: string;
+ service: string;
+ input_preview: string;
+ /** ISO 8601 */
+ start_time: string;
+ duration_ms: number;
+ status: SpanStatus;
+ span_count: number;
+ agent_count: number;
+ llm_calls: number;
+ tool_calls: number;
+ /** Spans with an error status; > 0 means the run shows as failed. */
+ error_count: number;
+ input_tokens: number;
+ output_tokens: number;
+ models: string[];
+}
+
+export interface Trace {
+ summary: TraceSummary;
+ agents: AgentNode[];
+ spans: Span[];
+}
+
+export interface TracePage {
+ data: TraceSummary[];
+ next_cursor: string | null;
+}
+
+/** `input` / `output` are JSON strings (messages for llm spans, raw args / result for tools). */
+export interface SpanDetail {
+ span_id: string;
+ input: string;
+ output: string;
+ attributes: Record;
+}
+
+export interface TraceToolCall {
+ name: string;
+ args: unknown;
+}
+
+export interface TraceMessage {
+ role: string;
+ content: string;
+ name?: string;
+ tool_calls?: TraceToolCall[];
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts
new file mode 100644
index 00000000000..bc51be42f1e
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts
@@ -0,0 +1,269 @@
+import { describe, expect, it } from "vitest";
+
+import deepAgentTrace from "./__fixtures__/deep_agent_trace.json";
+import researchTrace from "./__fixtures__/research_trace.json";
+import swarmTrace from "./__fixtures__/swarm_trace.json";
+import { type SpanTreeState, type TreeRow } from "./traceTree";
+import type { Span, Trace } from "./traceTypes";
+import {
+ buildTreeRows,
+ buildVisibleTree,
+ errorSource,
+ firstErrorSpan,
+ fmtMs,
+ GROUP_PAGE_SIZE,
+ groupRowId,
+ isFrameworkSpan,
+ median,
+ parseMessages,
+ previewText,
+ revealSpanInState,
+ ROOT_KEY,
+} from "./traceUtils";
+
+const swarm = swarmTrace as Trace;
+const research = researchTrace as Trace;
+const deepAgent = deepAgentTrace as Trace;
+
+const STATE: SpanTreeState = {
+ hideFramework: true,
+ collapsedSpanIds: new Set(),
+ expandedGroupIds: new Set(),
+ groupRevealCounts: {},
+};
+
+type SpanOverrides = Partial & Pick;
+
+const span = (overrides: SpanOverrides): Span => ({
+ parent_span_id: null,
+ name: overrides.span_id,
+ type: "chain",
+ agent: "root",
+ start_offset_ms: 0,
+ duration_ms: 1,
+ status: "ok",
+ error: null,
+ input_preview: "",
+ model: null,
+ input_tokens: 0,
+ output_tokens: 0,
+ litellm_request_id: null,
+ ...overrides,
+});
+
+const groups = (rows: TreeRow[]) => rows.filter((r): r is Extract => r.kind === "group");
+const spanRows = (rows: TreeRow[]) => rows.filter((r): r is Extract => r.kind === "span");
+
+describe("formatting", () => {
+ it("formats durations", () => {
+ expect(fmtMs(4.25)).toBe("4.3ms");
+ expect(fmtMs(950)).toBe("950ms");
+ expect(fmtMs(51386)).toBe("51.39s");
+ });
+
+ it("pulls the user message out of a truncated JSON preview", () => {
+ expect(previewText('[{"role": "user", "content": "Customer acme-404 says billing is wrong."}]')).toBe(
+ "Customer acme-404 says billing is wrong.",
+ );
+ expect(previewText('[{"role": "system", "content": "sys"}, {"role": "user", "content": "Compare ingest thr')).toBe(
+ "Compare ingest thr",
+ );
+ expect(previewText("plain text")).toBe("plain text");
+ });
+
+ it("pulls the user message out of an OpenInference LangChain input", () => {
+ const input =
+ '{"messages": [{"type": "human", "data": {"content": "Customer acme-7 keeps hitting 429s", "type": "human"';
+ expect(previewText(input)).toBe("Customer acme-7 keeps hitting 429s");
+ });
+
+ it("takes the upper median", () => {
+ expect(median([3, 1, 2])).toBe(2);
+ expect(median([4, 1, 3, 2])).toBe(3);
+ expect(median([])).toBe(0);
+ });
+});
+
+describe("buildVisibleTree / isFrameworkSpan", () => {
+ it("hides framework spans and re-parents their children", () => {
+ const middleware: SpanOverrides = {
+ span_id: "mw",
+ parent_span_id: "root",
+ type: "framework",
+ name: "X.wrap_model_call",
+ };
+ const modelNode: SpanOverrides = { span_id: "model", parent_span_id: "mw", type: "chain", name: "model" };
+ const llmCall: SpanOverrides = { span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 };
+ const planner: SpanOverrides = {
+ span_id: "step",
+ parent_span_id: "root",
+ type: "chain",
+ name: "planner",
+ start_offset_ms: 1,
+ };
+ const spans = [
+ span({ span_id: "root", type: "agent" }),
+ span(middleware),
+ span(modelNode),
+ span(llmCall),
+ span(planner),
+ ];
+ const compact = buildVisibleTree(spans, false);
+ expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]);
+ expect(compact.children.get("root")?.map((s) => s.span_id)).toEqual(["step", "llm"]);
+ expect(buildVisibleTree(spans, true).visibleCount).toBe(5);
+ });
+
+ it("never hides the root span", () => {
+ expect(isFrameworkSpan(span({ span_id: "r", type: "framework" }))).toBe(false);
+ });
+});
+
+describe("buildTreeRows", () => {
+ it("starts the tree at the root span", () => {
+ const rows = buildTreeRows(research.spans, STATE);
+ expect(rows[0]).toMatchObject({
+ kind: "span",
+ depth: 0,
+ id: research.spans.find((s) => s.parent_span_id === null)?.span_id,
+ });
+ });
+
+ it("hides every framework / graph-node span of the real Deep Agents trace, keeping all LLM calls", () => {
+ const rows = buildTreeRows(research.spans, STATE);
+ const shown = new Set(spanRows(rows).map((r) => r.span.span_id));
+ expect(research.spans.filter(isFrameworkSpan).every((s) => !shown.has(s.span_id))).toBe(true);
+ const all = buildTreeRows(research.spans, { ...STATE, hideFramework: false });
+ expect(spanRows(all).length).toBeGreaterThan(spanRows(rows).length);
+ });
+
+ it("folds the swarm's 12 researcher invocations into one group row", () => {
+ const rows = buildTreeRows(swarm.spans, STATE);
+ const researcher = groups(rows).find((g) => g.name === "researcher" && g.type === "agent");
+ expect(researcher?.members).toHaveLength(12);
+ expect(researcher?.expanded).toBe(false);
+ expect(researcher?.p50Duration).toBeGreaterThan(0);
+ // folded members are not rendered until the group is expanded
+ expect(spanRows(rows).some((r) => r.span.name === "researcher")).toBe(false);
+ });
+
+ it("folds as few as 3 failed siblings of the same tool into a failure group", () => {
+ const parent = span({ span_id: "p", type: "agent" });
+ const failing = [0, 1, 2].map((i) => {
+ const failedGrep: SpanOverrides = {
+ span_id: `t${i}`,
+ parent_span_id: "p",
+ type: "tool",
+ name: "grep_code",
+ status: "error",
+ start_offset_ms: i,
+ };
+ return span(failedGrep);
+ });
+ const rows = buildTreeRows([parent, ...failing], STATE);
+ const group = groups(rows)[0];
+ expect(group).toMatchObject({ name: "grep_code", failedCount: 3, isFailureGroup: true });
+ });
+
+ it("never folds same-named calls from different agents into one group", () => {
+ const parent = span({ span_id: "p", type: "agent" });
+ const calls = [0, 1, 2, 3, 4, 5].map((i) => {
+ const call: SpanOverrides = {
+ span_id: `c${i}`,
+ parent_span_id: "p",
+ type: "llm",
+ name: "ChatOpenAI",
+ agent: i < 3 ? "planner" : "critic",
+ };
+ return span(call);
+ });
+ const rows = buildTreeRows([parent, ...calls], STATE);
+ expect(groups(rows)).toHaveLength(0);
+ expect(spanRows(rows).filter((r) => r.span.name === "ChatOpenAI")).toHaveLength(6);
+ });
+
+ it("leaves 5 healthy same-named siblings unfolded", () => {
+ const parent = span({ span_id: "p", type: "agent" });
+ const kids = [0, 1, 2, 3, 4].map((i) => {
+ const search: SpanOverrides = { span_id: `k${i}`, parent_span_id: "p", type: "tool", name: "search" };
+ return span(search);
+ });
+ expect(groups(buildTreeRows([parent, ...kids], STATE))).toHaveLength(0);
+ });
+
+ it("pages expanded groups 20 at a time with a load-more row", () => {
+ const parent = span({ span_id: "p", type: "agent" });
+ const kids = Array.from({ length: 45 }, (_, i) => {
+ const worker: SpanOverrides = {
+ span_id: `k${i}`,
+ parent_span_id: "p",
+ type: "agent",
+ name: "worker",
+ start_offset_ms: i,
+ };
+ return span(worker);
+ });
+ const all = [parent, ...kids];
+ const id = groupRowId("p", kids[0]);
+ const page1 = buildTreeRows(all, { ...STATE, expandedGroupIds: new Set([id]) });
+ expect(spanRows(page1).filter((r) => r.span.name === "worker")).toHaveLength(GROUP_PAGE_SIZE);
+ expect(page1.find((r) => r.kind === "load-more")).toMatchObject({ groupId: id, remaining: 25 });
+ const everything = buildTreeRows(all, {
+ ...STATE,
+ expandedGroupIds: new Set([id]),
+ groupRevealCounts: { [id]: 60 },
+ });
+ expect(spanRows(everything).filter((r) => r.span.name === "worker")).toHaveLength(45);
+ expect(everything.some((r) => r.kind === "load-more")).toBe(false);
+ });
+
+ it("hides the children of collapsed spans", () => {
+ const root = swarm.spans.find((s) => s.parent_span_id === null) as Span;
+ const rows = buildTreeRows(swarm.spans, { ...STATE, collapsedSpanIds: new Set([root.span_id]) });
+ expect(rows.map((r) => r.kind)).toEqual(["span"]);
+ });
+});
+
+describe("revealSpanInState", () => {
+ it("opens the path to a span nested in a folded group so the view can land on it", () => {
+ const failed = firstErrorSpan(swarm.spans) as Span;
+ const state = revealSpanInState(swarm.spans, STATE, failed.span_id);
+ const rows = buildTreeRows(swarm.spans, state);
+ expect(rows.some((r) => r.id === failed.span_id)).toBe(true);
+ });
+});
+
+describe("errorSource", () => {
+ it("blames the tool, LiteLLM, or the model", () => {
+ expect(errorSource(span({ span_id: "a", status: "ok" }))).toBeNull();
+ const failure = (span_id: string, type: Span["type"], error: string): Span => {
+ const failed: SpanOverrides = { span_id, type, error, status: "error" };
+ return span(failed);
+ };
+ expect(errorSource(failure("t", "tool", "boom"))).toBe("tool");
+ expect(errorSource(failure("l", "llm", "429 Rate limit exceeded"))).toBe("litellm");
+ expect(errorSource(failure("g", "llm", "Blocked by guardrail"))).toBe("litellm");
+ expect(errorSource(failure("m", "llm", "context length exceeded"))).toBe("model");
+ });
+
+ it("classifies the swarm's failing lookup_benchmark calls as tool errors", () => {
+ const failedTool = swarm.spans.find((s) => s.status === "error" && s.type === "tool") as Span;
+ expect(failedTool.name).toBe("lookup_benchmark");
+ expect(errorSource(failedTool)).toBe("tool");
+ });
+});
+
+describe("payload helpers", () => {
+ it("finds the earliest failing non-root span", () => {
+ const failed = firstErrorSpan(swarm.spans);
+ expect(failed?.status).toBe("error");
+ expect(failed?.parent_span_id).not.toBeNull();
+ expect(firstErrorSpan(deepAgent.spans)).toBeNull();
+ });
+
+ it("parses llm message payloads and rejects non-message JSON", () => {
+ expect(parseMessages('[{"role":"user","content":"hi"}]')).toEqual([{ role: "user", content: "hi" }]);
+ expect(parseMessages('{"file_path":"/tmp/x"}')).toBeNull();
+ expect(parseMessages("not json")).toBeNull();
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts
new file mode 100644
index 00000000000..d0b6e263f3b
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts
@@ -0,0 +1,314 @@
+/**
+ * Pure helpers for the agent trace views. No React in here: everything the span tree
+ * computes lives here so it can be unit-tested directly.
+ */
+import { type ErrorSource, type SpanTreeState, type TreeRow } from "./traceTree";
+import type { Span, TraceMessage, TraceSummary } from "./traceTypes";
+
+/* ------------------------------------------------------------------ */
+/* Formatting */
+/* ------------------------------------------------------------------ */
+
+export const fmtMs = (ms: number): string => {
+ if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
+ if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+ return `${Math.max(ms, 0).toFixed(ms < 10 ? 1 : 0)}ms`;
+};
+
+export const fmtTok = (n: number): string => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n));
+
+export const shortId = (id: string, length = 16): string => (id.length > length ? `${id.slice(0, length)}…` : id);
+
+/* ------------------------------------------------------------------ */
+/* Visible tree (framework spans hidden + children re-parented) */
+/* ------------------------------------------------------------------ */
+
+/** Root-level key in the children map. */
+export const ROOT_KEY = "__root__";
+
+export type ChildrenMap = Map;
+
+export interface VisibleTree {
+ children: ChildrenMap;
+ visibleCount: number;
+}
+
+/**
+ * Framework plumbing: middleware wrappers plus LangGraph's generic "model" / "tools"
+ * graph nodes. The root span is never hidden.
+ */
+const GRAPH_NODE_NAMES = new Set(["model", "tools"]);
+
+export const isFrameworkSpan = (span: Span): boolean => {
+ const isGraphNode = span.type === "chain" && GRAPH_NODE_NAMES.has(span.name);
+ const isPlumbing = span.type === "framework" || isGraphNode;
+ return span.parent_span_id !== null && isPlumbing;
+};
+
+const byStart = (a: Span, b: Span): number => a.start_offset_ms - b.start_offset_ms;
+
+export const indexSpans = (spans: readonly Span[]): Map => new Map(spans.map((s) => [s.span_id, s]));
+
+const pushChild = (children: ChildrenMap, key: string, span: Span): void => {
+ const list = children.get(key);
+ if (list) list.push(span);
+ else children.set(key, [span]);
+};
+
+/**
+ * Children map for the waterfall. With `showFramework` off, framework spans are
+ * dropped and their children attach to the nearest visible ancestor. Spans whose
+ * parent is missing from the trace attach to the root level.
+ */
+export function buildVisibleTree(spans: readonly Span[], showFramework: boolean): VisibleTree {
+ const byId = indexSpans(spans);
+ const hidden = (span: Span) => !showFramework && isFrameworkSpan(span);
+ const visibleParentKey = (span: Span): string => {
+ let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined;
+ while (parent && hidden(parent)) {
+ parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined;
+ }
+ return parent ? parent.span_id : ROOT_KEY;
+ };
+ const children: ChildrenMap = new Map();
+ let visibleCount = 0;
+ for (const span of spans) {
+ if (hidden(span)) continue;
+ visibleCount++;
+ pushChild(children, visibleParentKey(span), span);
+ }
+ children.forEach((list) => list.sort(byStart));
+ return { children, visibleCount };
+}
+
+/* ------------------------------------------------------------------ */
+/* Run view rows (Input, span tree with grouping, Output) */
+/* ------------------------------------------------------------------ */
+
+/** Same-named siblings fold into one row at this count ... */
+export const GROUP_THRESHOLD_OK = 6;
+/** ... or as soon as this many of them failed. */
+export const GROUP_THRESHOLD_ERROR = 3;
+/** Group rows reveal this many members at a time. */
+export const GROUP_PAGE_SIZE = 20;
+
+export const median = (values: readonly number[]): number => {
+ if (values.length === 0) return 0;
+ const sorted = [...values].sort((a, b) => a - b);
+ return sorted[Math.floor(sorted.length / 2)];
+};
+
+const LITELLM_ERROR = /rate.?limit|429|guardrail|budget|litellm/i;
+
+/** Who failed: the tool, LiteLLM (rate limit / guardrail / budget), or the model. Null when the span is fine. */
+export function errorSource(span: Span): ErrorSource | null {
+ if (span.status !== "error") return null;
+ if (span.type === "tool") return "tool";
+ if (LITELLM_ERROR.test(span.error ?? "")) return "litellm";
+ return "model";
+}
+
+type GroupOrSpan = Span | { group: Span[] };
+
+const groupKey = (span: Pick): string => `${span.agent}|${span.name}|${span.type}`;
+
+/** Siblings sharing agent + name + type fold into one group once there are enough of them (or enough failures). */
+function groupChildren(children: readonly Span[]): GroupOrSpan[] {
+ const byKey = new Map();
+ for (const child of children) {
+ const key = groupKey(child);
+ const list = byKey.get(key);
+ if (list) list.push(child);
+ else byKey.set(key, [child]);
+ }
+ const emitted = new Set();
+ const out: GroupOrSpan[] = [];
+ for (const child of children) {
+ const key = groupKey(child);
+ const group = byKey.get(key) ?? [child];
+ const failed = group.filter((s) => s.status === "error").length;
+ if (group.length >= GROUP_THRESHOLD_OK || failed >= GROUP_THRESHOLD_ERROR) {
+ if (!emitted.has(key)) {
+ emitted.add(key);
+ out.push({ group });
+ }
+ } else {
+ out.push(child);
+ }
+ }
+ return out;
+}
+
+export const groupRowId = (parentKey: string, span: Pick): string =>
+ `grp::${parentKey}::${groupKey(span)}`;
+
+interface RowContext {
+ children: ChildrenMap;
+ state: SpanTreeState;
+ rows: TreeRow[];
+}
+
+function pushSpan(ctx: RowContext, span: Span, depth: number): void {
+ const hasChildren = (ctx.children.get(span.span_id)?.length ?? 0) > 0;
+ const collapsed = ctx.state.collapsedSpanIds.has(span.span_id);
+ const row: TreeRow = { kind: "span", id: span.span_id, span, depth, hasChildren, collapsed };
+ ctx.rows.push(row);
+ if (hasChildren && !collapsed) pushLevel(ctx, span.span_id, depth + 1);
+}
+
+function pushGroup(ctx: RowContext, parentKey: string, members: Span[], depth: number): void {
+ const first = members[0];
+ const id = groupRowId(parentKey, first);
+ const failedCount = members.filter((s) => s.status === "error").length;
+ const expanded = ctx.state.expandedGroupIds.has(id);
+ const groupRow: TreeRow = {
+ kind: "group",
+ id,
+ depth,
+ name: first.name,
+ type: first.type,
+ agent: first.agent,
+ members,
+ failedCount,
+ p50Duration: median(members.map((s) => s.duration_ms)),
+ isFailureGroup: failedCount === members.length,
+ expanded,
+ };
+ ctx.rows.push(groupRow);
+ if (!expanded) return;
+ const reveal = Math.min(ctx.state.groupRevealCounts[id] ?? GROUP_PAGE_SIZE, members.length);
+ members.slice(0, reveal).forEach((member) => pushSpan(ctx, member, depth + 1));
+ if (reveal < members.length) {
+ const moreRow: TreeRow = {
+ kind: "load-more",
+ id: `${id}::more`,
+ depth: depth + 1,
+ groupId: id,
+ remaining: members.length - reveal,
+ };
+ ctx.rows.push(moreRow);
+ }
+}
+
+function pushLevel(ctx: RowContext, parentKey: string, depth: number): void {
+ for (const item of groupChildren(ctx.children.get(parentKey) ?? [])) {
+ if ("group" in item) pushGroup(ctx, parentKey, item.group, depth);
+ else pushSpan(ctx, item, depth);
+ }
+}
+
+/**
+ * Rows for the run view: the span tree, framework spans optionally hidden with their children lifted
+ * and same-named siblings folded into paged groups.
+ */
+export function buildTreeRows(spans: readonly Span[], state: SpanTreeState): TreeRow[] {
+ const ctx: RowContext = { children: buildVisibleTree(spans, !state.hideFramework).children, state, rows: [] };
+ pushLevel(ctx, ROOT_KEY, 0);
+ return ctx.rows;
+}
+
+/** Tree state with every visible ancestor of `spanId` expanded and any group holding it paged far enough. */
+export function revealSpanInState(spans: readonly Span[], state: SpanTreeState, spanId: string): SpanTreeState {
+ const { children } = buildVisibleTree(spans, !state.hideFramework);
+ const parentOf = new Map();
+ children.forEach((list, key) => list.forEach((s) => parentOf.set(s.span_id, key)));
+ if (!parentOf.has(spanId)) return state;
+ const collapsed = new Set(state.collapsedSpanIds);
+ const expanded = new Set(state.expandedGroupIds);
+ const reveal = { ...state.groupRevealCounts };
+ let current = spanId;
+ while (parentOf.has(current)) {
+ const parentKey = parentOf.get(current) as string;
+ collapsed.delete(parentKey);
+ for (const item of groupChildren(children.get(parentKey) ?? [])) {
+ if (!("group" in item)) continue;
+ const index = item.group.findIndex((s) => s.span_id === current);
+ if (index < 0) continue;
+ const id = groupRowId(parentKey, item.group[0]);
+ expanded.add(id);
+ reveal[id] = Math.max(reveal[id] ?? GROUP_PAGE_SIZE, Math.ceil((index + 1) / GROUP_PAGE_SIZE) * GROUP_PAGE_SIZE);
+ }
+ current = parentKey;
+ }
+ return { ...state, collapsedSpanIds: collapsed, expandedGroupIds: expanded, groupRevealCounts: reveal };
+}
+
+/** `spanId` if it shows in the tree, else its nearest ancestor that does (framework spans can be hidden). */
+export function nearestVisibleSpanId(spans: readonly Span[], spanId: string, hideFramework: boolean): string {
+ if (!hideFramework) return spanId;
+ const byId = new Map(spans.map((s) => [s.span_id, s]));
+ let current = byId.get(spanId);
+ while (current && isFrameworkSpan(current) && current.parent_span_id !== null) {
+ current = byId.get(current.parent_span_id);
+ }
+ return current?.span_id ?? spanId;
+}
+
+/* ------------------------------------------------------------------ */
+/* Trace-level rollups */
+/* ------------------------------------------------------------------ */
+
+/** Earliest failing non-root span (the root just echoes its children), else the root. */
+export function firstErrorSpan(spans: readonly Span[]): Span | null {
+ const failed = spans.filter((s) => s.status === "error").sort(byStart);
+ return failed.find((s) => s.parent_span_id !== null) ?? failed[0] ?? null;
+}
+
+/* ------------------------------------------------------------------ */
+/* Span detail payloads */
+/* ------------------------------------------------------------------ */
+
+export const parseJson = (value: string): unknown => {
+ if (!value) return null;
+ try {
+ return JSON.parse(value);
+ } catch {
+ return null;
+ }
+};
+
+const isMessage = (value: unknown): value is TraceMessage => {
+ const isObject = typeof value === "object" && value !== null;
+ return isObject && "role" in value && typeof (value as TraceMessage).role === "string";
+};
+
+/** An llm span's input (array of messages) or output (one message); null when it isn't one. */
+export function parseMessages(value: string): TraceMessage[] | null {
+ const parsed = parseJson(value);
+ if (Array.isArray(parsed)) return parsed.every(isMessage) ? parsed : null;
+ return isMessage(parsed) ? [parsed] : null;
+}
+
+/** Pretty JSON when the payload is JSON, else the raw string. */
+export const prettyPayload = (value: string): string => {
+ const parsed = parseJson(value);
+ if (parsed === null || typeof parsed === "string") return typeof parsed === "string" ? parsed : value;
+ return JSON.stringify(parsed, null, 2);
+};
+
+/* ------------------------------------------------------------------ */
+/* List view helpers */
+/* ------------------------------------------------------------------ */
+
+const PREVIEW_USER_CONTENT =
+ /"(?:role|type)":\s*"(?:user|human)",\s*"(?:content|data)":\s*(?:\{"content":\s*)?"((?:[^"\\]|\\.)*)/;
+
+/**
+ * Human text for an input preview. Previews are often a (possibly truncated) JSON
+ * message array; show the last user/tool message's content when we can find it.
+ */
+export function previewText(preview: string): string {
+ if (!preview) return "";
+ const messages = parseMessages(preview);
+ if (messages) {
+ const last = [...messages].reverse().find((m) => m.role === "user" || m.role === "tool") ?? messages.at(-1);
+ return last?.content || preview;
+ }
+ if (!/^\s*[[{]/.test(preview)) return preview;
+ const match = PREVIEW_USER_CONTENT.exec(preview);
+ return match ? match[1].replace(/\\n/g, " ").replace(/\\"/g, '"') : preview;
+}
+
+/** Trace display name; root spans without a name fall back to the service. */
+export const traceDisplayName = (summary: Pick): string =>
+ summary.name || summary.service || "(unnamed trace)";
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.test.ts
new file mode 100644
index 00000000000..cbced08785f
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+
+import { traceWindowStartMs } from "./useAgentTraces";
+import { spanLogWindow } from "./useSpanRequestLog";
+
+const HOUR = 3600 * 1000;
+
+describe("traceWindowStartMs", () => {
+ it("rolls a preset range forward with now so live tail keeps a fixed-length window", () => {
+ const start = "2026-09-29T10:00";
+ const end = "2026-09-30T10:00";
+ const mountedAt = Date.parse("2026-09-30T10:00:00");
+ const tenHoursLater = mountedAt + 10 * HOUR;
+ expect(tenHoursLater - traceWindowStartMs(start, end, false, tenHoursLater)).toBe(24 * HOUR);
+ expect(traceWindowStartMs(start, end, false, tenHoursLater)).toBeGreaterThan(
+ traceWindowStartMs(start, end, false, mountedAt),
+ );
+ });
+
+ it("keeps a custom range pinned to what the user picked", () => {
+ const start = "2026-09-01T00:00";
+ expect(traceWindowStartMs(start, "2026-09-02T00:00", true, Date.parse("2026-09-30T00:00:00"))).toBe(
+ Date.parse(start),
+ );
+ });
+});
+
+describe("spanLogWindow", () => {
+ it("looks up the request log around the span's own time, not the logs tab window", () => {
+ const spanStart = Date.parse("2026-08-01T12:00:00Z");
+ expect(spanLogWindow(spanStart)).toEqual({ start_date: "2026-08-01 11:30:00", end_date: "2026-08-01 12:30:00" });
+ });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts
new file mode 100644
index 00000000000..4e17331b001
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts
@@ -0,0 +1,91 @@
+import { useInfiniteQuery } from "@tanstack/react-query";
+import moment from "moment";
+import { useMemo } from "react";
+
+import { ApiError } from "@/lib/http/client";
+
+import { agentTraceListCall } from "../../networking";
+import { LIVE_TAIL_INTERVAL_MS } from "../log_filter_logic";
+import type { TracePage, TraceSummary } from "./traceTypes";
+
+export const TRACING_NOT_ENABLED_STATUS = 501;
+/** A proxy without the tracing routes at all answers 404; treat it like tracing being off. */
+const TRACING_ROUTE_MISSING_STATUS = 404;
+
+export const isTracingNotEnabled = (error: unknown): error is ApiError =>
+ error instanceof ApiError &&
+ (error.status === TRACING_NOT_ENABLED_STATUS || error.status === TRACING_ROUTE_MISSING_STATUS);
+
+interface UseAgentTracesOptions {
+ accessToken: string;
+ startTime: string;
+ endTime: string;
+ isCustomDate: boolean;
+ isLiveTail: boolean;
+ enabled: boolean;
+}
+
+export interface AgentTracesResult {
+ traces: TraceSummary[];
+ isLoading: boolean;
+ isFetching: boolean;
+ /** Set when the proxy answered 501: tracing isn't configured. */
+ notEnabledDetail: string | null;
+ error: Error | null;
+ hasMore: boolean;
+ loadMore: () => void;
+ refetch: () => void;
+}
+
+/** Start of the fetch window: a preset range rolls with "now", so live tail keeps a fixed-length window. */
+export const traceWindowStartMs = (startTime: string, endTime: string, isCustomDate: boolean, nowMs: number): number =>
+ isCustomDate ? moment(startTime).valueOf() : nowMs - (moment(endTime).valueOf() - moment(startTime).valueOf());
+
+/**
+ * GET /v1/traces for the Logs page time range, cursor-paginated ("Load more").
+ * Preset ranges re-read "now" on every fetch, moving both bounds so the window keeps its length.
+ */
+export function useAgentTraces({
+ accessToken,
+ startTime,
+ endTime,
+ isCustomDate,
+ isLiveTail,
+ enabled,
+}: UseAgentTracesOptions): AgentTracesResult {
+ const fetchPage = (pageParam: unknown): Promise => {
+ const nowMs = Date.now();
+ const listOptions: Parameters[0] = {
+ accessToken,
+ startMs: traceWindowStartMs(startTime, endTime, isCustomDate, nowMs),
+ endMs: isCustomDate ? moment(endTime).valueOf() : nowMs,
+ cursor: pageParam as string | null,
+ };
+ return agentTraceListCall(listOptions);
+ };
+ const queryOptions: Parameters>[0] = {
+ queryKey: ["agentTraces", accessToken, startTime, endTime, isCustomDate],
+ queryFn: ({ pageParam }) => fetchPage(pageParam),
+ initialPageParam: null,
+ getNextPageParam: (lastPage) => lastPage.next_cursor ?? undefined,
+ enabled,
+ retry: (failureCount, error) => !isTracingNotEnabled(error) && failureCount < 1,
+ refetchInterval: (q) => (isLiveTail && !isTracingNotEnabled(q.state.error) ? LIVE_TAIL_INTERVAL_MS : false),
+ refetchIntervalInBackground: false,
+ };
+ const query = useInfiniteQuery(queryOptions);
+
+ const traces = useMemo(() => query.data?.pages.flatMap((page) => page.data) ?? [], [query.data]);
+ const notEnabled = isTracingNotEnabled(query.error);
+
+ return {
+ traces,
+ isLoading: query.isLoading,
+ isFetching: query.isFetching,
+ notEnabledDetail: notEnabled ? query.error?.message || "Agent tracing is not enabled" : null,
+ error: notEnabled ? null : query.error,
+ hasMore: query.hasNextPage,
+ loadMore: () => void query.fetchNextPage(),
+ refetch: () => void query.refetch(),
+ };
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useSpanRequestLog.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useSpanRequestLog.ts
new file mode 100644
index 00000000000..083dceac790
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useSpanRequestLog.ts
@@ -0,0 +1,47 @@
+"use client";
+
+import { useQuery, type UseQueryOptions } from "@tanstack/react-query";
+import moment from "moment";
+
+import { uiSpendLogsCall } from "../../networking";
+import type { LogEntry } from "../columns";
+
+/** Spend-log timestamps are written when the call finishes, so pad the span start on both sides. */
+const LOOKUP_PAD_MINUTES = 30;
+const SPEND_LOG_TIME_FORMAT = "YYYY-MM-DD HH:mm:ss";
+
+export const spanLogWindow = (spanStartMs: number): { start_date: string; end_date: string } => ({
+ start_date: moment.utc(spanStartMs).subtract(LOOKUP_PAD_MINUTES, "minutes").format(SPEND_LOG_TIME_FORMAT),
+ end_date: moment.utc(spanStartMs).add(LOOKUP_PAD_MINUTES, "minutes").format(SPEND_LOG_TIME_FORMAT),
+});
+
+/**
+ * The LiteLLM request log behind an LLM span, looked up around the span's own time rather than
+ * the Request Logs tab's window, so runs older than that window still resolve.
+ */
+export function useSpanRequestLog(
+ accessToken: string,
+ requestId: string | null,
+ spanStartMs: number,
+ enabled: boolean,
+) {
+ const fetchLog = async (): Promise => {
+ if (requestId === null) return null;
+ const logsOptions: Parameters[0] = {
+ accessToken,
+ ...spanLogWindow(spanStartMs),
+ page: 1,
+ page_size: 1,
+ params: { request_id: requestId },
+ };
+ const response = await uiSpendLogsCall(logsOptions);
+ return response.data.find((log: LogEntry) => log.request_id === requestId) ?? null;
+ };
+ const queryOptions: UseQueryOptions = {
+ queryKey: ["logs", "spanRequest", requestId, spanStartMs, accessToken],
+ queryFn: fetchLog,
+ enabled: enabled && requestId !== null,
+ staleTime: Infinity,
+ };
+ return useQuery(queryOptions);
+}
diff --git a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx
index 70a259ae9eb..e904080ede5 100644
--- a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/index.test.tsx
@@ -65,12 +65,14 @@ describe("SpendLogsTable", () => {
useOrganizationsMock.mockReturnValue({ data: [] });
});
- it("renders the four log tabs", () => {
+ it("renders the log tabs, with Agent Traces marked new and Request Logs selected", () => {
renderAs("Admin");
for (const label of ["Request Logs", "Audit Logs", "Deleted Keys", "Deleted Teams"]) {
expect(screen.getByRole("tab", { name: label })).toBeInTheDocument();
}
+ expect(screen.getByRole("tab", { name: /Agent Traces/ })).toHaveTextContent("New");
+ expect(screen.getByRole("tab", { name: "Request Logs" })).toHaveAttribute("aria-selected", "true");
});
it("marks only the visible tab's panel active so background tabs do not query", async () => {
@@ -115,7 +117,7 @@ describe("SpendLogsTable", () => {
it("does not hand an org admin the Audit Logs tab, which the backend still refuses them", () => {
renderAs("Internal User", ORG_ADMIN_MEMBERSHIPS);
- expect(tabNames()).toEqual(["Request Logs", "Deleted Keys", "Deleted Teams"]);
+ expect(tabNames()).toEqual(["Request Logs", "Agent TracesNew", "Deleted Keys", "Deleted Teams"]);
expect(screen.queryByTestId("audit-logs-panel")).not.toBeInTheDocument();
});
@@ -124,7 +126,7 @@ describe("SpendLogsTable", () => {
{ organization_id: "org-1", members: [{ user_id: "user-1", user_role: "internal_user" }] },
]);
- expect(tabNames()).toEqual(["Request Logs", "Deleted Keys"]);
+ expect(tabNames()).toEqual(["Request Logs", "Agent TracesNew", "Deleted Keys"]);
});
it("activates the org admin's selected tab rather than the one at the four-tab index", async () => {
diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx
index aadf90fad6c..c30a960a500 100644
--- a/ui/litellm-dashboard/src/components/view_logs/index.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx
@@ -4,6 +4,7 @@ import DeletedKeysPage from "../DeletedKeysPage/DeletedKeysPage";
import DeletedTeamsPage from "../DeletedTeamsPage/DeletedTeamsPage";
import AuditLogsPanel from "./AuditLogsPanel";
import RequestLogsPanel from "./RequestLogsPanel";
+import AgentTracesPage from "./TraceView/AgentTracesPage";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
@@ -15,20 +16,24 @@ interface SpendLogsTableProps {
premiumUser: boolean;
}
-type LogsTabId = "request logs" | "audit logs" | "deleted keys" | "deleted teams";
+type LogsTabId = "request logs" | "agent traces" | "audit logs" | "deleted keys" | "deleted teams";
interface LogsTab {
id: LogsTabId;
label: string;
+ isNew?: boolean;
}
const REQUEST_LOGS_TAB: LogsTab = { id: "request logs", label: "Request Logs" };
+const AGENT_TRACES_TAB: LogsTab = { id: "agent traces", label: "Agent Traces", isNew: true };
const AUDIT_LOGS_TAB: LogsTab = { id: "audit logs", label: "Audit Logs" };
const DELETED_KEYS_TAB: LogsTab = { id: "deleted keys", label: "Deleted Keys" };
const DELETED_TEAMS_TAB: LogsTab = { id: "deleted teams", label: "Deleted Teams" };
const tabContentClassName = (tabId: LogsTabId): string =>
- tabId === REQUEST_LOGS_TAB.id ? "flex min-h-0 flex-1 flex-col" : "min-h-0 flex-1 overflow-y-auto";
+ tabId === REQUEST_LOGS_TAB.id || tabId === AGENT_TRACES_TAB.id
+ ? "flex min-h-0 flex-1 flex-col"
+ : "min-h-0 flex-1 overflow-y-auto";
export default function SpendLogsTable({ accessToken, token, userRole, userID, premiumUser }: SpendLogsTableProps) {
const [activeTab, setActiveTab] = useState(REQUEST_LOGS_TAB.id);
@@ -45,6 +50,7 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
const tabs: LogsTab[] = [
REQUEST_LOGS_TAB,
+ AGENT_TRACES_TAB,
...(canViewAuditLogs ? [AUDIT_LOGS_TAB] : []),
DELETED_KEYS_TAB,
...(canViewDeletedTeams ? [DELETED_TEAMS_TAB] : []),
@@ -62,6 +68,8 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
isActive={activeTab === "request logs"}
/>
);
+ case "agent traces":
+ return activeTab === "agent traces" ? : null;
case "audit logs":
return (
- setActiveTab(value as LogsTabId)} className="min-h-0 flex-1">
-
+
+ setActiveTab(value as LogsTabId)}
+ className="min-h-0 flex-1 gap-0"
+ >
+
{tabs.map((tab) => (
-
+
{tab.label}
+ {tab.isNew && (
+
+ New
+
+ )}
))}