Merge remote-tracking branch 'origin/main' into litellm_roi_calculator_v2

litellm/integrations/otel/README.md

@@ -213,6 +213,15 @@ nothing here imports outside it:
   `config.yaml` — the latter reach the config through the logger's constructor
   kwargs. `baggage_team_metadata_keys` is empty by default, so none of a team's
   free-form metadata is promoted until each sub-key is explicitly allowlisted.
+  `excluded_services` withholds datastore spans from key/team `callback_vars`
+  destinations while the operator's own exporters keep them: set
+  `LITELLM_OTEL_EXCLUDED_SERVICES` (comma-separated) or `excluded_services`
+  (a YAML list) under `callback_settings.otel`, naming the datastore services
+  to withhold (`redis`, `postgres`, `batch_write_to_db`, `redis_*`, or their
+  `db.system.name` spellings `redis` / `postgresql`). Unknown names are logged
+  as an error and ignored. A span is withheld when its `db.system.name` /
+  `db.system` attribute is in the set, so request root, auth, guardrail and
+  model spans can never be excluded.
 - [`baggage.py`](./model/baggage.py) — the single definition of which request-identity
   values are promoted into Baggage (so child spans inherit them) and under which
   attribute keys.

litellm/integrations/otel/logger.py

@@ -30,7 +30,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.otel.emitter import SpanEmitter, stamp_error
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.model.baggage import promoted_baggage
-from litellm.integrations.otel.model.config import OpenTelemetryV2Config
+from litellm.integrations.otel.model.config import OpenTelemetryV2Config, excluded_db_systems_from
 from litellm.integrations.otel.model.metadata import (
     LLMCallEvent,
     RequestIdentity,
@@ -898,12 +898,29 @@ def publish_global_otel_v2_provider(
     """
     global _published_v2_provider
     logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
-    attach_tenant_fan_out(logger.tracer_provider, *_v2_configs(in_memory_loggers, logger))
+    attach_tenant_fan_out(
+        logger.tracer_provider,
+        *_v2_configs(in_memory_loggers, logger),
+        excluded_db_systems=_excluded_db_systems(logger),
+    )
     set_global_provider(logger.tracer_provider)
     _published_v2_provider = logger.tracer_provider  # rebind-ok: startup records the one provider carrying the fan-out
     return logger
 
 
+def _excluded_db_systems(logger: "OpenTelemetryV2") -> frozenset[str]:
+    """The datastore services withheld from tenant destinations.
+
+    ``callback_settings.otel.excluded_services`` wins over the env var whichever
+    logger got published: with ``callbacks: [langfuse_otel, otel]`` the ``otel``
+    callback folds into the preset, whose config is env-only.
+    """
+    configured: Final = litellm.callback_settings.get("otel", {}).get("excluded_services")
+    if configured is None:
+        return logger.config.excluded_services
+    return excluded_db_systems_from(configured)
+
+
 def _v2_configs(in_memory_loggers: Sequence[object], logger: "OpenTelemetryV2") -> tuple[OpenTelemetryV2Config, ...]:
     """Every v2 logger's config, the published logger's first.
 
@@ -963,7 +980,11 @@ def fan_out_provider() -> ApiTracerProvider:
         return published
     logger: Final = _registered_v2_logger()
     if logger is not None:
-        attach_tenant_fan_out(logger.tracer_provider, logger.config)
+        attach_tenant_fan_out(
+            logger.tracer_provider,
+            logger.config,
+            excluded_db_systems=_excluded_db_systems(logger),
+        )
         return logger.tracer_provider
     return get_tracer_provider()
 

litellm/integrations/otel/model/config.py

@@ -4,14 +4,16 @@ from enum import Enum
 from functools import lru_cache
 from typing import Annotated, Any, Final
 
-from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
+from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, ValidationError, field_validator, model_validator
 from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
 
+from litellm._logging import verbose_logger
 from litellm.integrations.otel.model.baggage import (
     BAGGAGE_PROMOTED_KEYS,
     DEFAULT_BAGGAGE_METADATA_KEYS,
     DEFAULT_BAGGAGE_TEAM_METADATA_KEYS,
 )
+from litellm.integrations.otel.model.spans import POSTGRESQL, db_system
 from litellm.types.utils import OtelSpanScope
 
 #: Master feature-flag env var. The logger is inert until this is truthy.
@@ -174,6 +176,19 @@ class OpenTelemetryV2Config(BaseSettings):
             "key/team destinations are not affected."
         ),
     )
+    excluded_services: Annotated[frozenset[str], NoDecode] = Field(
+        default_factory=frozenset,
+        validation_alias=AliasChoices("excluded_services", "LITELLM_OTEL_EXCLUDED_SERVICES"),
+        description=(
+            "Datastore services whose spans are withheld from key/team ``callback_vars`` "
+            "OTel destinations (the operator's own exporters still receive them). Accepted "
+            "values are the datastore ``ServiceTypes`` names (``redis``, ``postgres``, "
+            "``batch_write_to_db``, ``redis_*``) or their ``db.system.name`` spellings "
+            "(``redis``, ``postgresql``); stored normalized to ``db.system.name`` values. "
+            "Configure via the ``LITELLM_OTEL_EXCLUDED_SERVICES`` env var (comma-separated) "
+            "or ``callback_settings.otel.excluded_services`` in config.yaml (a YAML list)."
+        ),
+    )
 
     # ----- explicit multi-destination / vocabulary configuration ------------ #
 
@@ -284,6 +299,11 @@ class OpenTelemetryV2Config(BaseSettings):
             return [item.strip() for item in value.split(",") if item.strip()]
         return value
 
+    @field_validator("excluded_services", mode="before")
+    @classmethod
+    def _read_excluded_services(cls, value: object) -> frozenset[str]:
+        return excluded_service_names(value)
+
     @model_validator(mode="after")
     def _normalize(self) -> "OpenTelemetryV2Config":
         # An endpoint with the default exporter kind implies OTLP/HTTP.
@@ -316,6 +336,7 @@ class OpenTelemetryV2Config(BaseSettings):
         if self.legacy_compat and "legacy" not in names:
             names.append("legacy")
         self.mapper_names = names
+        self.excluded_services = _normalize_excluded_services(self.excluded_services)
         return self
 
     @property
@@ -334,3 +355,55 @@ class OpenTelemetryV2Config(BaseSettings):
     @classmethod
     def from_env(cls) -> "OpenTelemetryV2Config":
         return cls()
+
+
+_EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...])
+
+
+def excluded_db_systems_from(value: object) -> frozenset[str]:
+    """Normalize a raw ``excluded_services`` value without building a settings model that rereads the env"""
+    return _normalize_excluded_services(excluded_service_names(value))
+
+
+def excluded_service_names(value: object) -> frozenset[str]:
+    """Read a YAML list or comma-separated string of service names, logging and dropping unusable input
+    so a malformed value cannot stop the OTel logger from being built"""
+    if value is None:
+        return frozenset()
+    try:
+        parsed: Final = _EXCLUDED_SERVICES_INPUT.validate_python(value)
+    except ValidationError:
+        verbose_logger.error("excluded_services must be a list or comma-separated string; %r ignored", value)
+        return frozenset()
+    items: Final = tuple(parsed.split(",")) if isinstance(parsed, str) else parsed
+    return frozenset(name for item in items if (name := _service_name(item)))
+
+
+def _service_name(item: object) -> str:
+    if not isinstance(item, str):
+        verbose_logger.error("excluded_services must be a list of service names; %r ignored", item)
+        return ""
+    return item.strip().lower()
+
+
+def _normalize_excluded_services(services: frozenset[str]) -> frozenset[str]:
+    """Fold each accepted spelling to its ``db.system.name`` value.
+
+    ``postgres`` and ``postgresql`` name the same system, as do every
+    ``ServiceTypes`` member that ``db_system`` maps. Anything else means the
+    operator pointed the setting at a span family it cannot cover; those names
+    are logged and dropped so a typo cannot take the proxy down.
+    """
+    resolved: Final = frozenset(
+        system for service in services if (system := _db_system_for_excluded_service(service)) is not None
+    )
+    return resolved
+
+
+def _db_system_for_excluded_service(service: str) -> str | None:
+    resolved: Final = db_system(service) if service != POSTGRESQL else POSTGRESQL
+    if resolved is None:
+        verbose_logger.error(
+            "excluded_services: %r is not a datastore service; ignored. Allowed: postgres, redis", service
+        )
+    return resolved

litellm/integrations/otel/plumbing/providers.py

@@ -418,6 +418,13 @@ def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool:
     return any(key in attributes for key in _DB_SYSTEM_KEYS)
 
 
+def _is_excluded_database_span(attributes: Mapping[str, AttributeValue], excluded: frozenset[str]) -> bool:
+    if not excluded:
+        return False
+    system: Final = attributes.get(DB.SYSTEM_NAME) or attributes.get(DB.SYSTEM_LEGACY)
+    return isinstance(system, str) and system in excluded
+
+
 def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool:
     return any(key in attributes for key in _TENANT_OWNED_KEYS)
 
@@ -549,10 +556,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
         processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None,
         shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS,
         operator_sinks: 'Mapping[_SinkKey, "OtelSpanScope"]' = MappingProxyType({}),
+        excluded_db_systems: frozenset[str] = frozenset(),
         pending_drains: int = _MAX_PENDING_DRAINS,
         drain_pool: _DrainPool | None = None,
     ) -> None:
         self._operator_sinks: Final = operator_sinks
+        self._excluded_db_systems: Final = excluded_db_systems
         self._drain_seconds: Final = shutdown_drain_seconds
         self._lock: Final = threading.Condition()
         self._closed = False  # guarded by ``_lock``: an unlocked read races the teardown it gates
@@ -567,9 +576,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
 
     def on_end(self, span: ReadableSpan) -> None:
         suppressed: Final = suppressed_backends()
+        attributes: Final = span.attributes or _NO_ATTRIBUTES
         for destination in request_destinations():
-            if self._operator_already_writes(span, destination, suppressed) or not _in_scope(
-                span, destination.span_scope
+            if (
+                self._operator_already_writes(span, destination, suppressed)
+                or not _in_scope(span, destination.span_scope)
+                or _is_excluded_database_span(attributes, self._excluded_db_systems)
             ):
                 continue
             processor = self._acquire(destination)
@@ -1155,7 +1167,9 @@ def build_tracer_provider(
 _FAN_OUT_ATTACH_LOCK: Final = threading.Lock()
 
 
-def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Config) -> None:
+def attach_tenant_fan_out(
+    provider: TracerProvider, *configs: OpenTelemetryV2Config, excluded_db_systems: frozenset[str] = frozenset()
+) -> None:
     """Give ``provider`` the fan-out that delivers spans to key/team destinations.
 
     Called on the one provider published as the OTel global, and idempotent so a
@@ -1164,12 +1178,18 @@ def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Con
     so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
     config per v2 logger since each keeps its own provider and still writes its
     account, so an additive destination pointing at any of them is delivered once
-    rather than twice.
+    rather than twice. ``excluded_db_systems`` only filters what the fan-out
+    delivers, never the operator's own exporters.
     """
     with _FAN_OUT_ATTACH_LOCK:
         if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
             return
-        provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_scopes(*configs)))
+        provider.add_span_processor(
+            TenantFanOutSpanProcessor(
+                operator_sinks=operator_sink_scopes(*configs),
+                excluded_db_systems=excluded_db_systems,
+            )
+        )
 
 
 def deliverable_destinations(

tests/integration/_support/otlp_sink.py

@@ -0,0 +1,528 @@
+"""OTLP/HTTP trace sink: records exported spans and exposes them over a control API.
+
+Accepts ``application/x-protobuf`` ``ExportTraceServiceRequest`` bodies and OTLP
+``http/json`` bodies on any path. Tests read spans through ``recorded_spans`` and
+steer the sink through ``configure``; the process can also be frozen with
+``SIGSTOP``/``SIGCONT`` after reading its pid from ``/__pid``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import os
+import signal
+import socket
+import ssl
+import subprocess
+import sys
+import threading
+import time
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import ExitStack, contextmanager
+from dataclasses import dataclass, field
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import httpx
+import psutil
+from pydantic import JsonValue, TypeAdapter
+from typing_extensions import ReadOnly, TypedDict
+
+INTERNAL_MARKERS: Final = ("gen_ai.operation.name", "mcp.method.name", "litellm.guardrail_name")
+
+
+class Span(TypedDict):
+    trace_id: ReadOnly[str]
+    span_id: ReadOnly[str]
+    parent_span_id: ReadOnly[str]
+    kind: ReadOnly[int]
+    name: ReadOnly[str]
+    attributes: ReadOnly[Mapping[str, JsonValue]]
+    resource: ReadOnly[Mapping[str, JsonValue]]
+
+
+class _SpanListing(TypedDict):
+    next: ReadOnly[int]
+    spans: ReadOnly[list[Span]]
+
+
+_SPAN_LISTING: Final = TypeAdapter(_SpanListing)
+
+
+def _proto_spans(body: bytes) -> list[Span]:
+    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
+    from opentelemetry.proto.common.v1.common_pb2 import AnyValue
+
+    def scalar(value: AnyValue) -> JsonValue:
+        match value.WhichOneof("value"):
+            case "string_value":
+                return value.string_value
+            case "bool_value":
+                return value.bool_value
+            case "int_value":
+                return int(value.int_value)
+            case "double_value":
+                return value.double_value
+            case "bytes_value":
+                return value.bytes_value.decode("utf-8", errors="replace")
+            case "array_value":
+                return [scalar(item) for item in value.array_value.values]
+            case "kvlist_value":
+                return {pair.key: scalar(pair.value) for pair in value.kvlist_value.values}
+            case _:
+                return None
+
+    request: Final = ExportTraceServiceRequest()
+    request.ParseFromString(body)
+    return [
+        Span(
+            trace_id=span.trace_id.hex(),
+            span_id=span.span_id.hex(),
+            parent_span_id=span.parent_span_id.hex(),
+            kind=span.kind,
+            name=span.name,
+            attributes={attribute.key: scalar(attribute.value) for attribute in span.attributes},
+            resource={attribute.key: scalar(attribute.value) for attribute in resource.resource.attributes},
+        )
+        for resource in request.resource_spans
+        for scope in resource.scope_spans
+        for span in scope.spans
+    ]
+
+
+def _json_spans(body: bytes) -> list[Span]:
+    payload: Final = json.loads(body)
+
+    def scalar(value: object) -> JsonValue:
+        if not isinstance(value, dict):
+            return value if isinstance(value, (str, int, float, bool)) or value is None else str(value)
+        for key in ("stringValue", "intValue", "doubleValue", "boolValue", "bytesValue"):
+            if key in value:
+                return value[key]
+        if "arrayValue" in value:
+            return [scalar(item) for item in value["arrayValue"].get("values", [])]
+        if "kvlistValue" in value:
+            return {pair["key"]: scalar(pair["value"]) for pair in value["kvlistValue"].get("values", [])}
+        return None
+
+    return [
+        Span(
+            trace_id=str(span.get("traceId", "")),
+            span_id=str(span.get("spanId", "")),
+            parent_span_id=str(span.get("parentSpanId", "")),
+            kind=int(span.get("kind", 0)),
+            name=str(span.get("name", "")),
+            attributes={attribute["key"]: scalar(attribute.get("value")) for attribute in span.get("attributes", [])},
+            resource={
+                attribute["key"]: scalar(attribute.get("value"))
+                for attribute in resource.get("resource", {}).get("attributes", [])
+            },
+        )
+        for resource in payload.get("resourceSpans", [])
+        for scope in resource.get("scopeSpans", [])
+        for span in scope.get("spans", [])
+    ]
+
+
+def decode_spans(body: bytes, content_type: str) -> list[Span]:
+    if "protobuf" in content_type:
+        return _proto_spans(body)
+    return _json_spans(body)
+
+
+def span_class(span: Span) -> str:
+    if span["kind"] == 2:
+        return "root"
+    if any(marker in span["attributes"] for marker in INTERNAL_MARKERS):
+        return "tenant"
+    return "internal"
+
+
+def spans_for_trace(spans: tuple[Span, ...], trace_id: str) -> tuple[Span, ...]:
+    return tuple(span for span in spans if span["trace_id"] == trace_id)
+
+
+@dataclass(slots=True)
+class _State:
+    spans: list[Span] = field(default_factory=list)
+    requests: list[dict[str, JsonValue]] = field(default_factory=list)
+    status: int = 200
+    delay_seconds: float = 0.0
+    pause: threading.Event = field(default_factory=threading.Event)
+
+    def __post_init__(self) -> None:
+        self.pause.set()
+
+
+class _Handler(BaseHTTPRequestHandler):
+    state: _State
+    protocol_version = "HTTP/1.1"
+
+    def _read_body(self) -> bytes:
+        return self.rfile.read(int(self.headers.get("content-length", "0")))
+
+    def _send_json(self, payload: object, status: int = 200) -> None:
+        body: Final = json.dumps(payload).encode()
+        self.send_response(status)
+        self.send_header("content-type", "application/json")
+        self.send_header("content-length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _record(self) -> None:
+        body: Final = self._read_body()
+        self.state.pause.wait(timeout=120)
+        if self.state.delay_seconds > 0:
+            time.sleep(self.state.delay_seconds)
+        recorded: Final = decode_spans(body, self.headers.get("content-type", ""))
+        self.state.spans.extend(recorded)
+        self.state.requests.append(
+            {
+                "path": self.path,
+                "count": len(recorded),
+                "host": self.headers.get("host", ""),
+                "headers": dict(self.headers),
+            }
+        )
+        self._send_json({"recorded": len(recorded)}, status=self.state.status)
+
+    do_POST = _record
+    do_PUT = _record
+
+    def do_GET(self) -> None:
+        parsed: Final = urlparse(self.path)
+        if parsed.path == "/__spans":
+            since: Final = int(dict(part.split("=", 1) for part in parsed.query.split("&") if part).get("since", "0"))
+            self._send_json({"next": len(self.state.spans), "spans": self.state.spans[since:]})
+            return
+        if parsed.path == "/__pid":
+            self._send_json({"pid": os.getpid()})
+            return
+        if parsed.path == "/__requests":
+            self._send_json({"requests": self.state.requests})
+            return
+        self._send_json({"error": "unknown"}, status=404)
+
+    def do_DELETE(self) -> None:
+        if urlparse(self.path).path == "/__spans":
+            self.state.spans.clear()
+            self.state.requests.clear()
+            self._send_json({"cleared": True})
+            return
+        self._send_json({"error": "unknown"}, status=404)
+
+    def do_PATCH(self) -> None:
+        if urlparse(self.path).path != "/__control":
+            self._send_json({"error": "unknown"}, status=404)
+            return
+        fields: Final = json.loads(self._read_body() or b"{}")
+        if "status" in fields:
+            self.state.status = int(fields["status"])
+        if "delay_seconds" in fields:
+            self.state.delay_seconds = float(fields["delay_seconds"])
+        if fields.get("paused") is True:
+            self.state.pause.clear()
+        if fields.get("paused") is False:
+            self.state.pause.set()
+        self._send_json({"status": self.state.status, "delay_seconds": self.state.delay_seconds})
+
+    def log_message(self, format: str, *args: object) -> None:
+        pass
+
+
+class _ConnectHandler(_Handler):
+    tunnel_context: ssl.SSLContext
+
+    def do_CONNECT(self) -> None:
+        self.state.requests.append({"connect": self.path})
+        self.connection.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n")
+        wrapped: Final = self.tunnel_context.wrap_socket(self.connection, server_side=True)
+        self.close_connection = True
+        type(self)(wrapped, self.client_address, self.server)
+
+
+_MITM_HOSTS: Final = ("otlp.nr-data.net", "otlp.eu01.nr-data.net")
+
+
+def _mitm_context(directory: Path) -> ssl.SSLContext:
+    from cryptography import x509
+    from cryptography.hazmat.primitives import hashes, serialization
+    from cryptography.hazmat.primitives.asymmetric import rsa
+    from cryptography.x509.oid import NameOID
+
+    directory.mkdir(parents=True, exist_ok=True)
+    now: Final = datetime.datetime.now(datetime.timezone.utc)
+    window: Final = datetime.timedelta(days=2)
+    ca_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    ca_name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "otlp-sink test CA")])
+    ca_cert: Final = (
+        x509.CertificateBuilder()
+        .subject_name(ca_name)
+        .issuer_name(ca_name)
+        .public_key(ca_key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(now - window)
+        .not_valid_after(now + window)
+        .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+        .sign(ca_key, hashes.SHA256())
+    )
+    leaf_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    leaf_cert: Final = (
+        x509.CertificateBuilder()
+        .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, _MITM_HOSTS[0])]))
+        .issuer_name(ca_cert.subject)
+        .public_key(leaf_key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(now - window)
+        .not_valid_after(now + window)
+        .add_extension(x509.SubjectAlternativeName([x509.DNSName(host) for host in _MITM_HOSTS]), critical=False)
+        .sign(ca_key, hashes.SHA256())
+    )
+    ca_pem: Final = directory / "ca.pem"
+    ca_pem.write_bytes(ca_cert.public_bytes(serialization.Encoding.PEM))
+    leaf_pem: Final = directory / "leaf.pem"
+    leaf_pem.write_bytes(leaf_cert.public_bytes(serialization.Encoding.PEM))
+    leaf_key_pem: Final = directory / "leaf-key.pem"
+    leaf_key_pem.write_bytes(
+        leaf_key.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.TraditionalOpenSSL,
+            serialization.NoEncryption(),
+        )
+    )
+    context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    context.load_cert_chain(str(leaf_pem), str(leaf_key_pem))
+    return context
+
+
+def _grpc_trace_server(state: _State, port: int) -> object:
+    from concurrent import futures
+
+    import grpc
+    from opentelemetry.proto.collector.trace.v1 import trace_service_pb2, trace_service_pb2_grpc
+
+    class _TraceService(trace_service_pb2_grpc.TraceServiceServicer):
+        def Export(self, request: object, context: grpc.ServicerContext) -> object:
+            state.pause.wait(timeout=120)
+            if state.delay_seconds > 0:
+                time.sleep(state.delay_seconds)
+            recorded: Final = _proto_spans(request.SerializeToString())
+            state.spans.extend(recorded)
+            state.requests.append(
+                {
+                    "grpc": "Export",
+                    "metadata": {key: value for key, value in context.invocation_metadata()},
+                    "count": len(recorded),
+                }
+            )
+            return trace_service_pb2.ExportTraceServiceResponse()
+
+    server: Final = grpc.server(futures.ThreadPoolExecutor(max_workers=4))
+    trace_service_pb2_grpc.add_TraceServiceServicer_to_server(_TraceService(), server)
+    server.add_insecure_port(f"127.0.0.1:{port}")
+    server.start()
+    return server
+
+
+def recorded_spans(url: str, since: int = 0) -> tuple[int, tuple[Span, ...]]:
+    response: Final = httpx.get(f"{url}/__spans", params={"since": since}, trust_env=False, timeout=15)
+    response.raise_for_status()
+    listing: Final = _SPAN_LISTING.validate_python(response.json())
+    return listing["next"], tuple(listing["spans"])
+
+
+def configure_sink(url: str, **fields: JsonValue) -> None:
+    httpx.request("PATCH", f"{url}/__control", json=dict(fields), trust_env=False, timeout=15).raise_for_status()
+
+
+def reset_sink(url: str) -> None:
+    httpx.delete(f"{url}/__spans", trust_env=False, timeout=15).raise_for_status()
+
+
+def sink_pid(url: str) -> int:
+    return int(httpx.get(f"{url}/__pid", trust_env=False, timeout=15).json()["pid"])
+
+
+_REQUEST_LISTING: Final = TypeAdapter(list[dict[str, JsonValue]])
+
+
+def recorded_requests(url: str) -> tuple[Mapping[str, JsonValue], ...]:
+    response: Final = httpx.get(f"{url}/__requests", trust_env=False, timeout=15)
+    response.raise_for_status()
+    return tuple(_REQUEST_LISTING.validate_python(response.json()["requests"]))
+
+
+@dataclass(frozen=True, slots=True)
+class SpanSinks:
+    operator: str
+    tenant: str
+    arize: str
+
+
+@dataclass(frozen=True, slots=True)
+class GrpcSink:
+    url: str
+    control_url: str
+
+
+@dataclass(frozen=True, slots=True)
+class ConnectSink:
+    proxy_url: str
+    control_url: str
+    ca_pem: str
+
+
+def _free_port() -> int:
+    with socket.socket() as reserve:
+        reserve.bind(("127.0.0.1", 0))
+        return int(reserve.getsockname()[1])
+
+
+def _pid_reachable(url: str) -> bool:
+    try:
+        return httpx.get(f"{url}/__pid", trust_env=False, timeout=2).status_code == 200
+    except httpx.TransportError:
+        return False
+
+
+@contextmanager
+def owned_sinks(directory: Path) -> Iterator[SpanSinks]:
+    from integration._support.process import group_members, signal_group, stop_root_process
+
+    directory.mkdir(parents=True, exist_ok=True)
+    ports: Final = tuple(_free_port() for _ in range(3))
+    root: Final = Path(__file__).resolve().parents[3]
+    with ExitStack() as stack:
+        processes: Final = tuple(
+            subprocess.Popen(
+                [sys.executable, "-P", "-m", "integration._support.otlp_sink", "--port", str(port)],
+                cwd=root,
+                stdout=stack.enter_context((directory / f"otlp-sink-{port}.log").open("w")),
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+            )
+            for port in ports
+        )
+        try:
+            urls: Final = tuple(f"http://127.0.0.1:{port}" for port in ports)
+            deadline: Final = time.monotonic() + 30
+            while True:
+                alive: Final = all(process.poll() is None for process in processes)
+                assert alive, "OTLP sink exited before readiness"
+                if all(_pid_reachable(url) for url in urls):
+                    break
+                assert time.monotonic() < deadline, "OTLP sink readiness deadline exceeded"
+                time.sleep(0.05)
+            yield SpanSinks(operator=urls[0], tenant=urls[1], arize=urls[2])
+        finally:
+            for process in processes:
+                stopped: Final = stop_root_process(process)
+                residual: Final = group_members(process.pid)
+                if residual:
+                    signal_group(process.pid, signal.SIGKILL)
+                    psutil.wait_procs(residual, timeout=5)
+                survivors: Final = group_members(process.pid)
+                assert not survivors and stopped, "OTLP sink required forced cleanup"
+
+
+@contextmanager
+def _spawn_sink(directory: Path, log_name: str, argv: Sequence[str]) -> Iterator[None]:
+    from integration._support.process import group_members, signal_group, stop_root_process
+
+    directory.mkdir(parents=True, exist_ok=True)
+    root: Final = Path(__file__).resolve().parents[3]
+    with (directory / log_name).open("w") as log:
+        process: Final = subprocess.Popen(
+            [sys.executable, "-P", "-m", "integration._support.otlp_sink", *argv],
+            cwd=root,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+        try:
+            yield
+        finally:
+            stopped: Final = stop_root_process(process)
+            residual: Final = group_members(process.pid)
+            if residual:
+                signal_group(process.pid, signal.SIGKILL)
+                psutil.wait_procs(residual, timeout=5)
+            survivors: Final = group_members(process.pid)
+            assert not survivors and stopped, "OTLP sink required forced cleanup"
+
+
+def _await_sink(url: str) -> None:
+    deadline: Final = time.monotonic() + 30
+    while not _pid_reachable(url):
+        assert time.monotonic() < deadline, "OTLP sink readiness deadline exceeded"
+        time.sleep(0.05)
+
+
+@contextmanager
+def owned_grpc_sink(directory: Path) -> Iterator[GrpcSink]:
+    http_port: Final = _free_port()
+    grpc_port: Final = _free_port()
+    with _spawn_sink(
+        directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)]
+    ):
+        control_url: Final = f"http://127.0.0.1:{http_port}"
+        _await_sink(control_url)
+        yield GrpcSink(url=f"http://127.0.0.1:{grpc_port}", control_url=control_url)
+
+
+@contextmanager
+def owned_connect_sink(directory: Path) -> Iterator[ConnectSink]:
+    http_port: Final = _free_port()
+    tunnel_port: Final = _free_port()
+    ca_dir: Final = directory / "mitm"
+    with _spawn_sink(
+        directory,
+        "otlp-connect-sink.log",
+        ["--port", str(http_port), "--connect-port", str(tunnel_port), "--ca-dir", str(ca_dir)],
+    ):
+        control_url: Final = f"http://127.0.0.1:{http_port}"
+        _await_sink(control_url)
+        yield ConnectSink(
+            proxy_url=f"http://127.0.0.1:{tunnel_port}",
+            control_url=control_url,
+            ca_pem=str(ca_dir / "ca.pem"),
+        )
+
+
+def main() -> None:
+    parser: Final = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, required=True)
+    parser.add_argument("--grpc-port", type=int, default=0)
+    parser.add_argument("--connect-port", type=int, default=0)
+    parser.add_argument("--ca-dir", type=Path, default=None)
+    arguments: Final = parser.parse_args()
+    bound_state: Final = _State()
+
+    class BoundHandler(_Handler):
+        state = bound_state
+
+    if arguments.grpc_port:
+        grpc_server: Final = _grpc_trace_server(bound_state, arguments.grpc_port)
+        assert grpc_server is not None
+    if arguments.connect_port:
+        assert arguments.ca_dir is not None, "--connect-port needs --ca-dir"
+        bound_context: Final = _mitm_context(arguments.ca_dir)
+
+        class BoundConnectHandler(_ConnectHandler):
+            state = bound_state
+            tunnel_context = bound_context  # pyright: ignore[reportIncompatibleVariableOverride]  # bound context, not a new field
+
+        tunnel: Final = ThreadingHTTPServer(("127.0.0.1", arguments.connect_port), BoundConnectHandler)
+        tunnel.daemon_threads = True
+        threading.Thread(target=tunnel.serve_forever, daemon=True).start()
+    server: Final = ThreadingHTTPServer(("127.0.0.1", arguments.port), BoundHandler)
+    server.daemon_threads = True
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

tests/integration/observability/conftest.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlparse
+
+import pytest
+import yaml
+from integration._support.otlp_sink import SpanSinks, owned_sinks
+from pydantic import JsonValue
+
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+
+@pytest.fixture(scope="module")
+def audit_sinks(tmp_path_factory: pytest.TempPathFactory) -> Iterator[SpanSinks]:
+    directory: Final = tmp_path_factory.mktemp("otel-audit-sinks")
+    with owned_sinks(directory) as sinks:
+        yield sinks
+
+
+@pytest.fixture(scope="module")
+def otel_audit_config(audit_sinks: SpanSinks) -> AuditConfigWriter:
+    tenant_host: Final = urlparse(audit_sinks.tenant).netloc
+
+    def write(directory: Path, litellm_settings: Mapping[str, JsonValue] = {}) -> Path:
+        config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+        config["litellm_settings"] = {
+            **config.get("litellm_settings", {}),
+            "callbacks": ["otel"],
+            "provider_url_destination_allowed_hosts": [tenant_host],
+            **dict(litellm_settings),
+        }
+        config["callback_settings"] = {
+            "otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True}
+        }
+        config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2}
+        path: Final = directory / f"otel-audit-{uuid.uuid4().hex}.yaml"
+        path.write_text(yaml.safe_dump(config))
+        return path
+
+    return write
+
+
+@pytest.fixture(scope="module")
+def langfuse_vars(audit_sinks: SpanSinks) -> dict[str, JsonValue]:
+    return {
+        "langfuse_public_key": "pk-lf-audit",
+        "langfuse_secret_key": "sk-lf-audit",
+        "langfuse_host": audit_sinks.tenant,
+    }

tests/integration/observability/test_otel_excluded_services.py

@@ -0,0 +1,392 @@
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import (
+    Gateway,
+    eventually,
+    gateway_from_environment,
+)
+from integration._support.otlp_sink import (
+    Span,
+    SpanSinks,
+    recorded_spans,
+    spans_for_trace,
+)
+from integration._support.process import owned_proxy, owned_proxy_process
+from pydantic import JsonValue
+
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+DB_SYSTEM_KEYS: Final = frozenset({"db.system.name", "db.system"})
+
+
+@pytest.fixture(scope="module")
+def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]:
+    with gateway_from_environment() as base:
+        yield base
+
+
+def _config_with(
+    directory: Path,
+    otel_audit_config: AuditConfigWriter,
+    *,
+    otel: Mapping[str, JsonValue] = MappingProxyType({}),
+    extra: Callable[[dict[str, JsonValue]], None] | None = None,
+) -> Path:
+    config: Final = yaml.safe_load(otel_audit_config(directory, {}).read_text())
+    config["callback_settings"]["otel"].update(dict(otel))
+    if extra is not None:
+        extra(config)
+    path: Final = directory / f"otel-excl-{uuid.uuid4().hex}.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+def _operator_langfuse(audit_sinks: SpanSinks) -> dict[str, str]:
+    return {
+        "LANGFUSE_HOST": audit_sinks.operator,
+        "LANGFUSE_PUBLIC_KEY": "pk-lf-operator",
+        "LANGFUSE_SECRET_KEY": "sk-lf-operator",
+        "OTEL_EXPORTER": "http/json",
+        "OTEL_ENDPOINT": audit_sinks.operator,
+    }
+
+
+def _add_callback(gateway: Gateway, team_id: str, callback_vars: Mapping[str, JsonValue]) -> httpx.Response:
+    return gateway.request(
+        "POST",
+        f"/team/{team_id}/callback",
+        {"callback_name": "langfuse_otel", "callback_vars": dict(callback_vars)},
+    )
+
+
+def _drive(candidate: Gateway, langfuse_vars: Mapping[str, JsonValue]) -> httpx.Response:
+    with candidate.scenario() as scenario:
+        model: Final = scenario.model(model="openai/audit-chat", api_base=f"{candidate.upstream_url}/v1")
+        team_id: Final = scenario.team()
+        callback: Final = _add_callback(candidate, team_id, langfuse_vars)
+        assert callback.status_code == 200, callback.text
+        key: Final = scenario.key(team_id=team_id)
+        response: Final = candidate.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": model, "messages": [{"role": "user", "content": f"otel-excl-{uuid.uuid4().hex}"}]},
+            key=key,
+        )
+        assert response.status_code == 200, response.text
+        return response
+
+
+def _trace_id(sink_url: str, response: httpx.Response, seconds: float = 40) -> str:
+    call_id: Final = response.headers.get("x-litellm-call-id")
+    response_id: Final = response.json().get("id")
+
+    def look() -> str | None:
+        _, spans = recorded_spans(sink_url)
+        return next(
+            (
+                str(span["trace_id"])
+                for span in spans
+                if (call_id is not None and span["attributes"].get("litellm.call_id") == call_id)
+                or (response_id is not None and span["attributes"].get("gen_ai.response.id") == response_id)
+            ),
+            None,
+        )
+
+    found: Final = eventually(look, lambda value: value is not None, seconds=seconds)
+    assert found is not None
+    return found
+
+
+def _trace_spans(sink_url: str, trace_id: str, seconds: float = 30) -> tuple[Span, ...]:
+    """The trace's spans once the post-call tail has landed.
+
+    The spend-writer and other post-response spans flush after the request
+    answers, so absence assertions poll for the whole window instead of
+    settling at the first glimpse of the root span.
+    """
+    deadline: Final = time.monotonic() + seconds
+    group: tuple[Span, ...] = ()  # rebind-ok: drains samples until the post-call tail lands
+    while time.monotonic() < deadline:
+        _, spans = recorded_spans(sink_url)
+        group = spans_for_trace(spans, trace_id)
+        time.sleep(0.5)
+    assert group, f"trace {trace_id} never reached {sink_url}"
+    return group
+
+
+def _await_db_span(sink_url: str, trace_id: str | None, needle: str, seconds: float = 40, since: int = 0) -> None:
+    def seen() -> bool:
+        _, spans = recorded_spans(sink_url, since)
+        group: Final = spans if trace_id is None else spans_for_trace(spans, trace_id)
+        return any(
+            needle in str(span["name"]) or needle in {str(span["attributes"].get(k)) for k in DB_SYSTEM_KEYS}
+            for span in group
+        )
+
+    landed: Final = eventually(seen, bool, seconds=seconds)
+    assert landed, f"{needle} span never landed at {sink_url}"
+
+
+def _db_systems(spans: tuple[Span, ...]) -> set[str]:
+    return {str(span["attributes"][key]) for span in spans for key in DB_SYSTEM_KEYS if key in span["attributes"]}
+
+
+def _assert_core_spans_present(spans: tuple[Span, ...]) -> None:
+    attributes_by_span: Final = tuple(span["attributes"] for span in spans)
+    assert any(span["kind"] == 2 for span in spans), "request root span missing"
+    assert any("gen_ai.operation.name" in attrs for attrs in attributes_by_span), "model span missing"
+    assert any("litellm.guardrail.name" in attrs for attrs in attributes_by_span), "guardrail span missing"
+    names: Final = sorted(str(span["name"]) for span in spans)
+    assert any(name.startswith("auth") for name in names), f"auth span missing in {names}"
+
+
+def _assert_tenant_keeps_redis_without_postgres(
+    candidate: Gateway, audit_sinks: SpanSinks, langfuse_vars: Mapping[str, JsonValue]
+) -> None:
+    tenant_start, _ = recorded_spans(audit_sinks.tenant)
+    operator_start, _ = recorded_spans(audit_sinks.operator)
+    traffic: Final = _drive(candidate, langfuse_vars)
+    _await_db_span(audit_sinks.operator, None, "batch_write_to_db", seconds=60, since=operator_start)
+    tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+    _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+    systems: Final = _db_systems(_trace_spans(audit_sinks.tenant, tenant_trace, seconds=15))
+    assert "redis" in systems, f"redis spans missing at tenant: {systems}"
+    _, all_tenant = recorded_spans(audit_sinks.tenant, tenant_start)
+    assert "postgresql" not in _db_systems(all_tenant), f"postgresql spans reached tenant: {_db_systems(all_tenant)}"
+
+
+def _guardrail_block(config: dict) -> None:
+    config["guardrails"] = [
+        {
+            "guardrail_name": f"excl-filter-{uuid.uuid4().hex[:8]}",
+            "litellm_params": {
+                "guardrail": "litellm_content_filter",
+                "mode": "pre_call",
+                "default_on": True,
+                "patterns": [
+                    {
+                        "pattern_type": "regex",
+                        "pattern_name": "excl_secret",
+                        "pattern": "TOPSECRET\\d{9}",
+                        "action": "BLOCK",
+                    }
+                ],
+            },
+        }
+    ]
+
+
+@pytest.mark.timeout(180)
+def test_excluded_services_drops_db_spans_at_tenant_only(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(
+        tmp_path, otel_audit_config, otel={"excluded_services": ["redis", "postgres"]}, extra=_guardrail_block
+    )
+    with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+        ten_start, _ = recorded_spans(audit_sinks.tenant)
+        op_start, _ = recorded_spans(audit_sinks.operator)
+        traffic: Final = _drive(candidate, langfuse_vars)
+        _await_db_span(audit_sinks.operator, None, "postgresql", seconds=60, since=op_start)
+        tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+        tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace)
+        _assert_core_spans_present(tenant_spans)
+        assert _db_systems(tenant_spans) == set(), (
+            f"db spans reached tenant: {sorted(str(s['name']) for s in tenant_spans)}"
+        )
+        operator_trace: Final = _trace_id(audit_sinks.operator, traffic)
+        assert operator_trace == tenant_trace
+        trace_systems: Final = _db_systems(_trace_spans(audit_sinks.operator, operator_trace))
+        assert "redis" in trace_systems, f"operator trace lost redis spans: {trace_systems}"
+        _, all_operator = recorded_spans(audit_sinks.operator, op_start)
+        operator_systems: Final = _db_systems(all_operator)
+        assert "postgresql" in operator_systems, f"operator lost aux db spans: {operator_systems}"
+        _, all_tenant = recorded_spans(audit_sinks.tenant, ten_start)
+        names: Final = sorted(str(span["name"]) for span in all_tenant)
+        assert _db_systems(all_tenant) == set(), f"aux db spans reached tenant: {names}"
+        assert not any("batch_write_to_db" in name for name in names), f"spend writer reached tenant: {names}"
+
+
+@pytest.mark.timeout(180)
+def test_without_excluded_services_the_tenant_still_gets_redis_and_postgres_spans(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(tmp_path, otel_audit_config, extra=_guardrail_block)
+    with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+        tenant_start, _ = recorded_spans(audit_sinks.tenant)
+        traffic: Final = _drive(candidate, langfuse_vars)
+        _await_db_span(audit_sinks.tenant, None, "batch_write_to_db", seconds=60, since=tenant_start)
+        tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+        _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+        _assert_core_spans_present(_trace_spans(audit_sinks.tenant, tenant_trace, seconds=15))
+        _, all_tenant = recorded_spans(audit_sinks.tenant, tenant_start)
+        systems: Final = _db_systems(all_tenant)
+        assert {"redis", "postgresql"} <= systems, f"datastore spans missing at tenant: {systems}"
+
+
+def test_env_excluded_services_drops_only_redis(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(tmp_path, otel_audit_config)
+    with owned_proxy(
+        gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "redis"}, config=config, workers=2
+    ) as candidate:
+        start, _ = recorded_spans(audit_sinks.tenant)
+        _drive(candidate, langfuse_vars)
+        _await_db_span(audit_sinks.tenant, None, "postgresql", seconds=60, since=start)
+        _, tenant_spans = recorded_spans(audit_sinks.tenant, start)
+        systems: Final = _db_systems(tenant_spans)
+        assert "postgresql" in systems, f"postgresql spans missing at tenant: {systems}"
+        assert "redis" not in systems, f"redis spans reached tenant: {sorted(str(s['name']) for s in tenant_spans)}"
+
+
+@pytest.mark.timeout(180)
+def test_config_excluded_services_wins_over_env(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    def with_langfuse_otel(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["otel", "langfuse_otel"]
+
+    config: Final = _config_with(
+        tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=with_langfuse_otel
+    )
+    with owned_proxy(
+        gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "redis"}, config=config, workers=2
+    ) as candidate:
+        _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_excluded_services_applies_with_preset_ordered_first(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    def preset_first(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["langfuse_otel", "otel"]
+
+    config: Final = _config_with(
+        tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=preset_first
+    )
+    overrides: Final = {"LITELLM_OTEL_V2": "1", **_operator_langfuse(audit_sinks)}
+    with owned_proxy(gateway, tmp_path, overrides, config=config, workers=2) as candidate:
+        _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_service_logs_error_and_drops_at_proxy_start(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["auth", "postgres"]})
+    with owned_proxy_process(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as owned:
+        assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+        _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_valid_config_excluded_services_tolerates_bogus_env(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]})
+    with owned_proxy(
+        gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth"}, config=config, workers=2
+    ) as candidate:
+        _assert_tenant_keeps_redis_without_postgres(candidate, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_services_env_logs_and_drops_with_preset_alongside_otel(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    def with_langfuse_otel(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["otel", "langfuse_otel"]
+
+    config: Final = _config_with(tmp_path, otel_audit_config, extra=with_langfuse_otel)
+    overrides: Final = {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth,postgres"}
+    with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned:
+        assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+        _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+@pytest.mark.timeout(180)
+def test_bogus_excluded_services_env_logs_and_drops_without_otel_callback(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    def presets_only(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["langfuse_otel"]
+
+    config: Final = _config_with(tmp_path, otel_audit_config, extra=presets_only)
+    overrides: Final = {
+        "LITELLM_OTEL_V2": "1",
+        "LITELLM_OTEL_EXCLUDED_SERVICES": "auth,postgres",
+        **_operator_langfuse(audit_sinks),
+    }
+    with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned:
+        assert "'auth' is not a datastore service; ignored" in owned.log.read_text(), owned.log.read_text()[-3000:]
+        _assert_tenant_keeps_redis_without_postgres(owned.gateway, audit_sinks, langfuse_vars)
+
+
+def test_postgres_exclusion_covers_batch_write_to_db(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    config: Final = _config_with(tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]})
+    with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+        op_start, _ = recorded_spans(audit_sinks.operator)
+        ten_start, _ = recorded_spans(audit_sinks.tenant)
+        traffic: Final = _drive(candidate, langfuse_vars)
+        _await_db_span(audit_sinks.operator, None, "batch_write_to_db", seconds=60, since=op_start)
+        tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+        _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+        tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace, seconds=15)
+        _, all_tenant = recorded_spans(audit_sinks.tenant, ten_start)
+        names: Final = sorted(str(span["name"]) for span in all_tenant)
+        assert "redis" in _db_systems(tenant_spans), f"redis spans missing at tenant: {names}"
+        assert not any("batch_write_to_db" in name for name in names), f"spend writer reached tenant: {names}"

tests/integration/observability/test_otel_excluded_services_matrix.py

@@ -0,0 +1,713 @@
+import asyncio
+import json
+import os
+import re
+import signal
+import uuid
+from collections.abc import Callable, Generator, Iterator, Mapping
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, Literal
+
+import anthropic
+import httpx
+import openai
+import psutil
+import pytest
+import yaml
+from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value
+from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_spans, spans_for_trace
+from integration._support.process import OwnedProxy, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+MARKER: Final = re.compile(rb"excl-[0-9a-f]{32}")
+FAILING: Final = re.compile(rb"excl-fail-[0-9a-f]{32}")
+JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+REPLY_TEXT: Final = "excluded ok"
+SERVER: Final = 2
+INVALID_NAME_LOG: Final = "is not a datastore service"
+INVALID_VALUE_LOG: Final = "excluded_services must be"
+Endpoint = Literal["chat", "responses", "messages"]
+Client = Literal["raw", "sdk", "async_sdk"]
+ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "responses", "messages")
+CLIENTS: Final[tuple[Client, ...]] = ("raw", "sdk", "async_sdk")
+AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]
+
+
+def _marker() -> str:
+    return "excl-" + uuid.uuid4().hex
+
+
+def _chat_reply(identity: str, stream: bool) -> Reply:
+    if not stream:
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": identity,
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {"index": 0, "message": {"role": "assistant", "content": REPLY_TEXT}, "finish_reason": "stop"}
+                    ],
+                    "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9},
+                }
+            ).encode()
+        )
+    chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"}
+    first, _, rest = REPLY_TEXT.partition(" ")
+    deltas: Final[tuple[dict[str, JsonValue], ...]] = (
+        {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": first}}]},
+        {**chunk, "choices": [{"index": 0, "delta": {"content": " " + rest}, "finish_reason": "stop"}]},
+        {**chunk, "choices": [], "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}},
+    )
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"),
+    )
+
+
+def _responses_reply(identity: str, stream: bool) -> Reply:
+    response: Final[dict[str, JsonValue]] = {
+        "id": identity,
+        "object": "response",
+        "created_at": 1,
+        "status": "completed",
+        "model": "gpt-4o-mini",
+        "output": [
+            {
+                "id": "msg_" + identity,
+                "type": "message",
+                "role": "assistant",
+                "status": "completed",
+                "content": [{"type": "output_text", "text": REPLY_TEXT, "annotations": []}],
+            }
+        ],
+        "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9},
+    }
+    if not stream:
+        return Reply(body=json.dumps(response).encode())
+    events: Final[tuple[dict[str, JsonValue], ...]] = (
+        {
+            "type": "response.created",
+            "sequence_number": 0,
+            "response": {**response, "status": "in_progress", "output": []},
+        },
+        {
+            "type": "response.output_text.delta",
+            "sequence_number": 1,
+            "item_id": "msg_" + identity,
+            "output_index": 0,
+            "content_index": 0,
+            "delta": REPLY_TEXT,
+        },
+        {"type": "response.completed", "sequence_number": 2, "response": response},
+    )
+    return Reply(
+        content_type="text/event-stream",
+        chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events),
+    )
+
+
+def _upstream(request: Request) -> Reply:
+    if FAILING.search(request.body) is not None:
+        return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}')
+    found: Final = MARKER.search(request.body)
+    if found is None:
+        return Reply(status=404, body=b'{"error":"no marker"}')
+    marker: Final = found.group(0).decode()
+    stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True
+    if request.target.endswith("/responses"):
+        return _responses_reply(f"resp_{marker}", stream)
+    return _chat_reply(f"chatcmpl-{marker}", stream)
+
+
+def _at(payload: JsonValue, *path: str | int) -> JsonValue:
+    if not path:
+        return payload
+    step: Final = path[0]
+    if isinstance(step, int):
+        assert isinstance(payload, list), payload
+        return _at(payload[step], *path[1:])
+    return _at(object_value(payload)[step], *path[1:])
+
+
+def _sse(body: str) -> tuple[JsonValue, ...]:
+    return tuple(
+        JSON.validate_json(line[6:])
+        for line in body.splitlines()
+        if line.startswith("data: ") and line != "data: [DONE]"
+    )
+
+
+def _raw_text(endpoint: Endpoint, stream: bool, body: str) -> str:
+    if not stream:
+        path: Final[tuple[str | int, ...]] = {
+            "chat": ("choices", 0, "message", "content"),
+            "responses": ("output", 0, "content", 0, "text"),
+            "messages": ("content", 0, "text"),
+        }[endpoint]
+        return str(_at(JSON.validate_json(body), *path))
+    events: Final = _sse(body)
+    if endpoint == "chat":
+        return "".join(
+            str(object_value(_at(event, "choices", 0, "delta")).get("content") or "")
+            for event in events
+            if _at(event, "choices")
+        )
+    if endpoint == "responses":
+        return "".join(
+            str(_at(event, "delta")) for event in events if _at(event, "type") == "response.output_text.delta"
+        )
+    return "".join(
+        str(_at(event, "delta", "text"))
+        for event in events
+        if _at(event, "type") == "content_block_delta" and _at(event, "delta", "type") == "text_delta"
+    )
+
+
+def _body(model: str, endpoint: Endpoint, marker: str, stream: bool) -> tuple[str, dict[str, JsonValue]]:
+    if endpoint == "chat":
+        return "/v1/chat/completions", {
+            "model": model,
+            "messages": [{"role": "user", "content": marker}],
+            "stream": stream,
+        }
+    if endpoint == "responses":
+        return "/v1/responses", {"model": model, "input": marker, "stream": stream}
+    return "/v1/messages", {
+        "model": model,
+        "max_tokens": 16,
+        "messages": [{"role": "user", "content": marker}],
+        "stream": stream,
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class Sent:
+    call_id: str
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class Cursors:
+    operator: int
+    tenant: int
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    owned: OwnedProxy
+    scenario: Scenario
+    model: str
+    key: str
+    upstream: Wire
+    sinks: SpanSinks
+
+    def cursors(self) -> Cursors:
+        self.upstream.drain()
+        return Cursors(recorded_spans(self.sinks.operator)[0], recorded_spans(self.sinks.tenant)[0])
+
+    def upstream_hits(self, marker: str) -> int:
+        return sum(1 for request in self.upstream.drain() if marker.encode() in request.body)
+
+    def base_url(self) -> str:
+        return str(self.proxy.client.base_url)
+
+    def raw(
+        self, endpoint: Endpoint, marker: str, stream: bool, key: str | None = None, trace_id: str | None = None
+    ) -> Sent:
+        path, body = _body(self.model, endpoint, marker, stream)
+        auth: Final = {"Authorization": f"Bearer {key or self.key}"}
+        parent: Final = {} if trace_id is None else {"traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"}
+        with self.proxy.client.stream("POST", path, json=body, headers={**auth, **parent}) as response:
+            text: Final = response.read().decode()
+            assert response.status_code == 200, text
+            return Sent(response.headers["x-litellm-call-id"], _raw_text(endpoint, stream, text))
+
+    def sdk(self, endpoint: Endpoint, marker: str, stream: bool) -> Sent:
+        if endpoint == "messages":
+            messages: Final = anthropic.Anthropic(base_url=self.base_url(), api_key=self.key, max_retries=0).messages
+            if not stream:
+                reply: Final = messages.with_raw_response.create(
+                    model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}]
+                )
+                block: Final = reply.parse().content[0]
+                assert isinstance(block, anthropic.types.TextBlock), block
+                return Sent(reply.headers["x-litellm-call-id"], block.text)
+            with messages.with_streaming_response.create(
+                model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}], stream=True
+            ) as streamed:
+                return Sent(
+                    streamed.headers["x-litellm-call-id"],
+                    "".join(
+                        event.delta.text
+                        for event in streamed.parse()
+                        if event.type == "content_block_delta" and event.delta.type == "text_delta"
+                    ),
+                )
+        client: Final = openai.OpenAI(base_url=self.base_url() + "/v1", api_key=self.key, max_retries=0)
+        if endpoint == "chat":
+            if not stream:
+                completion: Final = client.chat.completions.with_raw_response.create(
+                    model=self.model, messages=[{"role": "user", "content": marker}]
+                )
+                return Sent(
+                    completion.headers["x-litellm-call-id"], completion.parse().choices[0].message.content or ""
+                )
+            with client.chat.completions.with_streaming_response.create(
+                model=self.model, messages=[{"role": "user", "content": marker}], stream=True
+            ) as chunks:
+                return Sent(
+                    chunks.headers["x-litellm-call-id"],
+                    "".join(chunk.choices[0].delta.content or "" for chunk in chunks.parse() if chunk.choices),
+                )
+        if not stream:
+            created: Final = client.responses.with_raw_response.create(model=self.model, input=marker)
+            return Sent(created.headers["x-litellm-call-id"], created.parse().output_text)
+        with client.responses.with_streaming_response.create(model=self.model, input=marker, stream=True) as events:
+            return Sent(
+                events.headers["x-litellm-call-id"],
+                "".join(event.delta for event in events.parse() if event.type == "response.output_text.delta"),
+            )
+
+    async def async_sdk(self, endpoint: Endpoint, marker: str, stream: bool) -> Sent:
+        if endpoint == "messages":
+            messages: Final = anthropic.AsyncAnthropic(
+                base_url=self.base_url(), api_key=self.key, max_retries=0
+            ).messages
+            if not stream:
+                reply: Final = await messages.with_raw_response.create(
+                    model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}]
+                )
+                block: Final = reply.parse().content[0]
+                assert isinstance(block, anthropic.types.TextBlock), block
+                return Sent(reply.headers["x-litellm-call-id"], block.text)
+            async with messages.with_streaming_response.create(
+                model=self.model, max_tokens=16, messages=[{"role": "user", "content": marker}], stream=True
+            ) as streamed:
+                pieces: Final = [
+                    event.delta.text
+                    async for event in await streamed.parse()
+                    if event.type == "content_block_delta" and event.delta.type == "text_delta"
+                ]
+                return Sent(streamed.headers["x-litellm-call-id"], "".join(pieces))
+        client: Final = openai.AsyncOpenAI(base_url=self.base_url() + "/v1", api_key=self.key, max_retries=0)
+        if endpoint == "chat":
+            if not stream:
+                completion: Final = await client.chat.completions.with_raw_response.create(
+                    model=self.model, messages=[{"role": "user", "content": marker}]
+                )
+                return Sent(
+                    completion.headers["x-litellm-call-id"], completion.parse().choices[0].message.content or ""
+                )
+            async with client.chat.completions.with_streaming_response.create(
+                model=self.model, messages=[{"role": "user", "content": marker}], stream=True
+            ) as chunks:
+                deltas: Final = [
+                    chunk.choices[0].delta.content or "" async for chunk in await chunks.parse() if chunk.choices
+                ]
+                return Sent(chunks.headers["x-litellm-call-id"], "".join(deltas))
+        if not stream:
+            created: Final = await client.responses.with_raw_response.create(model=self.model, input=marker)
+            return Sent(created.headers["x-litellm-call-id"], created.parse().output_text)
+        async with client.responses.with_streaming_response.create(
+            model=self.model, input=marker, stream=True
+        ) as events:
+            texts: Final = [
+                event.delta async for event in await events.parse() if event.type == "response.output_text.delta"
+            ]
+            return Sent(events.headers["x-litellm-call-id"], "".join(texts))
+
+    def send(self, endpoint: Endpoint, client: Client, marker: str, stream: bool) -> Sent:
+        if client == "raw":
+            return self.raw(endpoint, marker, stream)
+        if client == "sdk":
+            return self.sdk(endpoint, marker, stream)
+        return asyncio.run(self.async_sdk(endpoint, marker, stream))
+
+
+def _db_systems(spans: tuple[Span, ...]) -> set[str]:
+    return {
+        str(system)
+        for span in spans
+        if (system := span["attributes"].get("db.system.name") or span["attributes"].get("db.system")) is not None
+    }
+
+
+def _names(spans: tuple[Span, ...]) -> list[str]:
+    return sorted(span["name"] for span in spans)
+
+
+def _has_root(spans: tuple[Span, ...]) -> bool:
+    return any(span["kind"] == SERVER for span in spans)
+
+
+def _trace_of_call(sink: str, call_id: str, since: int) -> tuple[Span, ...]:
+    _, spans = recorded_spans(sink, since)
+    traces: Final = {span["trace_id"] for span in spans if span["attributes"].get("litellm.call_id") == call_id}
+    return tuple(span for span in spans if span["trace_id"] in traces)
+
+
+def _operator_trace(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
+    trace: Final = eventually(
+        lambda: _trace_of_call(rig.sinks.operator, sent.call_id, cursors.operator),
+        lambda spans: _has_root(spans) and "redis" in _db_systems(spans),
+        seconds=40,
+    )
+    assert len({span["trace_id"] for span in trace}) == 1, _names(trace)
+    return trace
+
+
+def _traced_raw(rig: Rig, endpoint: Endpoint, marker: str) -> tuple[str, Sent]:
+    trace_id: Final = uuid.uuid4().hex
+    return trace_id, rig.raw(endpoint, marker, stream=False, trace_id=trace_id)
+
+
+def _operator_trace_by_id(rig: Rig, trace_id: str, cursors: Cursors) -> tuple[Span, ...]:
+    return eventually(
+        lambda: spans_for_trace(recorded_spans(rig.sinks.operator, cursors.operator)[1], trace_id),
+        _has_root,
+        seconds=40,
+    )
+
+
+def _tenant_mirror(rig: Rig, operator: tuple[Span, ...], cursors: Cursors) -> tuple[Span, ...]:
+    kept: Final = frozenset(span["name"] for span in operator if not _db_systems((span,)))
+    return eventually(
+        lambda: spans_for_trace(recorded_spans(rig.sinks.tenant, cursors.tenant)[1], operator[0]["trace_id"]),
+        lambda spans: kept <= {span["name"] for span in spans},
+        seconds=40,
+    )
+
+
+def _assert_tenant_mirrors(rig: Rig, operator: tuple[Span, ...], cursors: Cursors) -> tuple[Span, ...]:
+    tenant: Final = _tenant_mirror(rig, operator, cursors)
+    assert _db_systems(tenant) == set(), f"datastore spans reached the tenant: {_names(tenant)}"
+    assert sum(1 for span in tenant if span["kind"] == SERVER) == 1, _names(tenant)
+    return tenant
+
+
+def _assert_withheld(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
+    tenant: Final = _assert_tenant_mirrors(rig, _operator_trace(rig, sent, cursors), cursors)
+    assert any("gen_ai.operation.name" in span["attributes"] for span in tenant), _names(tenant)
+    return tenant
+
+
+def _config(directory: Path, otel_audit_config: AuditConfigWriter, otel: Mapping[str, JsonValue], name: str) -> Path:
+    written: Final = otel_audit_config(directory, {})
+    loaded: Final = object_value(JSON.validate_python(yaml.safe_load(written.read_text())))
+    settings: Final = object_value(loaded["callback_settings"])
+    config: Final = {**loaded, "callback_settings": {**settings, "otel": {**object_value(settings["otel"]), **otel}}}
+    path: Final = directory / f"{name}.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@contextmanager
+def _started(
+    provider: Wire,
+    sinks: SpanSinks,
+    config: Path,
+    directory: Path,
+    langfuse_vars: Mapping[str, JsonValue],
+    workers: int,
+) -> Generator[Rig]:
+    with (
+        gateway_from_environment() as gateway,
+        owned_proxy_process(
+            gateway,
+            directory,
+            {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"},
+            config=config,
+            remove_environment=("LITELLM_OTEL_EXCLUDED_SERVICES",),
+            workers=workers,
+        ) as owned,
+        owned.gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(api_base=provider.url + "/v1")
+        team: Final = scenario.team()
+        attached: Final = owned.gateway.request(
+            "POST", f"/team/{team}/callback", {"callback_name": "langfuse_otel", "callback_vars": dict(langfuse_vars)}
+        )
+        assert attached.status_code == 200, attached.text
+        yield Rig(owned.gateway, owned, scenario, model, scenario.key(team_id=team), provider, sinks)
+
+
+@pytest.fixture(scope="module")
+def provider() -> Iterator[Wire]:
+    with wire_server(_upstream) as wire:
+        yield wire
+
+
+@pytest.fixture(scope="module")
+def rig(
+    provider: Wire,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path_factory: pytest.TempPathFactory,
+) -> Iterator[Rig]:
+    directory: Final = tmp_path_factory.mktemp("excluded-matrix")
+    config: Final = _config(directory, otel_audit_config, {"excluded_services": ["redis", "postgres"]}, "matrix")
+    with _started(provider, audit_sinks, config, directory, langfuse_vars, workers=2) as started:
+        yield started
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("stream", [False, True], ids=["unary", "stream"])
+@pytest.mark.parametrize("client", CLIENTS)
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_tenant_trace_keeps_request_spans_without_datastore_spans(
+    rig: Rig, endpoint: Endpoint, client: Client, stream: bool
+) -> None:
+    cursors: Final = rig.cursors()
+    marker: Final = _marker()
+    sent: Final = rig.send(endpoint, client, marker, stream)
+    assert sent.text == REPLY_TEXT, sent
+    assert rig.upstream_hits(marker) == 1
+    _assert_withheld(rig, sent, cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("endpoint", ["chat", "messages"])
+def test_cache_hit_twin_keeps_datastore_spans_off_the_tenant(rig: Rig, endpoint: Endpoint) -> None:
+    marker: Final = _marker()
+    first: Final = rig.raw(endpoint, marker, stream=False)
+    assert first.text == REPLY_TEXT, first
+    assert rig.upstream_hits(marker) == 1
+    cursors: Final = rig.cursors()
+    trace_id, hit = eventually(
+        lambda: _traced_raw(rig, endpoint, marker), lambda sent: rig.upstream_hits(marker) == 0, seconds=20
+    )
+    assert hit.text == REPLY_TEXT, hit
+    _assert_tenant_mirrors(rig, _operator_trace_by_id(rig, trace_id, cursors), cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("endpoint", ENDPOINTS)
+def test_failed_upstream_call_keeps_datastore_spans_off_the_tenant(rig: Rig, endpoint: Endpoint) -> None:
+    cursors: Final = rig.cursors()
+    marker: Final = "excl-fail-" + uuid.uuid4().hex
+    trace_id: Final = uuid.uuid4().hex
+    path, body = _body(rig.model, endpoint, marker, stream=False)
+    failed: Final = rig.proxy.client.post(
+        path,
+        json=body,
+        headers={"Authorization": f"Bearer {rig.key}", "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01"},
+    )
+    assert failed.status_code == 500, failed.text
+    assert rig.upstream_hits(marker) >= 1
+    operator: Final = eventually(
+        lambda: spans_for_trace(recorded_spans(rig.sinks.operator, cursors.operator)[1], trace_id),
+        lambda spans: _has_root(spans) and "redis" in _db_systems(spans),
+        seconds=40,
+    )
+    _assert_tenant_mirrors(rig, operator, cursors)
+
+
+@pytest.mark.timeout(120)
+def test_key_level_callback_vars_destination_is_filtered_too(rig: Rig, langfuse_vars: dict[str, JsonValue]) -> None:
+    key: Final = rig.scenario.key(
+        metadata={
+            "logging": [
+                {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": dict(langfuse_vars)}
+            ]
+        }
+    )
+    cursors: Final = rig.cursors()
+    marker: Final = _marker()
+    sent: Final = rig.raw("chat", marker, stream=False, key=key)
+    assert sent.text == REPLY_TEXT, sent
+    assert rig.upstream_hits(marker) == 1
+    _assert_withheld(rig, sent, cursors)
+
+
+@pytest.mark.timeout(120)
+@pytest.mark.parametrize("status", [403, 404])
+def test_rejecting_tenant_destination_leaves_serving_and_the_operator_trace_intact(rig: Rig, status: int) -> None:
+    configure_sink(rig.sinks.tenant, status=status)
+    try:
+        cursors: Final = rig.cursors()
+        marker: Final = _marker()
+        sent: Final = rig.raw("chat", marker, stream=True)
+        assert sent.text == REPLY_TEXT, sent
+        assert rig.upstream_hits(marker) == 1
+        _assert_withheld(rig, sent, cursors)
+    finally:
+        configure_sink(rig.sinks.tenant, status=200)
+    after: Final = rig.cursors()
+    _assert_withheld(rig, rig.raw("responses", _marker(), stream=False), after)
+
+
+def _burst(rig: Rig, count: int) -> tuple[Sent | str, ...]:
+    def one(index: int) -> Sent | str:
+        try:
+            return rig.raw(ENDPOINTS[index % 3], _marker(), stream=index % 2 == 0)
+        except (httpx.HTTPError, AssertionError) as error:
+            return repr(error)
+
+    with ThreadPoolExecutor(max_workers=10) as pool:
+        return tuple(pool.map(one, range(count)))
+
+
+def _served(results: tuple[Sent | str, ...]) -> tuple[Sent, ...]:
+    return tuple(result for result in results if isinstance(result, Sent))
+
+
+def _assert_operator_exactly_once(rig: Rig, served: tuple[Sent, ...], cursors: Cursors) -> set[str]:
+    wanted: Final = {sent.call_id for sent in served}
+
+    def roots() -> dict[str, int]:
+        _, spans = recorded_spans(rig.sinks.operator, cursors.operator)
+        traced: Final = {
+            span["trace_id"]: str(span["attributes"]["litellm.call_id"])
+            for span in spans
+            if span["attributes"].get("litellm.call_id") in wanted
+        }
+        counts: Final = {call: 0 for call in wanted}
+        for span in spans:
+            if span["kind"] == SERVER and span["trace_id"] in traced:
+                counts[traced[span["trace_id"]]] += 1
+        return counts
+
+    landed: Final = eventually(roots, lambda counts: all(count >= 1 for count in counts.values()), seconds=90)
+    assert landed == {call: 1 for call in wanted}, landed
+    _, spans = recorded_spans(rig.sinks.operator, cursors.operator)
+    return {span["trace_id"] for span in spans if span["attributes"].get("litellm.call_id") in wanted}
+
+
+def _assert_tenant_never_saw_datastore_spans(rig: Rig, cursors: Cursors, traces: set[str]) -> None:
+    tenant: Final = eventually(
+        lambda: recorded_spans(rig.sinks.tenant, cursors.tenant)[1],
+        lambda spans: traces <= {span["trace_id"] for span in spans if span["kind"] == SERVER},
+        seconds=90,
+    )
+    assert _db_systems(tenant) == set(), _names(tenant)
+
+
+@pytest.mark.timeout(300)
+def test_tenant_outage_during_a_mixed_burst_keeps_serving_and_never_leaks_datastore_spans(rig: Rig) -> None:
+    cursors: Final = rig.cursors()
+    configure_sink(rig.sinks.tenant, status=503)
+    try:
+        results: Final = _burst(rig, 30)
+    finally:
+        configure_sink(rig.sinks.tenant, status=200)
+    served: Final = _served(results)
+    assert len(served) == 30, [result for result in results if isinstance(result, str)]
+    assert all(sent.text == REPLY_TEXT for sent in served), served
+    traces: Final = _assert_operator_exactly_once(rig, served, cursors)
+    _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+    after: Final = rig.cursors()
+    _assert_withheld(rig, rig.raw("messages", _marker(), stream=True), after)
+
+
+@pytest.mark.timeout(300)
+def test_stalled_tenant_destination_during_a_burst_does_not_block_responses(rig: Rig) -> None:
+    cursors: Final = rig.cursors()
+    configure_sink(rig.sinks.tenant, paused=True)
+    try:
+        results: Final = _burst(rig, 20)
+    finally:
+        configure_sink(rig.sinks.tenant, paused=False)
+    served: Final = _served(results)
+    assert len(served) == 20, [result for result in results if isinstance(result, str)]
+    traces: Final = _assert_operator_exactly_once(rig, served, cursors)
+    _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+
+
+@pytest.mark.timeout(300)
+def test_killing_one_of_two_workers_mid_burst_keeps_the_filter_on_the_survivor(rig: Rig) -> None:
+    root: Final = psutil.Process(rig.owned.process.pid)
+    workers: Final = eventually(
+        lambda: tuple(child for child in root.children() if "resource_tracker" not in " ".join(child.cmdline())),
+        lambda found: len(found) == 2,
+        seconds=30,
+    )
+    cursors: Final = rig.cursors()
+
+    def one(index: int) -> Sent | str:
+        if index == 6:
+            os.kill(workers[0].pid, signal.SIGKILL)
+        try:
+            return rig.raw("chat", _marker(), stream=index % 2 == 0)
+        except (httpx.HTTPError, AssertionError) as error:
+            return repr(error)
+
+    with ThreadPoolExecutor(max_workers=6) as pool:
+        results: Final = tuple(pool.map(one, range(18)))
+    assert rig.owned.process.poll() is None, "Proxy root exited after a worker was killed"
+    failures: Final = tuple(result for result in results if isinstance(result, str))
+    assert all(failure.startswith(("ReadError(", "RemoteProtocolError(", "ConnectError(")) for failure in failures), (
+        failures
+    )
+    assert len(failures) <= 6, failures
+    settled: Final = tuple(result for index, result in enumerate(results) if index > 12 and isinstance(result, Sent))
+    traces: Final = _assert_operator_exactly_once(rig, settled, cursors)
+    _assert_tenant_never_saw_datastore_spans(rig, cursors, traces)
+    after: Final = rig.cursors()
+    _assert_withheld(rig, rig.raw("chat", _marker(), stream=False), after)
+
+
+@dataclass(frozen=True, slots=True)
+class Setting:
+    otel: Mapping[str, JsonValue]
+    withholds_redis: bool
+    logs: str | None
+
+
+SETTINGS: Final[dict[str, Setting]] = {
+    "missing": Setting({}, False, None),
+    "null": Setting({"excluded_services": None}, False, None),
+    "empty_list": Setting({"excluded_services": []}, False, None),
+    "empty_string": Setting({"excluded_services": ""}, False, None),
+    "yaml_string": Setting({"excluded_services": "redis"}, True, None),
+    "duplicates": Setting({"excluded_services": ["redis", "redis"]}, True, None),
+    "case_and_space": Setting({"excluded_services": ["REDIS", " Postgres "]}, True, None),
+    "integer": Setting({"excluded_services": 7}, False, INVALID_VALUE_LOG),
+    "mapping": Setting({"excluded_services": {"redis": True}}, False, INVALID_VALUE_LOG),
+    "non_string_item": Setting({"excluded_services": [7, "redis"]}, True, INVALID_VALUE_LOG),
+    "oversized_name": Setting({"excluded_services": "x" * 5000}, False, INVALID_NAME_LOG),
+}
+
+
+@pytest.mark.timeout(180)
+@pytest.mark.parametrize("name", SETTINGS)
+def test_excluded_services_setting_shapes_boot_and_resolve(
+    name: str,
+    provider: Wire,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    setting: Final = SETTINGS[name]
+    config: Final = _config(tmp_path, otel_audit_config, setting.otel, name)
+    with _started(provider, audit_sinks, config, tmp_path, langfuse_vars, workers=1) as started:
+        cursors: Final = started.cursors()
+        marker: Final = _marker()
+        sent: Final = started.raw("chat", marker, stream=False)
+        assert sent.text == REPLY_TEXT, sent
+        assert started.upstream_hits(marker) == 1
+        operator: Final = _operator_trace(started, sent, cursors)
+        tenant: Final = _tenant_mirror(started, operator, cursors)
+        if setting.withholds_redis:
+            assert "redis" not in _db_systems(tenant), _names(tenant)
+        else:
+            eventually(
+                lambda: _db_systems(
+                    spans_for_trace(recorded_spans(started.sinks.tenant, cursors.tenant)[1], tenant[0]["trace_id"])
+                ),
+                lambda systems: "redis" in systems,
+                seconds=30,
+            )
+        log: Final = started.owned.log.read_text()
+        if setting.logs is None:
+            assert INVALID_NAME_LOG not in log and INVALID_VALUE_LOG not in log, log[-2000:]
+        else:
+            assert setting.logs in log, log[-4000:]

tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py

@@ -11,6 +11,7 @@
 """
 
 import asyncio
+import logging
 
 import pytest
 
@@ -22,17 +23,18 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (  # noqa: E4
 )
 
 from litellm.integrations.otel import LiteLLM, OpenTelemetryV2Config  # noqa: E402
-from litellm.integrations.otel.plumbing import providers  # noqa: E402
+from litellm.integrations.otel.logger import OpenTelemetryV2  # noqa: E402
 from litellm.integrations.otel.model.baggage import (  # noqa: E402
     BAGGAGE_PROMOTED_KEYS,
     DEFAULT_BAGGAGE_METADATA_KEYS,
 )
-from litellm.integrations.otel.logger import OpenTelemetryV2  # noqa: E402
+from litellm.integrations.otel.model.config import excluded_db_systems_from  # noqa: E402
 from litellm.integrations.otel.model.payloads import GuardrailSpanData  # noqa: E402
 from litellm.integrations.otel.model.spans import (  # noqa: E402
     LITELLM_PROXY_REQUEST_SPAN_NAME,
     SpanRole,
 )
+from litellm.integrations.otel.plumbing import providers  # noqa: E402
 
 # --------------------------------------------------------------------------- #
 #  Area 1 — baggage allowlists configurable
@@ -74,13 +76,11 @@ def test_baggage_keys_from_config_yaml_kwargs():
 
 
 def test_baggage_processor_allowlist_uses_config_keys():
-    cfg = OpenTelemetryV2Config(
-        exporter="in_memory", baggage_promoted_keys=[LiteLLM.TEAM_ID]
-    )
+    cfg = OpenTelemetryV2Config(exporter="in_memory", baggage_promoted_keys=[LiteLLM.TEAM_ID])
     provider, exporter = providers.in_memory_provider(cfg)
-    from litellm.integrations.otel.plumbing import context as ctx_mod
     from litellm.integrations.otel.emitter import SpanEmitter
     from litellm.integrations.otel.model.payloads import ServiceSpanData
+    from litellm.integrations.otel.plumbing import context as ctx_mod
 
     engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
     ctx = ctx_mod.set_request_baggage({LiteLLM.TEAM_ID: "t1", LiteLLM.TEAM_ALIAS: "ta"})
@@ -90,6 +90,68 @@ def test_baggage_processor_allowlist_uses_config_keys():
     assert LiteLLM.TEAM_ALIAS not in span.attributes  # not in this allowlist
 
 
+@pytest.mark.parametrize(
+    "given,expected",
+    [
+        (["redis"], frozenset({"redis"})),
+        (["postgres"], frozenset({"postgresql"})),
+        (["postgresql"], frozenset({"postgresql"})),
+        (["batch_write_to_db"], frozenset({"postgresql"})),
+        (["redis_spend_update_queue"], frozenset({"redis"})),
+        (["redis", "postgres"], frozenset({"redis", "postgresql"})),
+    ],
+)
+def test_excluded_services_normalize_to_db_system_names(given, expected):
+    assert OpenTelemetryV2Config(excluded_services=given).excluded_services == expected
+
+
+def test_excluded_services_from_env_csv(monkeypatch):
+    monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis, postgres")
+    assert OpenTelemetryV2Config().excluded_services == frozenset({"redis", "postgresql"})
+
+
+def test_excluded_services_config_wins_over_env(monkeypatch):
+    monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis")
+    assert OpenTelemetryV2Config(excluded_services=["postgres"]).excluded_services == frozenset({"postgresql"})
+
+
+def test_excluded_services_drops_a_non_datastore_service_and_logs(caplog):
+    with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+        config = OpenTelemetryV2Config(excluded_services=["auth", "redis"])
+    assert config.excluded_services == frozenset({"redis"})
+    assert any("'auth' is not a datastore service; ignored" in record.message for record in caplog.records)
+
+
+def test_excluded_services_env_drops_a_bad_value_and_logs(monkeypatch, caplog):
+    monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "auth,postgres")
+    with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+        config = OpenTelemetryV2Config()
+    assert config.excluded_services == frozenset({"postgresql"})
+    assert any("'auth' is not a datastore service; ignored" in record.message for record in caplog.records)
+
+
+@pytest.mark.parametrize(
+    "given,expected,logged",
+    [
+        (None, frozenset(), None),
+        ("", frozenset(), None),
+        ([], frozenset(), None),
+        (["REDIS", " Postgres "], frozenset({"redis", "postgresql"}), None),
+        (7, frozenset(), "excluded_services must be a list or comma-separated string; 7 ignored"),
+        ({"redis": True}, frozenset(), "excluded_services must be a list or comma-separated string"),
+        ([7, "redis"], frozenset({"redis"}), "excluded_services must be a list of service names; 7 ignored"),
+    ],
+)
+def test_malformed_excluded_services_logs_and_still_builds_the_config(given, expected, logged, caplog):
+    with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+        config = OpenTelemetryV2Config(excluded_services=given)
+        resolved = excluded_db_systems_from(given)
+    assert config.excluded_services == expected
+    assert resolved == expected
+    messages = [record.message for record in caplog.records]
+    assert (logged is None and messages == []) or any(logged in message for message in messages), messages
+
+
 # --------------------------------------------------------------------------- #
 #  Area 2 — pass-through LLM span parents to the ambient server span
 # --------------------------------------------------------------------------- #
@@ -124,9 +186,7 @@ def test_passthrough_llm_span_parents_to_ambient_server_span():
     later (possibly detached) success callback only closes the already-parented
     span, so it never becomes a separate root trace."""
     logger, exporter = _logger()
-    server = logger._emitter.start_span(
-        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
-    )
+    server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
     kwargs = {
         "standard_logging_object": _payload(),
         "litellm_params": {"metadata": {}},
@@ -150,9 +210,7 @@ def test_llm_span_unaffected_by_phase_span_active_at_close():
     successor to the old auth-failure-401 case where the LLM log nested under
     ``auth``: the span is now born after auth, parented to the request root."""
     logger, exporter = _logger()
-    server = logger._emitter.start_span(
-        SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
-    )
+    server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
     kwargs = {
         "standard_logging_object": _payload(),
         "litellm_params": {"metadata": {}},

tests/unit/integrations/otel/test_otel_v2_destinations.py

@@ -514,6 +514,53 @@ class TestFanOut:
         for child in ("auth /v1/chat/completions", "chat gpt-4"):
             assert by_name[child].parent.span_id == root.context.span_id
 
+    def test_excluded_services_drop_only_the_datastore_spans_at_the_tenant(self):
+        """The exclusion is per ``db.system.*`` value: a span naming an excluded
+        datastore never reaches the tenant, while every span of the request's
+        own work (root, auth, guardrail, model) still does, and the operator's
+        own exporter keeps the full tree."""
+        dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter()
+        provider = TracerProvider()
+        provider.add_span_processor(SimpleSpanProcessor(operator_exporter))
+        provider.add_span_processor(
+            TenantFanOutSpanProcessor(
+                processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter),
+                excluded_db_systems=frozenset({"redis", "postgresql"}),
+            )
+        )
+        tracer = get_tracer(provider, "litellm")
+
+        def run():
+            set_request_destinations((LANGFUSE_DEST,))
+            with tracer.start_as_current_span("POST /v1/chat/completions"):
+                with tracer.start_as_current_span("auth /v1/chat/completions"):
+                    pass
+                with tracer.start_as_current_span("execute_guardrail pii"):
+                    pass
+                with tracer.start_as_current_span("redis async_get_cache") as redis_span:
+                    redis_span.set_attribute("db.system.name", "redis")
+                with tracer.start_as_current_span("batch_write_to_db _PROXY_track_cost_callback") as spend_span:
+                    spend_span.set_attribute("db.system", "postgresql")
+                with tracer.start_as_current_span("chat gpt-4"):
+                    pass
+
+        in_fresh_context(run)
+
+        assert {s.name for s in dest_exporter.get_finished_spans()} == {
+            "POST /v1/chat/completions",
+            "auth /v1/chat/completions",
+            "execute_guardrail pii",
+            "chat gpt-4",
+        }
+        assert {s.name for s in operator_exporter.get_finished_spans()} == {
+            "POST /v1/chat/completions",
+            "auth /v1/chat/completions",
+            "execute_guardrail pii",
+            "redis async_get_cache",
+            "batch_write_to_db _PROXY_track_cost_callback",
+            "chat gpt-4",
+        }
+
     def test_a_team_naming_two_backends_gets_the_trace_at_both(self):
         """The fan-out rides one provider, so it cannot skip a destination on the
         grounds that some other backend owns it: nothing else would deliver it."""
@@ -1022,6 +1069,89 @@ class TestProviderWiring:
         assert kinds(published).count("TenantFanOutSpanProcessor") == 1
         assert "TenantFanOutSpanProcessor" not in kinds(other)
 
+    @staticmethod
+    def _fan_out_of(logger: OpenTelemetryV2) -> TenantFanOutSpanProcessor:
+        return next(
+            processor
+            for processor in logger._tracer_provider._active_span_processor._span_processors
+            if isinstance(processor, TenantFanOutSpanProcessor)
+        )
+
+    def test_callback_settings_excluded_services_win_over_the_published_preset_env_config(self, monkeypatch):
+        """A preset builds its config env-only, so the fan-out must read
+        ``callback_settings.otel.excluded_services`` itself rather than the
+        published logger's config, or the env value would win."""
+        monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+        preset = OpenTelemetryV2(
+            config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")], excluded_services=["redis"]),
+            callback_name="langfuse_otel",
+        )
+
+        publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+        assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+
+    def test_callback_settings_excluded_services_apply_even_when_other_otel_env_vars_are_malformed(self, monkeypatch):
+        """Reading the setting must not rebuild the whole settings model, or an unrelated bad env
+        value the operator overrode in config would stop publication before the fan-out is attached"""
+        preset = OpenTelemetryV2(
+            config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")]),
+            callback_name="langfuse_otel",
+        )
+        monkeypatch.setenv("LITELLM_OTEL_LEGACY_COMPAT", "not-a-bool")
+        monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+
+        publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+        assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+
+    def test_excluded_services_fall_back_to_the_published_logger_config_without_callback_settings(self, monkeypatch):
+        monkeypatch.setattr(litellm, "callback_settings", {"otel": {"exporter": "in_memory"}}, raising=False)
+        preset = OpenTelemetryV2(
+            config=OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")], excluded_services=["redis"]),
+            callback_name="langfuse_otel",
+        )
+
+        publish_global_otel_v2_provider([], lambda _p: None, registered=preset)
+
+        assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"redis"})
+
+    def test_otel_after_a_preset_reuses_it_and_still_takes_callback_settings_exclusions(self, monkeypatch):
+        """``callbacks: [langfuse_otel, otel]`` keeps one v2 logger, exactly as
+        before ``excluded_services`` existed, and the exclusion still comes from
+        ``callback_settings.otel`` rather than the preset's env-only config."""
+        from litellm.litellm_core_utils import litellm_logging as logging_module
+
+        logging_module._in_memory_loggers.clear()
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk")
+        monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk")
+        monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis")
+        is_otel_v2_enabled.cache_clear()
+        monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+        try:
+
+            def init(name: str) -> CustomLogger | None:
+                return logging_module._init_custom_logger_compatible_class(
+                    logging_integration=name,  # pyright: ignore[reportArgumentType]  # test passes a literal callback name
+                    internal_usage_cache=None,
+                    llm_router=None,
+                    custom_logger_init_args={},
+                )
+
+            preset = init("langfuse_otel")
+            otel_cb = init("otel")
+
+            assert isinstance(preset, OpenTelemetryV2)
+            assert otel_cb is preset
+            v2_loggers = [cb for cb in logging_module._in_memory_loggers if isinstance(cb, OpenTelemetryV2)]
+            assert v2_loggers == [preset], v2_loggers
+            publish_global_otel_v2_provider(logging_module._in_memory_loggers, lambda _p: None, registered=preset)
+            assert self._fan_out_of(preset)._excluded_db_systems == frozenset({"postgresql"})
+        finally:
+            logging_module._in_memory_loggers.clear()
+            is_otel_v2_enabled.cache_clear()
+
     @pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"])
     def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical):
         monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive")

ui/litellm-dashboard/public/assets/logos/crewai-color.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>CrewAI</title><path d="M19.41 10.783a2.753 2.753 0 012.471 1.355c.483.806.622 1.772.385 2.68l-.136.522a9.994 9.994 0 01-3.156 5.058c-.605.517-1.283 1.062-2.083 1.524l-.028.017c-.402.232-.884.511-1.398.756-1.19.602-2.475.997-3.798 1.167-.854.111-1.716.155-2.577.132H9.072a8.588 8.588 0 01-5.046-1.87l-.012-.01-.012-.01A8.024 8.024 0 011.22 17.42a10.916 10.916 0 01-.102-3.779A15.622 15.622 0 012.88 8.4a21.758 21.758 0 012.432-3.678 15.44 15.44 0 013.56-3.182A9.958 9.958 0 0112.44.104h.004l.003-.002c2.057-.384 3.743.374 5.024 1.26a8.28 8.28 0 012.395 2.513l.024.04.023.042a5.474 5.474 0 01.508 4.012c-.239.97-.577 1.914-1.01 2.814z" fill="#461816"></path><path d="M18.861 13.165a.748.748 0 011.256.031c.199.332.256.73.159 1.103l-.137.522a7.936 7.936 0 01-2.504 4.014c-.572.49-1.138.939-1.774 1.306-.427.247-.857.496-1.303.707a9.628 9.628 0 01-3.155.973 14.33 14.33 0 01-2.257.116 6.531 6.531 0 01-3.837-1.422 5.967 5.967 0 01-2.071-3.494 8.859 8.859 0 01-.085-3.08 13.56 13.56 0 011.54-4.568 19.701 19.701 0 012.212-3.348 13.382 13.382 0 013.088-2.76 7.9 7.9 0 012.832-1.14c1.307-.245 2.434.207 3.481.933a6.222 6.222 0 011.806 1.892c.423.767.536 1.668.314 2.515a12.394 12.394 0 01-.99 2.67l-.223.497c-.321.713-.642 1.426-.97 2.137a.762.762 0 01-.97.467 3.39 3.39 0 01-2.283-2.49c-.095-.83.04-1.669.39-2.426.288-.746.61-1.477.933-2.208l.248-.563a.53.53 0 00-.204-.742 2.35 2.35 0 00-1.2.702 25.291 25.291 0 00-1.614 1.767 21.561 21.561 0 00-2.619 4.184 7.59 7.59 0 00-.816 2.753 7.042 7.042 0 00.07 2.219 2.055 2.055 0 001.934 1.715c1.801.1 3.59-.363 5.116-1.328.582-.4 1.141-.831 1.675-1.294.752-.71 1.376-1.519 1.958-2.36z" fill="#fff"></path></svg>

ui/litellm-dashboard/public/assets/logos/langchain.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>LangChain</title><path d="M7.531 15.976a7.534 7.534 0 000-10.651L2.206 0A7.537 7.537 0 000 5.326c0 1.996.794 3.913 2.206 5.325l5.325 5.325zM18.674 16.469a7.535 7.535 0 00-10.65 0l5.325 5.325a7.536 7.536 0 0010.651 0l-5.326-5.325zM2.218 21.782a7.536 7.536 0 005.326 2.206v-7.531H.012c0 1.996.795 3.914 2.206 5.325zM20.73 8.595a7.534 7.534 0 00-10.651.001l5.325 5.326 5.326-5.327z" fill="#7FC8FF"></path></svg>

ui/litellm-dashboard/public/assets/logos/langgraph-color.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>LangGraph</title><path clip-rule="evenodd" d="M6.099 6H17.9C21.264 6 24 8.692 24 12s-2.736 6-6.099 6H6.1C2.736 18 0 15.308 0 12s2.736-6 6.099-6zm5.419 9.3c.148.154.367.146.561.106l.002.001c.09-.072-.038-.163-.16-.25-.074-.052-.145-.102-.166-.147.068-.08-.133-.265-.289-.408a1.52 1.52 0 01-.15-.148c-.11-.119-.155-.268-.2-.418-.03-.1-.06-.2-.11-.292-.304-.694-.653-1.383-1.143-1.97-.315-.39-.674-.74-1.033-1.09a19.384 19.384 0 01-.683-.688c-.226-.229-.362-.511-.499-.794-.114-.236-.228-.473-.396-.68-.507-.735-2.107-.936-2.342.104 0 .032-.01.052-.039.073-.13.094-.245.2-.342.327-.238.326-.274.877.022 1.17l.001-.019c.01-.147.02-.286.139-.391.228.193.576.262.841.117.32.45.422.995.525 1.54.085.456.17.912.382 1.316l.014.022c.124.203.25.41.41.587.059.089.178.184.297.279.157.125.314.25.329.359v.143c-.001.285-.002.58.184.813.103.205-.15.41-.352.385-.112.015-.233-.014-.354-.042-.165-.04-.329-.078-.462-.003-.038.04-.091.04-.145.042-.064.002-.129.004-.167.07-.008.019-.026.04-.045.063-.042.05-.087.105-.033.146l.015-.01c.082-.062.16-.12.27-.084-.014.08.039.102.092.123l.027.012a.344.344 0 01-.008.056c-.009.045-.017.088.018.127a.598.598 0 00.046-.054c.037-.046.073-.092.139-.11.144.19.289.111.471.013.206-.111.459-.248.81-.055-.135-.006-.255.01-.345.12-.023.024-.042.052-.002.084.207-.132.294-.085.375-.04.06.032.115.063.212.024l.07-.036c.155-.083.314-.166.499-.137-.139.039-.188.125-.242.218-.026.047-.054.095-.094.14-.021.021-.03.046-.007.08.29-.023.4-.095.548-.192.07-.046.15-.099.261-.154.124-.075.248-.027.368.02.13.05.255.098.371-.014.037-.033.083-.034.129-.034.016 0 .033 0 .05-.002-.037-.19-.24-.188-.448-.186-.24.003-.483.006-.475-.289.222-.149.224-.407.226-.651 0-.06 0-.117.005-.173.163.09.336.16.508.229.162.065.323.13.474.21.158.25.404.58.732.558.008-.026.016-.047.026-.073.019.004.039.008.059.014.086.02.178.044.223-.056zm6.429-2.829c.19.186.447.29.716.29.269 0 .526-.104.716-.29a.98.98 0 00.297-.7.98.98 0 00-.297-.7 1.024 1.024 0 00-1.08-.224l-.58-.831-.405.272.583.835a.978.978 0 00.05 1.348zm-1.817-2.69a1.03 1.03 0 001.056-.095.991.991 0 00.363-.507.97.97 0 00-.016-.62.994.994 0 00-.39-.488 1.028 1.028 0 00-1.298.14.987.987 0 00-.263.856.98.98 0 00.187.42c.095.125.218.225.36.294zm0 5.752a1.032 1.032 0 001.056-.095.991.991 0 00.363-.507.97.97 0 00-.016-.62.994.994 0 00-.39-.488 1.027 1.027 0 00-1.298.14.986.986 0 00-.263.856.98.98 0 00.187.42c.095.125.218.225.36.294zm.93-3.516v-.492h-1.55a.977.977 0 00-.217-.404l.584-.847-.425-.276-.583.847a1.023 1.023 0 00-1.047.23.973.973 0 00-.296.696c0 .261.107.512.296.696a1.023 1.023 0 001.047.23l.583.847.42-.276-.579-.847a.977.977 0 00.217-.404h1.55z" fill="#1C3C3C" fill-rule="evenodd"></path></svg>

ui/litellm-dashboard/public/assets/logos/llamaindex-color.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>LlamaIndex</title><path d="M15.855 17.122c-2.092.924-4.358.545-5.23.24 0 .21-.01.857-.048 1.78-.038.924-.332 1.507-.475 1.684.016.577.029 1.837-.047 2.26a1.93 1.93 0 01-.476.914H8.295c.114-.577.555-.946.761-1.058.114-1.193-.11-2.229-.238-2.597-.126.449-.437 1.49-.665 2.068a6.418 6.418 0 01-.713 1.299h-.951c-.048-.578.27-.77.475-.77.095-.177.323-.731.476-1.54.152-.807-.064-2.324-.19-2.981v-2.068c-1.522-.818-2.092-1.636-2.473-2.55-.304-.73-.222-1.843-.142-2.308-.096-.176-.373-.625-.476-1.25-.142-.866-.063-1.491 0-1.828-.095-.096-.285-.587-.285-1.78 0-1.192.349-1.811.523-1.972v-.529c-.666-.048-1.331-.336-1.712-.721-.38-.385-.095-.962.143-1.154.238-.193.475-.049.808-.145.333-.096.618-.192.76-.48C4.512 1.403 4.287.448 4.16 0c.57.077.935.577 1.046.818V0c.713.337 1.997 1.154 2.425 2.934.342 1.424.586 4.409.665 5.723 1.823.016 4.137-.26 6.229.193 1.901.412 2.757 1.25 3.755 1.25.999 0 1.57-.577 2.282-.096.714.481 1.094 1.828.999 2.838-.076.808-.697 1.074-.998 1.106-.38 1.27 0 2.485.237 2.934v1.827c.111.16.333.655.333 1.347 0 .693-.222 1.154-.333 1.299.19 1.077-.08 2.18-.238 2.597h-1.283c.152-.385.412-.481.523-.481.228-1.193.063-2.293-.048-2.693-.722-.424-1.188-1.17-1.331-1.491.016.272-.029 1.029-.333 1.875-.304.847-.76 1.347-.95 1.491v1.01h-1.284c0-.615.348-.737.523-.721.222-.4.76-1.01.76-2.212 0-1.015-.713-1.492-1.236-2.405-.248-.434-.127-.978-.047-1.203z" fill="url(#lobe-icons-llama-index-_R_0_)"></path><defs><linearGradient gradientUnits="userSpaceOnUse" id="lobe-icons-llama-index-_R_0_" x1="4.021" x2="24.613" y1="2.02" y2="19.277"><stop offset=".062" stop-color="#F6DCD9"></stop><stop offset=".326" stop-color="#FFA5EA"></stop><stop offset=".589" stop-color="#45DFF8"></stop><stop offset="1" stop-color="#BC8DEB"></stop></linearGradient></defs></svg>

ui/litellm-dashboard/public/assets/logos/openai-agents.svg

@@ -0,0 +1 @@
+<svg fill="currentColor" fill-rule="evenodd" height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>OpenAI</title><path d="M9.205 8.658v-2.26c0-.19.072-.333.238-.428l4.543-2.616c.619-.357 1.356-.523 2.117-.523 2.854 0 4.662 2.212 4.662 4.566 0 .167 0 .357-.024.547l-4.71-2.759a.797.797 0 00-.856 0l-5.97 3.473zm10.609 8.8V12.06c0-.333-.143-.57-.429-.737l-5.97-3.473 1.95-1.118a.433.433 0 01.476 0l4.543 2.617c1.309.76 2.189 2.378 2.189 3.948 0 1.808-1.07 3.473-2.76 4.163zM7.802 12.703l-1.95-1.142c-.167-.095-.239-.238-.239-.428V5.899c0-2.545 1.95-4.472 4.591-4.472 1 0 1.927.333 2.712.928L8.23 5.067c-.285.166-.428.404-.428.737v6.898zM12 15.128l-2.795-1.57v-3.33L12 8.658l2.795 1.57v3.33L12 15.128zm1.796 7.23c-1 0-1.927-.332-2.712-.927l4.686-2.712c.285-.166.428-.404.428-.737v-6.898l1.974 1.142c.167.095.238.238.238.428v5.233c0 2.545-1.974 4.472-4.614 4.472zm-5.637-5.303l-4.544-2.617c-1.308-.761-2.188-2.378-2.188-3.948A4.482 4.482 0 014.21 6.327v5.423c0 .333.143.571.428.738l5.947 3.449-1.95 1.118a.432.432 0 01-.476 0zm-.262 3.9c-2.688 0-4.662-2.021-4.662-4.519 0-.19.024-.38.047-.57l4.686 2.71c.286.167.571.167.856 0l5.97-3.448v2.26c0 .19-.07.333-.237.428l-4.543 2.616c-.619.357-1.356.523-2.117.523zm5.899 2.83a5.947 5.947 0 005.827-4.756C22.287 18.339 24 15.84 24 13.296c0-1.665-.713-3.282-1.998-4.448.119-.5.19-.999.19-1.498 0-3.401-2.759-5.947-5.946-5.947-.642 0-1.26.095-1.88.31A5.962 5.962 0 0010.205 0a5.947 5.947 0 00-5.827 4.757C1.713 5.447 0 7.945 0 10.49c0 1.666.713 3.283 1.998 4.448-.119.5-.19 1-.19 1.499 0 3.401 2.759 5.946 5.946 5.946.642 0 1.26-.095 1.88-.309a5.96 5.96 0 004.162 1.713z"></path></svg>

ui/litellm-dashboard/public/assets/logos/opentelemetry.svg

@@ -0,0 +1 @@
+<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>OpenTelemetry</title><path d="M12.6974 13.1173c-1.0224 1.0224-1.0224 2.68 0 3.7024 1.0224 1.0224 2.68 1.0224 3.7024 0 1.0224-1.0223 1.0224-2.68 0-3.7024-1.0223-1.0223-2.68-1.0223-3.7024 0zm2.7677 2.7701c-.5063.5063-1.3267.5063-1.833 0s-.5063-1.3266 0-1.833c.5063-.5062 1.3267-.5062 1.833 0 .5063.504.5063 1.3267 0 1.833zM16.356.2355l-1.6041 1.6042c-.314.314-.314.83 0 1.144L21.015 9.247c.314.314.83.314 1.144 0l1.6042-1.6041c.314-.314.314-.83 0-1.144L17.4976.2354c-.314-.314-.8276-.314-1.1416 0zM5.1173 20.734c.2848-.2848.2848-.7497 0-1.0345l-.8155-.8155c-.2848-.2848-.7497-.2848-1.0345 0l-1.6845 1.6845-.0024.0024-.4625-.4625c-.2556-.2556-.6718-.2556-.925 0-.2556.2556-.2556.6718 0 .925l2.775 2.775c.2556.2556.6718.2556.925 0 .2532-.2556.2556-.6718 0-.925l-.4625-.4625.0024-.0024zm8.4856-15.893-3.5637 3.5637c-.3164.3164-.3164.8374 0 1.1538l2.2006 2.2005c1.5554-1.1197 3.7365-.981 5.1361.4187l1.7819-1.7818c.3164-.3165.3164-.8374 0-1.1538l-4.401-4.401c-.3165-.319-.8374-.319-1.1539 0zm-2.2881 7.8455-1.2999-1.2999c-.3043-.3043-.8033-.3043-1.1076 0l-4.5836 4.586c-.3042.3043-.3042.8033 0 1.1076l2.5973 2.5973c.3043.3043.8033.3043 1.1076 0l2.9478-2.9527c-.6231-1.2877-.5112-2.8431.3384-4.0383z"/></svg>

ui/litellm-dashboard/public/assets/logos/pydantic-ai-color.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>PydanticAI</title><path d="M13.223 22.86c-.605.83-1.844.83-2.448 0L5.74 15.944a1.514 1.514 0 01.73-2.322l5.035-1.738c.32-.11.668-.11.988 0l5.035 1.738c.962.332 1.329 1.5.73 2.322l-5.035 6.916zm-1.224-1.259l4.688-6.439-4.688-1.618-4.688 1.618L12 21.602z" fill="#E72564"></path><path d="M23.71 13.463c.604.832.221 2.01-.756 2.328l-8.133 2.652a1.514 1.514 0 01-1.983-1.412l-.097-5.326c-.006-.338.101-.67.305-.94l3.209-4.25a1.514 1.514 0 012.434.022l5.022 6.926zm-1.574.775L17.46 7.79l-2.988 3.958.09 4.959 7.574-2.469z" fill="#E723A0"></path><path d="M18.016.591a1.514 1.514 0 011.98 1.44l.009 8.554a1.514 1.514 0 01-1.956 1.45l-5.095-1.554a1.514 1.514 0 01-.8-.58l-3.05-4.366a1.514 1.514 0 01.774-2.308L18.016.591zm.25 1.738L10.69 4.783l2.841 4.065 4.744 1.446-.008-7.965z" fill="#E520E9"></path><path d="M5.99.595a1.514 1.514 0 00-1.98 1.44L4 10.588a1.514 1.514 0 001.956 1.45l5.095-1.554c.323-.098.605-.303.799-.58l3.052-4.366a1.514 1.514 0 00-.775-2.308L5.99.595zm-.25 1.738l7.577 2.454-2.842 4.065-4.743 1.446.007-7.965z" fill="#E520E9"></path><path d="M.29 13.461c-.604.832-.221 2.01.756 2.329l8.133 2.651a1.514 1.514 0 001.983-1.412l.097-5.325a1.515 1.515 0 00-.305-.94L7.745 6.513a1.514 1.514 0 00-2.434.023L.289 13.461zm1.574.776L6.54 7.788l2.988 3.959-.09 4.958-7.574-2.468z" fill="#E723A0"></path><path d="M16.942 17.751l1.316-1.806c.119-.164.2-.341.245-.523l-2.63.858-1.627 2.235a1.51 1.51 0 00.575-.072l2.121-.692zM12.746 11.971l.033 1.842 1.742.602-.034-1.843-1.741-.6zM20.003 8.349l-1.314-1.812a1.513 1.513 0 00-.419-.393l.003 2.767 1.624 2.24c.07-.173.108-.364.108-.566l-.002-2.236zM14.965 11.095l-1.762-.537 1.11-1.471 1.762.537-1.11 1.47zM12.004 9.685l1.056-1.51-1.056-1.51-1.056 1.51 1.056 1.51zM9.368 3.509c.145-.122.316-.219.51-.282l2.12-.686 2.13.69c.191.062.36.157.503.276l-2.634.853-2.629-.851zM10.801 10.562L9.691 9.09l-1.762.537 1.11 1.47 1.762-.537zM4.105 11.146L5.733 8.9l.003-2.763c-.16.1-.305.232-.425.398L4.003 8.339l-.002 2.25c0 .198.037.386.104.557zM11.254 11.97l-1.741.601-.034 1.843 1.742-.601.033-1.843zM9.75 18.513l-1.628-2.237-2.629-.857c.046.183.127.36.247.525l1.313 1.804 2.126.693c.192.062.385.085.571.072z" fill="#FF96D1"></path></svg>

ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx

@@ -23,7 +23,9 @@ vi.mock("@/components/DashboardHeader", () => ({
 }));
 
 vi.mock("@/app/(dashboard)/components/SidebarProvider", () => ({
-  default: () => <div data-testid="sidebar" />,
+  default: ({ sidebarCollapsed }: { sidebarCollapsed: boolean }) => (
+    <div data-testid="sidebar" data-collapsed={String(sidebarCollapsed)} />
+  ),
 }));
 
 vi.mock("@/components/DebugWarningBanner", () => ({
@@ -112,6 +114,27 @@ describe("(dashboard) Layout", () => {
     },
   );
 
+  it("collapses the sidebar on Logs for a full-screen view and expands it again after leaving", async () => {
+    const dashboard = () => (
+      <AuthProvider>
+        <Layout>
+          <div data-testid="page-content" />
+        </Layout>
+      </AuthProvider>
+    );
+    const { rerender } = render(dashboard());
+    pendingUiConfig.resolve();
+    expect(await screen.findByTestId("sidebar")).toHaveAttribute("data-collapsed", "false");
+
+    vi.mocked(usePathname).mockReturnValue("/ui/logs");
+    rerender(dashboard());
+    expect(screen.getByTestId("sidebar")).toHaveAttribute("data-collapsed", "true");
+
+    vi.mocked(usePathname).mockReturnValue("/ui/api-keys");
+    rerender(dashboard());
+    expect(screen.getByTestId("sidebar")).toHaveAttribute("data-collapsed", "false");
+  });
+
   it("does not mount route content until getUiConfig has resolved", async () => {
     render(
       <AuthProvider>

ui/litellm-dashboard/src/app/(dashboard)/layout.tsx

@@ -99,11 +99,18 @@ export function AgentControlPlaneView() {
   );
 }
 
+const FULL_BLEED_SEGMENTS = new Set(["logs"]);
+
 function DashboardShell({ children }: { children: React.ReactNode }) {
   const { accessToken } = useAuth();
-  const [sidebarCollapsed, setSidebarCollapsed] = useState(false);
   const { mode } = usePluginMode();
-  const isPlayground = routeSegmentForPathname(usePathname()) === "playground";
+  const routeSegment = routeSegmentForPathname(usePathname());
+  const isPlayground = routeSegment === "playground";
+  const isFullBleed = FULL_BLEED_SEGMENTS.has(routeSegment);
+  // A manual toggle holds only for the route it was made on; full-bleed routes default to collapsed.
+  const [sidebarOverride, setSidebarOverride] = useState<{ segment: string; collapsed: boolean } | null>(null);
+  const sidebarCollapsed = sidebarOverride?.segment === routeSegment ? sidebarOverride.collapsed : isFullBleed;
+  const toggleSidebar = () => setSidebarOverride({ segment: routeSegment, collapsed: !sidebarCollapsed });
 
   const isGateway = mode === "ai-gateway";
 
@@ -133,7 +140,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
   // so the page can't be dragged past the end of the nav.
   return (
     <div className="flex h-screen overflow-hidden bg-background">
-      <SidebarProvider sidebarCollapsed={sidebarCollapsed} onToggleCollapsed={() => setSidebarCollapsed((v) => !v)} />
+      <SidebarProvider sidebarCollapsed={sidebarCollapsed} onToggleCollapsed={toggleSidebar} />
       <div className="flex min-w-0 flex-1 flex-col overflow-hidden">
         <DashboardHeader />
         <DebugWarningBanner accessToken={accessToken} />

ui/litellm-dashboard/src/components/networking.tsx

@@ -115,6 +115,7 @@ import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity
 import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets";
 import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab";
 import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard";
+import type { SpanDetail, Trace, TracePage } from "./view_logs/TraceView/traceTypes";
 import {
   createApiClient,
   deriveErrorMessage,
@@ -2101,6 +2102,33 @@ export const uiSpendLogsCall = async ({
   }
 };
 
+/**
+ * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not
+ * configured; callers can detect that through the thrown `ApiError`'s `status`.
+ */
+export const agentTraceListCall = async ({
+  accessToken,
+  startMs,
+  endMs,
+  cursor,
+}: {
+  accessToken: string;
+  startMs: number;
+  endMs: number;
+  cursor?: string | null;
+}): Promise<TracePage> => {
+  const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined };
+  return apiClient.get<TracePage>(`/v1/traces`, { accessToken, query });
+};
+
+export const agentTraceCall = async (accessToken: string, traceId: string): Promise<Trace> =>
+  apiClient.get<Trace>(`/v1/traces/${encodeURIComponent(traceId)}`, { accessToken });
+
+export const agentTraceSpanCall = async (accessToken: string, traceId: string, spanId: string): Promise<SpanDetail> =>
+  apiClient.get<SpanDetail>(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, {
+    accessToken,
+  });
+
 export const adminSpendLogsCall = async (accessToken: string) => {
   try {
     const data = await apiClient.get(`/global/spend/logs`, { accessToken });

ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx

@@ -0,0 +1,42 @@
+"use client";
+
+import moment from "moment";
+import { useMemo, useState } from "react";
+
+import { AgentTracesSection } from "./AgentTracesSection";
+
+const DEFAULT_RANGE_HOURS = 24;
+const TIME_FORMAT = "YYYY-MM-DDTHH:mm";
+
+/** Agent Traces: agent runs (developer view). Lives as the "Agent Traces" tab of the Logs page. */
+export default function AgentTracesPage({ accessToken }: { accessToken: string }) {
+  const [rangeHours, setRangeHours] = useState(DEFAULT_RANGE_HOURS);
+  const [live, setLive] = useState(true);
+  const [anchor, setAnchor] = useState(() => moment());
+  const { startTime, endTime } = useMemo(
+    () => ({
+      startTime: anchor.clone().subtract(rangeHours, "hours").format(TIME_FORMAT),
+      endTime: anchor.format(TIME_FORMAT),
+    }),
+    [anchor, rangeHours],
+  );
+
+  const changeRange = (hours: number) => {
+    setRangeHours(hours);
+    setAnchor(moment());
+  };
+
+  return (
+    <div className="flex min-h-0 flex-1 flex-col pt-3">
+      <AgentTracesSection
+        accessToken={accessToken}
+        isActive
+        startTime={startTime}
+        endTime={endTime}
+        isCustomDate={false}
+        isLiveTail={live}
+        timeControls={{ rangeHours, onRangeHoursChange: changeRange, onLiveChange: setLive }}
+      />
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx

@@ -0,0 +1,265 @@
+import { fireEvent, screen, within } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { ApiError } from "@/lib/http/client";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import traceList from "./__fixtures__/trace_list.json";
+import AgentTracesPage from "./AgentTracesPage";
+import { AgentTracesSection, filterRuns } from "./AgentTracesSection";
+import type { TracePage, TraceSummary } from "./traceTypes";
+
+vi.mock("../../networking", () => ({
+  agentTraceListCall: vi.fn(),
+  agentTraceCall: vi.fn(),
+  agentTraceSpanCall: vi.fn(),
+  getProxyBaseUrl: () => "http://localhost:4000",
+}));
+
+vi.mock("./TraceDrawer", () => ({
+  RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => (
+    <div data-testid="run-view">
+      run {traceId}
+      <button onClick={onBack}>back</button>
+    </div>
+  ),
+}));
+
+import { agentTraceListCall } from "../../networking";
+
+const runs = (traceList as TracePage).data as TraceSummary[];
+
+const renderSection = () =>
+  renderWithProviders(
+    <AgentTracesSection
+      accessToken="sk-test"
+      isActive
+      startTime="2026-09-29T00:00"
+      endTime="2026-09-30T00:00"
+      isCustomDate={false}
+      isLiveTail={false}
+    />,
+  );
+
+// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone.
+const renderWindowed = () =>
+  renderWithProviders(
+    <AgentTracesSection
+      accessToken="sk-test"
+      isActive
+      startTime="2026-09-30T00:00Z"
+      endTime="2026-10-01T00:00Z"
+      isCustomDate
+      isLiveTail={false}
+    />,
+  );
+
+const bucketRunCounts = () =>
+  screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs")));
+
+describe("AgentTracesSection", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  beforeEach(() => {
+    vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({
+      left: 0,
+      width: 600,
+      top: 0,
+      height: 56,
+      right: 600,
+      bottom: 56,
+      x: 0,
+      y: 0,
+      toJSON: () => ({}),
+    } as DOMRect);
+    testQueryClient.clear();
+    vi.mocked(agentTraceListCall).mockReset();
+  });
+
+  it("renders the setup snippet when the proxy answers 501", async () => {
+    vi.mocked(agentTraceListCall).mockRejectedValue(
+      new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }),
+    );
+    renderSection();
+
+    const card = await screen.findByTestId("tracing-setup-card");
+    expect(card).toHaveTextContent("Tracing is not enabled");
+    expect(card).toHaveTextContent("store: clickhouse");
+    expect(card).toHaveTextContent("OTEL_EXPORTER_OTLP_ENDPOINT=");
+    expect(card).not.toHaveTextContent(/langsmith/i);
+    expect(card).toHaveTextContent('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"');
+    expect(card).toHaveTextContent("Let Claude Code or Codex set it up");
+  });
+
+  it("shows the waiting guide when tracing is on but no runs have arrived", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] });
+    renderSection();
+
+    const card = await screen.findByTestId("tracing-setup-card");
+    expect(card).toHaveTextContent("Waiting for traces");
+    expect(card).toHaveTextContent("No traces detected yet");
+    expect(card).not.toHaveTextContent("store: clickhouse");
+  });
+
+  it("treats a proxy without the trace routes (404) like tracing being off", async () => {
+    vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" }));
+    renderSection();
+
+    const card = await screen.findByTestId("tracing-setup-card");
+    expect(card).toHaveTextContent("Tracing is not enabled");
+    expect(card).toHaveTextContent("CLICKHOUSE_READER_URL");
+  });
+
+  it("lists every run with its input, counts and failed column", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderSection();
+
+    const rows = await screen.findAllByTestId("agent-trace-row");
+    expect(rows).toHaveLength(runs.length);
+    const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans"));
+    expect(lead).toBeDefined();
+    const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement;
+    expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument();
+    expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument();
+    expect(screen.queryByRole("columnheader", { name: "Cost" })).not.toBeInTheDocument();
+  });
+
+  it("filters by input text and by trace id", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderSection();
+    await screen.findAllByTestId("agent-trace-row");
+
+    const search = screen.getByLabelText("Search runs");
+    fireEvent.change(search, { target: { value: "acme-404" } });
+    expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1);
+
+    const lead = runs.find((r) => r.name === "research_lead") as TraceSummary;
+    fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } });
+    const rows = screen.getAllByTestId("agent-trace-row");
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans");
+  });
+
+  it("status filter 'Failed' keeps only runs with errors", () => {
+    const failed = filterRuns(runs, "", "all", "error");
+    expect(failed.length).toBeGreaterThan(0);
+    expect(failed.every((r) => r.error_count > 0)).toBe(true);
+    const ok = filterRuns(runs, "", "all", "ok");
+    expect(ok.every((r) => r.error_count === 0)).toBe(true);
+    expect(failed.length + ok.length).toBe(runs.length);
+  });
+
+  it("opens the run in place and goes back to the list", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderSection();
+    const rows = await screen.findAllByTestId("agent-trace-row");
+
+    fireEvent.click(rows[0]);
+    expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`);
+    expect(screen.queryByTestId("runs-table")).not.toBeInTheDocument();
+
+    fireEvent.click(screen.getByText("back"));
+    expect(screen.getByTestId("runs-table")).toBeInTheDocument();
+  });
+
+  it("plots every loaded run on the timeline", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderWindowed();
+    await screen.findAllByTestId("agent-trace-row");
+
+    expect(screen.getByTestId("traces-timeline")).toBeInTheDocument();
+    const counts = bucketRunCounts();
+    expect(counts).toHaveLength(60);
+    expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length);
+  });
+
+  it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderWindowed();
+    await screen.findAllByTestId("agent-trace-row");
+    const area = screen.getByTestId("timeline-area");
+    const x = (bucket: number) => bucket * 10 + 5;
+    const drag = (target: HTMLElement, from: number, to: number) => {
+      fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 });
+      fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 });
+      fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 });
+    };
+    const rowCount = () => screen.queryAllByTestId("agent-trace-row").length;
+    const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : []));
+    const first = withRuns[0];
+    // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run.
+    expect(first).toBeGreaterThan(1);
+    expect(first).toBeLessThan(30);
+
+    drag(area, 0, 1);
+    expect(screen.getByTestId("timeline-selection")).toBeInTheDocument();
+    expect(rowCount()).toBe(0);
+
+    drag(screen.getByTestId("timeline-handle-hi"), 1, first);
+    expect(rowCount()).toBeGreaterThan(0);
+
+    drag(screen.getByTestId("timeline-selection"), 1, 1 - first);
+    expect(rowCount()).toBeGreaterThan(0);
+    drag(screen.getByTestId("timeline-selection"), 0, 59);
+    expect(rowCount()).toBe(0);
+
+    fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" });
+    expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument();
+    expect(rowCount()).toBe(runs.length);
+  });
+});
+
+describe("AgentTracesPage", () => {
+  beforeEach(() => {
+    testQueryClient.clear();
+    vi.mocked(agentTraceListCall).mockReset();
+  });
+
+  it("shows the actual range, switches presets from the popover, and toggles Live", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderWithProviders(<AgentTracesPage accessToken="sk-test" />);
+    await screen.findByTestId("runs-table");
+
+    const trigger = screen.getByRole("button", { name: "Time range" });
+    expect(trigger).toHaveTextContent(/ to /);
+    expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d");
+
+    fireEvent.click(trigger);
+    fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" }));
+    expect(await screen.findByText("Total 7d")).toBeInTheDocument();
+    const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0];
+    expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000);
+
+    const live = screen.getByRole("button", { name: "Live" });
+    expect(live).toHaveAttribute("aria-pressed", "true");
+    fireEvent.click(live);
+    expect(live).toHaveAttribute("aria-pressed", "false");
+    expect(screen.getByRole("button", { name: "Reset zoom" })).toBeDisabled();
+  });
+
+  it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderWithProviders(<AgentTracesPage accessToken="sk-test" />);
+    await screen.findByTestId("runs-table");
+
+    vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] });
+    fireEvent.click(screen.getByRole("button", { name: "Time range" }));
+    fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" }));
+
+    expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument();
+    expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument();
+  });
+
+  it("asks the proxy for the last 24 hours by default", async () => {
+    vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage);
+    renderWithProviders(<AgentTracesPage accessToken="sk-test" />);
+    await screen.findByTestId("runs-table");
+
+    const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0];
+    expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000);
+    expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000);
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx

@@ -0,0 +1,185 @@
+"use client";
+
+import moment from "moment";
+import { useMemo, useState } from "react";
+
+import { AgentTracesTable } from "./AgentTracesTable";
+import { ALL_SERVICES, RunsToolbar, type RunStatusFilter } from "./RunsToolbar";
+import { RunView } from "./TraceDrawer";
+import type { TraceSummary } from "./traceTypes";
+import { previewText } from "./traceUtils";
+import { TimeRangeControls } from "./TimeRangeControls";
+import { TracesTimeline, type TimeWindow } from "./TracesTimeline";
+import { TracingSetupCard } from "./TracingSetupCard";
+import { traceWindowStartMs, useAgentTraces } from "./useAgentTraces";
+
+/** Client-side search (input text or trace id) plus service / status filters over the loaded runs. */
+export function filterRuns(
+  runs: TraceSummary[],
+  query: string,
+  service: string,
+  status: RunStatusFilter,
+): TraceSummary[] {
+  const q = query.trim().toLowerCase();
+  return runs.filter((run) => {
+    const haystack = [run.trace_id, previewText(run.input_preview), run.name].map((s) => s.toLowerCase());
+    const matchesQuery = !q || haystack.some((text) => text.includes(q));
+    const matchesService = service === ALL_SERVICES || run.service === service;
+    const failed = run.error_count > 0;
+    const matchesStatus = status === "all" || (status === "error" ? failed : !failed);
+    return matchesQuery && matchesService && matchesStatus;
+  });
+}
+
+const filterByWindow = (runs: TraceSummary[], range: TimeWindow): TraceSummary[] =>
+  runs.filter((run) => {
+    const t = moment(run.start_time).valueOf();
+    return t >= range.startMs && t < range.endMs;
+  });
+
+export interface TimeControls {
+  rangeHours: number;
+  onRangeHoursChange: (hours: number) => void;
+  onLiveChange: (live: boolean) => void;
+}
+
+interface AgentTracesSectionProps {
+  accessToken: string;
+  isActive: boolean;
+  startTime: string;
+  endTime: string;
+  isCustomDate: boolean;
+  isLiveTail: boolean;
+  /** Page-owned time range + live state; when given, the toolbar shows the range / Live control group. */
+  timeControls?: TimeControls;
+  /** Called when a run opens / closes, so the page can hide its own header while a run fills the view. */
+  onRunOpenChange?: (open: boolean) => void;
+}
+
+/** The Runs view: filters, the runs table and footer — or one run, in place, once a row is clicked. */
+export function AgentTracesSection({
+  accessToken,
+  isActive,
+  startTime,
+  endTime,
+  isCustomDate,
+  isLiveTail,
+  timeControls,
+  onRunOpenChange,
+}: AgentTracesSectionProps) {
+  const [openTraceId, setOpenTraceId] = useState<string | null>(null);
+  const [query, setQuery] = useState("");
+  const [service, setService] = useState(ALL_SERVICES);
+  const [status, setStatus] = useState<RunStatusFilter>("all");
+  const [showSetup, setShowSetup] = useState(false);
+  const [zoom, setZoom] = useState<TimeWindow | null>(null);
+  const [rangeChanged, setRangeChanged] = useState(false);
+  const traceQuery = { accessToken, startTime, endTime, isCustomDate, isLiveTail, enabled: isActive };
+  const traces = useAgentTraces(traceQuery);
+
+  const services = useMemo(() => Array.from(new Set(traces.traces.map((t) => t.service))).sort(), [traces.traces]);
+  // Relative ranges end "now" (the list query uses Date.now() too); round to the minute so the histogram is stable.
+  const endMs = isCustomDate ? moment(endTime).valueOf() : moment().endOf("minute").valueOf();
+  const range = useMemo(
+    () => ({ startMs: traceWindowStartMs(startTime, endTime, isCustomDate, endMs), endMs }),
+    [startTime, endTime, isCustomDate, endMs],
+  );
+  const filtered = useMemo(
+    () => filterRuns(traces.traces, query, service, status),
+    [traces.traces, query, service, status],
+  );
+  const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]);
+
+  const changeRange = (hours: number, apply: (hours: number) => void) => {
+    setZoom(null);
+    setRangeChanged(true);
+    apply(hours);
+  };
+
+  const openRun = (traceId: string | null) => {
+    setOpenTraceId(traceId);
+    onRunOpenChange?.(traceId !== null);
+  };
+
+  if (traces.notEnabledDetail !== null) return <TracingSetupCard detail={traces.notEnabledDetail} />;
+  // Onboarding only on the first, default view; an empty range the user picked keeps its controls.
+  const isEmpty = !traces.isLoading && !traces.error && traces.traces.length === 0;
+  if (isEmpty && !rangeChanged) return <TracingSetupCard detail={null} />;
+  if (showSetup) {
+    return (
+      <div>
+        <button
+          type="button"
+          onClick={() => setShowSetup(false)}
+          className="mb-3 text-[13px] text-muted-foreground hover:text-foreground"
+        >
+          ← Back to traces
+        </button>
+        <TracingSetupCard detail={null} connected />
+      </div>
+    );
+  }
+
+  if (openTraceId !== null) {
+    return <RunView traceId={openTraceId} accessToken={accessToken} onBack={() => openRun(null)} />;
+  }
+
+  return (
+    <div className="flex min-h-[560px] flex-1 flex-col overflow-hidden border-y border-border bg-card">
+      <RunsToolbar
+        query={query}
+        service={service}
+        status={status}
+        services={services}
+        onQueryChange={setQuery}
+        onServiceChange={setService}
+        onStatusChange={setStatus}
+      >
+        <button
+          type="button"
+          onClick={() => setShowSetup(true)}
+          className="shrink-0 px-1 text-[11px] text-muted-foreground underline-offset-2 hover:text-info hover:underline"
+        >
+          Set up tracing
+        </button>
+        {timeControls && (
+          <TimeRangeControls
+            range={zoom ?? range}
+            rangeHours={timeControls.rangeHours}
+            onRangeHoursChange={(hours) => changeRange(hours, timeControls.onRangeHoursChange)}
+            live={isLiveTail}
+            onLiveChange={timeControls.onLiveChange}
+            zoomed={zoom !== null}
+            onResetZoom={() => setZoom(null)}
+          />
+        )}
+      </RunsToolbar>
+      <TracesTimeline runs={filtered} range={range} selection={zoom} onSelect={setZoom} />
+      <AgentTracesTable
+        traces={runs}
+        isLoading={traces.isLoading}
+        error={traces.error}
+        hasMore={traces.hasMore}
+        onLoadMore={traces.loadMore}
+        onOpenTrace={openRun}
+      />
+      <footer
+        data-testid="runs-footer"
+        className="flex h-8 shrink-0 items-center border-t border-border bg-muted/40 px-3 font-mono text-[11px] text-muted-foreground"
+      >
+        {runs.length} {runs.length === 1 ? "run" : "runs"}
+        {zoom && (
+          <button
+            type="button"
+            onClick={() => setZoom(null)}
+            aria-label="Clear time zoom"
+            className="ml-3 rounded border border-info/40 bg-info/10 px-1.5 text-info hover:bg-info/20"
+          >
+            {moment(zoom.startMs).format("MMM DD, HH:mm")} to {moment(zoom.endMs).format("MMM DD, HH:mm")} ×
+          </button>
+        )}
+        <span className="ml-auto">{traces.isFetching ? "Updating…" : "Updated just now"}</span>
+      </footer>
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx

@@ -0,0 +1,148 @@
+"use client";
+
+import { ArrowDown, ChevronRight } from "lucide-react";
+
+import { Button } from "@/components/ui/button";
+
+import { StatusMark } from "./StatusMark";
+import type { TraceSummary } from "./traceTypes";
+import { fmtMs, previewText, traceDisplayName } from "./traceUtils";
+
+interface AgentTracesTableProps {
+  traces: TraceSummary[];
+  isLoading: boolean;
+  error: Error | null;
+  hasMore: boolean;
+  onLoadMore: () => void;
+  onOpenTrace: (traceId: string) => void;
+}
+
+/** Spend is only on summaries once the spend-enrichment PR lands; show Cost when it's there. */
+type SummaryWithSpend = TraceSummary & { spend?: number };
+
+const SECOND_MS = 1000;
+const MINUTE_S = 60;
+const HOUR_M = 60;
+const DAY_H = 24;
+
+export function relativeTime(iso: string, now: number = Date.now()): string {
+  const diffS = Math.round((now - new Date(iso).getTime()) / SECOND_MS);
+  if (diffS < 5) return "just now";
+  if (diffS < MINUTE_S) return `${diffS}s ago`;
+  const diffM = Math.round(diffS / MINUTE_S);
+  if (diffM < HOUR_M) return `${diffM}m ago`;
+  const diffH = Math.round(diffM / HOUR_M);
+  if (diffH < DAY_H) return `${diffH}h ago`;
+  return `${Math.round(diffH / DAY_H)}d ago`;
+}
+
+export const formatCost = (cost: number): string => {
+  if (cost === 0) return "$0.00";
+  if (cost < 0.01) return `$${cost.toFixed(4)}`;
+  return `$${cost.toFixed(2)}`;
+};
+
+const firstLine = (text: string): string => text.split("\n")[0] ?? text;
+
+const TH = "px-3 font-medium";
+const TH_NUM = "px-3 text-right font-medium";
+const TD_NUM = "px-3 text-right font-mono tabular-nums text-muted-foreground";
+
+/** Devtool-dense runs list: one row per agent run, newest first. */
+export function AgentTracesTable({
+  traces,
+  isLoading,
+  error,
+  hasMore,
+  onLoadMore,
+  onOpenTrace,
+}: AgentTracesTableProps) {
+  const showCost = traces.some((t) => typeof (t as SummaryWithSpend).spend === "number");
+  const isEmpty = !isLoading && !error && traces.length === 0;
+  return (
+    <div className="min-h-0 flex-1 overflow-auto" data-testid="runs-table">
+      <table aria-label="Agent runs" className="w-full min-w-[900px] table-fixed border-collapse text-left">
+        <thead className="sticky top-0 z-sticky bg-muted/40 backdrop-blur">
+          <tr className="h-8 border-b border-border text-[10px] tracking-[0.08em] text-muted-foreground uppercase">
+            <th className={`w-[96px] ${TH}`}>
+              <span className="inline-flex items-center gap-1">
+                Time <ArrowDown className="size-2.5" />
+              </span>
+            </th>
+            <th className={`w-[160px] ${TH}`}>Service</th>
+            <th className={TH}>Input</th>
+            <th className={`w-[72px] ${TH_NUM}`}>Agents</th>
+            <th className={`w-[74px] ${TH_NUM}`}>Steps</th>
+            <th className={`w-[86px] ${TH_NUM}`}>Duration</th>
+            {showCost && <th className={`w-[80px] ${TH_NUM}`}>Cost</th>}
+            <th className={`w-[72px] ${TH_NUM}`}>Failed</th>
+            <th className="w-8" />
+          </tr>
+        </thead>
+        <tbody>
+          {traces.map((run) => (
+            <tr
+              key={run.trace_id}
+              data-testid="agent-trace-row"
+              onClick={() => onOpenTrace(run.trace_id)}
+              className="h-9 cursor-pointer border-b border-border/60 text-[12px] hover:bg-accent/50"
+            >
+              <td
+                className="px-3 font-mono text-[11px] tabular-nums text-muted-foreground"
+                title={new Date(run.start_time).toLocaleString()}
+              >
+                {relativeTime(run.start_time)}
+              </td>
+              <td className="truncate px-3 text-muted-foreground" title={run.service}>
+                {run.service}
+              </td>
+              <td className="px-3">
+                <div className="flex min-w-0 items-center gap-2">
+                  <StatusMark status={run.error_count > 0 ? "error" : "ok"} subtle />
+                  <span className="truncate text-foreground">
+                    {firstLine(previewText(run.input_preview)) || traceDisplayName(run)}
+                  </span>
+                  <span className="hidden shrink-0 font-mono text-[10px] text-muted-foreground 2xl:inline">
+                    {run.trace_id}
+                  </span>
+                </div>
+              </td>
+              <td className={TD_NUM}>{run.agent_count.toLocaleString()}</td>
+              <td className={TD_NUM}>{run.span_count.toLocaleString()}</td>
+              <td className="px-3 text-right font-mono tabular-nums text-foreground">{fmtMs(run.duration_ms)}</td>
+              {showCost && (
+                <td className="px-3 text-right font-mono tabular-nums text-foreground">
+                  {formatCost((run as SummaryWithSpend).spend ?? 0)}
+                </td>
+              )}
+              <td className="px-3 text-right">
+                {run.error_count > 0 ? (
+                  <StatusMark status="error" count={run.error_count} />
+                ) : (
+                  <span className="font-mono text-[11px] text-muted-foreground/60">0</span>
+                )}
+              </td>
+              <td>
+                <ChevronRight className="size-3 text-muted-foreground/60" />
+              </td>
+            </tr>
+          ))}
+        </tbody>
+      </table>
+      {isLoading && <div className="py-16 text-center text-[12px] text-muted-foreground">Loading runs…</div>}
+      {error && (
+        <div className="py-16 text-center text-[12px] text-muted-foreground">Could not load runs: {error.message}</div>
+      )}
+      {isEmpty && (
+        <div className="py-16 text-center text-[12px] text-muted-foreground">No runs match these filters.</div>
+      )}
+      {hasMore && (
+        <div className="border-t border-border/60 px-3 py-2">
+          <Button size="xs" variant="ghost" onClick={onLoadMore}>
+            Load more
+          </Button>
+        </div>
+      )}
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/AttributesDetail.tsx

@@ -0,0 +1,33 @@
+"use client";
+
+import type { Span } from "./traceTypes";
+
+interface AttributesDetailProps {
+  traceId: string;
+  span: Span;
+  attributes: Record<string, string> | undefined;
+  isLoading: boolean;
+}
+
+/** Raw OTEL attributes as a key / value grid, ids first. */
+export function AttributesDetail({ traceId, span, attributes, isLoading }: AttributesDetailProps) {
+  const entries: [string, string][] = [
+    ["trace_id", traceId],
+    ["span_id", span.span_id],
+    ["parent_span_id", span.parent_span_id ?? "—"],
+    ...Object.entries(attributes ?? {}).sort(([a], [b]) => a.localeCompare(b)),
+  ];
+  return (
+    <div className="p-3">
+      <div className="overflow-hidden rounded border border-border bg-card font-mono text-[11px]">
+        {entries.map(([key, value]) => (
+          <div key={key} className="grid grid-cols-[minmax(120px,42%)_1fr] border-b border-border last:border-0">
+            <div className="border-r border-border bg-muted/40 px-2.5 py-2 break-all text-muted-foreground">{key}</div>
+            <div className="px-2.5 py-2 break-all text-foreground">{value}</div>
+          </div>
+        ))}
+      </div>
+      {isLoading && <div className="mt-2 font-mono text-[11px] text-muted-foreground">Loading attributes…</div>}
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/CopyButton.tsx

@@ -0,0 +1,63 @@
+"use client";
+
+import { Check, Copy } from "lucide-react";
+import { useEffect, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+const COPIED_RESET_MS = 1600;
+
+interface CopyButtonProps {
+  value: string;
+  label?: string;
+  copiedLabel?: string;
+  iconOnly?: boolean;
+  className?: string;
+}
+
+/** Copy → check for a moment. `iconOnly` renders a bare icon button with a tooltip. */
+export function CopyButton({
+  value,
+  label = "Copy",
+  copiedLabel = "Copied",
+  iconOnly = false,
+  className,
+}: CopyButtonProps) {
+  const [copied, setCopied] = useState(false);
+
+  useEffect(() => {
+    if (!copied) return;
+    const timeout = window.setTimeout(() => setCopied(false), COPIED_RESET_MS);
+    return () => window.clearTimeout(timeout);
+  }, [copied]);
+
+  const button = (
+    <Button
+      variant={iconOnly ? "ghost" : "outline"}
+      size={iconOnly ? "icon-xs" : "xs"}
+      className={cn(
+        "font-mono text-[10px] text-muted-foreground hover:text-foreground",
+        !iconOnly && "gap-1.5",
+        className,
+      )}
+      onClick={async () => setCopied(await copyToClipboard(value, copiedLabel))}
+      aria-label={label}
+    >
+      {copied ? <Check className="size-3" /> : <Copy className="size-3" />}
+      {!iconOnly && <span>{copied ? copiedLabel : label}</span>}
+    </Button>
+  );
+
+  if (!iconOnly) return button;
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger render={button} />
+        <TooltipContent>{copied ? copiedLabel : label}</TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx

@@ -0,0 +1,152 @@
+"use client";
+
+import { useQuery, type UseQueryOptions } from "@tanstack/react-query";
+import { AlertTriangle, Bot, CornerDownRight, Wrench } from "lucide-react";
+
+import { agentTraceSpanCall } from "../../networking";
+import type { ErrorSource } from "./traceTree";
+import type { Span, SpanDetail, TraceMessage } from "./traceTypes";
+import { errorSource, parseMessages, prettyPayload } from "./traceUtils";
+
+const ERROR_SOURCE_LABEL: Record<ErrorSource, string> = { tool: "Tool", model: "Model", litellm: "LiteLLM" };
+const TRACEBACK_MARKER = "Traceback (most recent call last):";
+
+/** LangSmith records `repr(exc)` + traceback with no separator; keep the exception line. */
+export const errorHeadline = (error: string): string =>
+  (error.split(TRACEBACK_MARKER, 1)[0].split("\n")[0] ?? "").trim() || error.trim();
+
+/** `ValueError('x not found')` → "ValueError"; plain text → "error". */
+const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(headline)?.[1] ?? "error";
+
+/** Shared lazy fetch of one span's full input / output / attributes. */
+export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null) {
+  const queryOptions: UseQueryOptions<SpanDetail, Error> = {
+    queryKey: ["agentTraceSpan", traceId, spanId, accessToken],
+    queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string),
+    enabled: spanId !== null,
+    staleTime: Infinity,
+  };
+  return useQuery(queryOptions);
+}
+
+export function SectionLabel({ children }: { children: React.ReactNode }) {
+  return (
+    <div className="flex items-center gap-1.5 font-mono text-[9px] font-medium tracking-[0.1em] text-muted-foreground uppercase">
+      {children}
+    </div>
+  );
+}
+
+export function TextBlock({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) {
+  return (
+    <section>
+      <SectionLabel>{label}</SectionLabel>
+      <div
+        className={`mt-1.5 max-h-72 overflow-auto rounded border border-border bg-card p-3 leading-5 whitespace-pre-wrap break-words text-foreground ${
+          mono ? "font-mono text-[11px]" : "text-[12px]"
+        }`}
+      >
+        {value}
+      </div>
+    </section>
+  );
+}
+
+function RoleIcon({ role }: { role: string }) {
+  if (role === "assistant") return <Bot className="size-3" />;
+  if (role === "tool") return <Wrench className="size-3" />;
+  return <CornerDownRight className="size-3" />;
+}
+
+export function MessageBlock({ message }: { message: TraceMessage }) {
+  return (
+    <section className="overflow-hidden rounded border border-border bg-card">
+      <div className="flex h-7 items-center gap-2 border-b border-border bg-muted/40 px-2.5 font-mono text-[9px] tracking-[0.1em] text-muted-foreground uppercase">
+        <RoleIcon role={message.role} />
+        {message.role}
+        {message.name ? <span className="normal-case">· {message.name}</span> : null}
+      </div>
+      {(message.tool_calls ?? []).map((call, i) => (
+        <div
+          key={`${call.name}-${i}`}
+          className="border-b border-border bg-muted/20 p-2.5 font-mono text-[11px] break-all text-foreground last:border-b-0"
+        >
+          {call.name}
+          <span className="text-muted-foreground">(</span>
+          <span className="text-muted-foreground">{JSON.stringify(call.args)}</span>
+          <span className="text-muted-foreground">)</span>
+        </div>
+      ))}
+      {message.content && (
+        <div className="p-2.5 text-[12px] leading-5 whitespace-pre-wrap break-words text-foreground">
+          {message.content}
+        </div>
+      )}
+    </section>
+  );
+}
+
+export function ErrorBlock({ span }: { span: Span }) {
+  const source = errorSource(span);
+  if (!source) return null;
+  const headline = errorHeadline(span.error ?? "") || "Span reported an error status.";
+  return (
+    <section aria-label="Error" className="rounded border border-destructive/40 bg-destructive/5 p-3">
+      <div className="flex items-center gap-2 font-mono text-[9px] font-medium tracking-[0.08em] text-destructive uppercase">
+        <AlertTriangle className="size-3" />
+        {ERROR_SOURCE_LABEL[source]} · {errorReason(headline)}
+      </div>
+      <pre className="mt-2 font-mono text-[11px] leading-5 whitespace-pre-wrap break-words text-foreground">
+        {headline}
+      </pre>
+    </section>
+  );
+}
+
+function Payload({ label, value, mono }: { label: string; value: string; mono: boolean }) {
+  const messages = parseMessages(value);
+  if (messages) {
+    return (
+      <>
+        <SectionLabel>{`${label}${messages.length > 1 ? ` · ${messages.length} messages` : ""}`}</SectionLabel>
+        {messages.map((message, i) => (
+          <MessageBlock key={`${message.role}-${i}`} message={message} />
+        ))}
+      </>
+    );
+  }
+  return <TextBlock label={label} value={prettyPayload(value)} mono={mono} />;
+}
+
+interface DetailContentProps {
+  accessToken: string;
+  traceId: string;
+  span: Span;
+}
+
+/** Content tab: the error first (if any), then what went in and what came out. */
+export function DetailContent({ accessToken, traceId, span }: DetailContentProps) {
+  const detailQuery = useSpanDetail(accessToken, traceId, span.span_id);
+  const detail = detailQuery.data;
+  const isTool = span.type === "tool";
+  const empty = detail && !detail.input && !detail.output;
+
+  return (
+    <div className="space-y-3 p-3">
+      <ErrorBlock span={span} />
+      {detailQuery.isLoading && <div className="font-mono text-[11px] text-muted-foreground">Loading span…</div>}
+      {detailQuery.isError && (
+        <div className="font-mono text-[11px] text-muted-foreground">
+          Could not load span: {detailQuery.error.message}
+        </div>
+      )}
+      {detail?.input ? <Payload label={isTool ? "Input" : "Input"} value={detail.input} mono={isTool} /> : null}
+      {detail?.output ? <Payload label="Output" value={detail.output} mono={isTool} /> : null}
+      {empty && span.status !== "error" && (
+        <div className="py-12 text-center font-mono text-[11px] text-muted-foreground">
+          No content recorded for this span.
+        </div>
+      )}
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx

@@ -0,0 +1,191 @@
+import { screen, waitFor } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import { DetailPane } from "./DetailPane";
+import type { GroupRowData, SpanRowData } from "./traceTree";
+import type { Span, SpanDetail, Trace } from "./traceTypes";
+
+vi.mock("../../networking", () => ({
+  agentTraceSpanCall: vi.fn(),
+  getProxyBaseUrl: () => "http://proxy.test/",
+}));
+
+import { agentTraceSpanCall } from "../../networking";
+
+const span = (overrides: Partial<Span> & Pick<Span, "span_id">): Span => ({
+  parent_span_id: "root",
+  name: overrides.span_id,
+  type: "chain",
+  agent: "support_triage_agent",
+  start_offset_ms: 0,
+  duration_ms: 1300,
+  status: "ok",
+  error: null,
+  input_preview: "",
+  model: null,
+  input_tokens: 0,
+  output_tokens: 0,
+  litellm_request_id: null,
+  ...overrides,
+});
+
+const rootFields: SpanFields = { span_id: "root", parent_span_id: null, name: "support_triage_agent", type: "agent" };
+const llmFields: SpanFields = {
+  span_id: "llm1",
+  name: "ChatOpenAI",
+  type: "llm",
+  model: "claude-sonnet-4-5",
+  input_tokens: 659,
+  output_tokens: 60,
+  litellm_request_id: "chatcmpl-abc",
+};
+const failedToolFields: SpanFields = {
+  span_id: "tool1",
+  name: "get_customer_plan",
+  type: "tool",
+  status: "error",
+  error:
+    "ValueError('customer acme-404 not found in billing DB')Traceback (most recent call last):\n  File \"x.py\", line 1",
+};
+const root = span(rootFields);
+const llm = span(llmFields);
+const failedTool = span(failedToolFields);
+
+const trace: Trace = {
+  summary: {
+    trace_id: "t1",
+    name: "support_triage_agent",
+    service: "research-agent",
+    input_preview: '[{"role": "user", "content": "Customer acme-404 says billing is wrong."}]',
+    start_time: "2026-09-30T06:43:52.928000+00:00",
+    duration_ms: 1310,
+    status: "ok",
+    span_count: 3,
+    agent_count: 1,
+    agent_invocations: 1,
+    llm_calls: 1,
+    tool_calls: 1,
+    error_count: 1,
+    input_tokens: 659,
+    output_tokens: 60,
+    models: ["claude-sonnet-4-5"],
+  } as Trace["summary"],
+  agents: [],
+  spans: [root, llm, failedTool],
+};
+
+const details: Record<string, SpanDetail> = {
+  llm1: {
+    span_id: "llm1",
+    input: JSON.stringify([
+      { role: "system", content: "You are a LiteLLM support agent." },
+      { role: "user", content: "Customer acme-404 says billing is wrong." },
+    ]),
+    output: JSON.stringify({
+      role: "assistant",
+      content: "",
+      tool_calls: [{ name: "get_customer_plan", args: { customer_id: "acme-404" } }],
+    }),
+    attributes: { "gen_ai.request.model": "claude-sonnet-4-5" },
+  },
+  tool1: { span_id: "tool1", input: '{"customer_id":"acme-404"}', output: "", attributes: {} },
+  root: {
+    span_id: "root",
+    input: JSON.stringify([{ role: "user", content: "Customer acme-404 says billing is wrong." }]),
+    output: JSON.stringify({ role: "assistant", content: "Customer acme-404 is on the Enterprise plan." }),
+    attributes: {},
+  },
+};
+
+const spanRow = (s: Span): SpanRowData => ({
+  kind: "span",
+  id: s.span_id,
+  span: s,
+  depth: 1,
+  hasChildren: false,
+  collapsed: false,
+});
+
+const renderPane = (row: SpanRowData | GroupRowData) =>
+  renderWithProviders(<DetailPane trace={trace} row={row} accessToken="sk-test" onClose={vi.fn()} />);
+
+describe("DetailPane", () => {
+  beforeEach(() => {
+    testQueryClient.clear();
+    vi.mocked(agentTraceSpanCall).mockReset();
+    vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, spanId) => details[spanId]);
+  });
+
+  it("renders the span tabs and the fetched LLM conversation with its tool call", async () => {
+    renderPane(spanRow(llm));
+    expect(screen.getByRole("tab", { name: "Content" })).toBeInTheDocument();
+    expect(screen.getByRole("tab", { name: "Request" })).toBeInTheDocument();
+    expect(screen.getByRole("tab", { name: "Attributes" })).toBeInTheDocument();
+    expect(await screen.findByText("You are a LiteLLM support agent.")).toBeInTheDocument();
+    expect(screen.getByText("get_customer_plan")).toBeInTheDocument();
+    expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1");
+  });
+
+  it("shows a tool failure as 'Tool · <reason>' with the exception line and no traceback", async () => {
+    renderPane(spanRow(failedTool));
+    const error = screen.getByRole("region", { name: "Error" });
+    expect(error).toHaveTextContent("Tool · ValueError");
+    expect(error).toHaveTextContent("ValueError('customer acme-404 not found in billing DB')");
+    expect(error).not.toHaveTextContent("Traceback");
+    // tool args render as pretty JSON under "Input"
+    expect(await screen.findByText(/"customer_id": "acme-404"/)).toBeInTheDocument();
+  });
+
+  it("shows the LiteLLM request facts on the Request tab", async () => {
+    const user = userEvent.setup();
+    renderPane(spanRow(llm));
+    await user.click(screen.getByRole("tab", { name: "Request" }));
+    expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument();
+    expect(screen.getByText("659")).toBeInTheDocument();
+    expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument();
+  });
+
+  it("summarizes a ×N group with its failure pattern", () => {
+    const members = Array.from({ length: 12 }, (_, i) => {
+      const timedOut: SpanFields = {
+        span_id: `f${i}`,
+        name: "lookup_benchmark",
+        type: "tool",
+        status: "error",
+        error: "TimeoutError('slow')",
+      };
+      return span(timedOut);
+    });
+    const groupRow: GroupRowData = {
+      kind: "group",
+      id: "grp",
+      depth: 1,
+      name: "lookup_benchmark",
+      type: "tool",
+      agent: "researcher",
+      members,
+      failedCount: 12,
+      p50Duration: 640,
+      isFailureGroup: true,
+      expanded: false,
+    };
+    renderPane(groupRow);
+    const pane = screen.getByRole("complementary", { name: "Group details" });
+    expect(pane).toHaveTextContent("lookup_benchmark ×12");
+    expect(pane).toHaveTextContent("Invocations12");
+    expect(pane).toHaveTextContent("Failed12");
+    expect(pane).toHaveTextContent("TimeoutError('slow')");
+  });
+
+  it("'Copy step' copies a curl for just this span as Markdown", async () => {
+    const user = userEvent.setup();
+    const writeText = vi.fn().mockResolvedValue(undefined);
+    Object.defineProperty(navigator, "clipboard", { value: { writeText }, configurable: true });
+    renderPane(spanRow(llm));
+    await user.click(screen.getByRole("button", { name: "Copy step" }));
+    await waitFor(() => expect(writeText).toHaveBeenCalled());
+    expect(writeText.mock.calls[0][0]).toContain("http://proxy.test/v1/traces/t1?format=md&span_id=llm1");
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx

@@ -0,0 +1,188 @@
+"use client";
+
+import { PanelRightClose } from "lucide-react";
+import { useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { cn } from "@/lib/cva.config";
+
+import { AttributesDetail } from "./AttributesDetail";
+import { CopyButton } from "./CopyButton";
+import { DetailContent, errorHeadline, useSpanDetail } from "./DetailContent";
+import { RequestDetail } from "./RequestDetail";
+import { agentHandoffText } from "./TraceDrawer";
+import type { GroupRowData, TreeRow } from "./traceTree";
+import type { Span, Trace } from "./traceTypes";
+import { fmtMs, fmtTok } from "./traceUtils";
+
+interface DetailPaneProps {
+  trace: Trace;
+  row: TreeRow | undefined;
+  accessToken: string;
+  onClose: () => void;
+}
+
+type Tab = "content" | "request" | "attributes";
+
+const TABS: { id: Tab; label: string }[] = [
+  { id: "content", label: "Content" },
+  { id: "request", label: "Request" },
+  { id: "attributes", label: "Attributes" },
+];
+
+function PaneHeader({ children, onClose }: { children: React.ReactNode; onClose: () => void }) {
+  return (
+    <div className="flex h-10 shrink-0 items-center gap-2 border-b border-border bg-card px-3">
+      {children}
+      <Button variant="ghost" size="icon-xs" onClick={onClose} aria-label="Close details" className="ml-auto">
+        <PanelRightClose className="size-3.5" />
+      </Button>
+    </div>
+  );
+}
+
+function PaneFooter({ children }: { children: React.ReactNode }) {
+  return <div className="flex h-9 shrink-0 items-center gap-2 border-t border-border bg-card px-3">{children}</div>;
+}
+
+function Meta({ label, value }: { label: string; value: string }) {
+  return (
+    <span>
+      <span className="text-muted-foreground/70">{label}=</span>
+      {value}
+    </span>
+  );
+}
+
+function SpanPane({
+  trace,
+  span,
+  accessToken,
+  onClose,
+}: {
+  trace: Trace;
+  span: Span;
+  accessToken: string;
+  onClose: () => void;
+}) {
+  const [tab, setTab] = useState<Tab>("content");
+  const traceId = trace.summary.trace_id;
+  const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null);
+  const tokens = span.input_tokens + span.output_tokens;
+  return (
+    <aside className="flex h-full min-w-0 flex-col bg-background" aria-label="Span details">
+      <PaneHeader onClose={onClose}>
+        <span
+          className={cn("size-1.5 rounded-full", span.status === "error" ? "bg-destructive" : "bg-muted-foreground/60")}
+        />
+        <div className="min-w-0 flex-1">
+          <div className="truncate font-mono text-[12px] font-medium text-foreground">
+            {span.type === "llm" ? span.model || span.name : span.name}
+          </div>
+          <div className="truncate font-mono text-[9px] text-muted-foreground">{span.span_id}</div>
+        </div>
+        <CopyButton value={span.span_id} label="Copy span ID" iconOnly />
+      </PaneHeader>
+      <div role="tablist" className="flex h-8 shrink-0 items-end gap-1 border-b border-border bg-card px-3">
+        {TABS.map((t) => (
+          <button
+            key={t.id}
+            type="button"
+            role="tab"
+            aria-selected={tab === t.id}
+            onClick={() => setTab(t.id)}
+            className={cn(
+              "-mb-px h-8 border-b-2 px-2 font-mono text-[10px]",
+              tab === t.id
+                ? "border-foreground text-foreground"
+                : "border-transparent text-muted-foreground hover:text-foreground",
+            )}
+          >
+            {t.label}
+          </button>
+        ))}
+      </div>
+      <div className="min-h-0 flex-1 overflow-auto">
+        {tab === "content" && <DetailContent accessToken={accessToken} traceId={traceId} span={span} />}
+        {tab === "request" && (
+          <RequestDetail span={span} accessToken={accessToken} traceStartMs={Date.parse(trace.summary.start_time)} />
+        )}
+        {tab === "attributes" && (
+          <AttributesDetail
+            traceId={traceId}
+            span={span}
+            attributes={detailQuery.data?.attributes}
+            isLoading={detailQuery.isLoading}
+          />
+        )}
+      </div>
+      <PaneFooter>
+        <CopyButton value={agentHandoffText(traceId, span.span_id)} label="Copy step" copiedLabel="Command copied" />
+        <div className="ml-auto flex items-center gap-3 font-mono text-[10px] tabular-nums text-muted-foreground">
+          <Meta label="time" value={fmtMs(span.duration_ms)} />
+          {tokens > 0 && <Meta label="tokens" value={fmtTok(tokens)} />}
+        </div>
+      </PaneFooter>
+    </aside>
+  );
+}
+
+function GroupMetric({ label, value }: { label: string; value: string }) {
+  return (
+    <div className="border-r border-b border-border p-3">
+      <div className="font-mono text-[9px] tracking-[0.1em] text-muted-foreground uppercase">{label}</div>
+      <div className="mt-1 font-mono text-[12px] tabular-nums text-foreground">{value}</div>
+    </div>
+  );
+}
+
+/** ×N group: rollup of every invocation plus the first failure's message. */
+function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; onClose: () => void }) {
+  const tokens = row.members.reduce((sum, m) => sum + m.input_tokens + m.output_tokens, 0);
+  const firstFailure = row.members.find((m) => m.status === "error" && m.error);
+  return (
+    <aside className="flex h-full min-w-0 flex-col bg-background" aria-label="Group details">
+      <PaneHeader onClose={onClose}>
+        <span className={cn("size-1.5 rounded-full", row.failedCount ? "bg-destructive" : "bg-muted-foreground/60")} />
+        <span className="truncate font-mono text-[12px] font-medium text-foreground">
+          {row.name} <span className="text-muted-foreground">×{row.members.length}</span>
+        </span>
+      </PaneHeader>
+      <div className="min-h-0 flex-1 overflow-auto p-3">
+        <div className="grid grid-cols-2 overflow-hidden rounded border border-border bg-card">
+          <GroupMetric label="Invocations" value={row.members.length.toLocaleString()} />
+          <GroupMetric label="Failed" value={row.failedCount.toLocaleString()} />
+          <GroupMetric label="p50 latency" value={fmtMs(row.p50Duration)} />
+          <GroupMetric label="Tokens" value={fmtTok(tokens)} />
+          <GroupMetric label="Agent" value={row.agent || "—"} />
+          <GroupMetric label="Type" value={row.type} />
+        </div>
+        {firstFailure?.error && (
+          <section className="mt-3 rounded border border-destructive/30 p-3">
+            <div className="font-mono text-[9px] tracking-[0.1em] text-destructive uppercase">Failure pattern</div>
+            <p className="mt-2 font-mono text-[11px] leading-5 text-foreground">{errorHeadline(firstFailure.error)}</p>
+          </section>
+        )}
+      </div>
+      <PaneFooter>
+        <CopyButton
+          value={agentHandoffText(trace.summary.trace_id, (firstFailure ?? row.members[0]).span_id)}
+          label="Copy group sample"
+        />
+      </PaneFooter>
+    </aside>
+  );
+}
+
+/** Right pane of the run view: switches on the selected tree row. */
+export function DetailPane({ trace, row, accessToken, onClose }: DetailPaneProps) {
+  if (!row || row.kind === "load-more") {
+    return (
+      <div className="grid h-full place-items-center bg-background font-mono text-[11px] text-muted-foreground">
+        Select a span to inspect it.
+      </div>
+    );
+  }
+  if (row.kind === "group") return <GroupPane key={row.id} trace={trace} row={row} onClose={onClose} />;
+  return <SpanPane key={row.id} trace={trace} span={row.span} accessToken={accessToken} onClose={onClose} />;
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/DurationBar.tsx

@@ -0,0 +1,27 @@
+import { cn } from "@/lib/cva.config";
+
+interface DurationBarProps {
+  startMs: number;
+  durationMs: number;
+  totalMs: number;
+  error?: boolean;
+  className?: string;
+}
+
+/** Waterfall bar: a hairline track with the span's slice of the run's timeline. */
+export function DurationBar({ startMs, durationMs, totalMs, error = false, className }: DurationBarProps) {
+  const left = totalMs > 0 ? Math.max(0, Math.min(98, (startMs / totalMs) * 100)) : 0;
+  const width = totalMs > 0 ? Math.max(1.5, Math.min(100 - left, (durationMs / totalMs) * 100)) : 1.5;
+  return (
+    <div className={cn("relative h-3 w-full overflow-hidden", className)} aria-hidden="true">
+      <div className="absolute top-[5px] left-0 h-px w-full bg-border" />
+      <div
+        className={cn(
+          "absolute top-[3px] h-[5px] min-w-[2px] rounded-[1px]",
+          error ? "bg-destructive/70" : "bg-muted-foreground/60",
+        )}
+        style={{ left: `${left}%`, width: `${width}%` }}
+      />
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/RequestDetail.tsx

@@ -0,0 +1,91 @@
+"use client";
+
+import { ArrowUpRight } from "lucide-react";
+import { useState } from "react";
+
+import { Button } from "@/components/ui/button";
+
+import { LogDetailsDrawer } from "../LogDetailsDrawer";
+import { CopyButton } from "./CopyButton";
+import type { Span } from "./traceTypes";
+import { fmtMs, fmtTok } from "./traceUtils";
+import { useSpanRequestLog } from "./useSpanRequestLog";
+
+interface RequestDetailProps {
+  span: Span;
+  accessToken: string;
+  /** The run's start, so the request log is looked up at the span's time, whatever range the tabs show. */
+  traceStartMs: number;
+}
+
+/** Request tab for LLM spans; "Open request log" opens the LiteLLM request drawer over the run. */
+export function RequestDetail({ span, accessToken, traceStartMs }: RequestDetailProps) {
+  const [drawerOpen, setDrawerOpen] = useState(false);
+  const spanStartMs = traceStartMs + span.start_offset_ms;
+  const logQuery = useSpanRequestLog(accessToken, span.litellm_request_id, spanStartMs, drawerOpen);
+  const lookupDone = drawerOpen && logQuery.isSuccess;
+  const logNotFound = lookupDone && logQuery.data === null;
+
+  if (span.type !== "llm") {
+    return (
+      <div className="py-16 text-center font-mono text-[11px] text-muted-foreground">
+        This span is not a model request.
+      </div>
+    );
+  }
+
+  const rows: [string, string][] = [
+    ["Model", span.model ?? "—"],
+    ["Input tokens", fmtTok(span.input_tokens)],
+    ["Output tokens", fmtTok(span.output_tokens)],
+    ["Total tokens", fmtTok(span.input_tokens + span.output_tokens)],
+    ["Latency", fmtMs(span.duration_ms)],
+  ];
+
+  return (
+    <div className="p-3">
+      <dl className="overflow-hidden rounded border border-border bg-card">
+        {rows.map(([label, value]) => (
+          <div key={label} className="grid min-h-8 grid-cols-[128px_1fr] border-b border-border last:border-0">
+            <dt className="flex items-center bg-muted/40 px-2.5 font-mono text-[10px] text-muted-foreground">
+              {label}
+            </dt>
+            <dd className="flex min-w-0 items-center px-2.5 font-mono text-[11px] tabular-nums text-foreground">
+              {value}
+            </dd>
+          </div>
+        ))}
+      </dl>
+      <div className="mt-3 rounded border border-border bg-muted/40 p-3">
+        <div className="flex items-center font-mono text-[9px] tracking-[0.1em] text-muted-foreground uppercase">
+          Request ID
+          {span.litellm_request_id && (
+            <CopyButton value={span.litellm_request_id} label="Copy request ID" iconOnly className="ml-auto" />
+          )}
+        </div>
+        <div className="mt-1.5 font-mono text-[11px] break-all text-foreground">
+          {span.litellm_request_id ?? "Not linked to a LiteLLM request"}
+        </div>
+      </div>
+      {span.litellm_request_id && (
+        <Button
+          variant="outline"
+          size="sm"
+          className="mt-3 w-full gap-1.5 font-mono text-[11px]"
+          onClick={() => setDrawerOpen(true)}
+        >
+          Open request log <ArrowUpRight className="size-3" />
+        </Button>
+      )}
+      {logNotFound && (
+        <p className="mt-2 font-mono text-[11px] text-muted-foreground">No request log found for this call.</p>
+      )}
+      <LogDetailsDrawer
+        open={drawerOpen && Boolean(logQuery.data)}
+        onClose={() => setDrawerOpen(false)}
+        logEntry={logQuery.data ?? null}
+        accessToken={accessToken}
+      />
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/RunsToolbar.tsx

@@ -0,0 +1,92 @@
+"use client";
+
+import { Search } from "lucide-react";
+
+import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+
+export type RunStatusFilter = "all" | "ok" | "error";
+
+export const ALL_SERVICES = "all";
+
+interface RunsToolbarProps {
+  query: string;
+  service: string;
+  status: RunStatusFilter;
+  services: string[];
+  onQueryChange: (value: string) => void;
+  onServiceChange: (value: string) => void;
+  onStatusChange: (value: RunStatusFilter) => void;
+  /** Extra controls (time range, live tail) rendered on the right. */
+  children?: React.ReactNode;
+}
+
+const STATUS_ITEMS: { value: RunStatusFilter; label: string }[] = [
+  { value: "all", label: "All status" },
+  { value: "ok", label: "Succeeded" },
+  { value: "error", label: "Failed" },
+];
+
+/** Search + service / status filters for the Runs table. Filtering is client-side over the loaded page. */
+export function RunsToolbar({
+  query,
+  service,
+  status,
+  services,
+  onQueryChange,
+  onServiceChange,
+  onStatusChange,
+  children,
+}: RunsToolbarProps) {
+  const serviceItems = [
+    { value: ALL_SERVICES, label: "All services" },
+    ...services.map((s) => ({ value: s, label: s })),
+  ];
+  return (
+    <div className="flex min-h-11 shrink-0 flex-wrap items-center gap-2 border-b border-border bg-card px-3 py-2">
+      <div className="relative w-full max-w-[380px]">
+        <Search className="pointer-events-none absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted-foreground" />
+        <Input
+          value={query}
+          onChange={(e) => onQueryChange(e.target.value)}
+          placeholder="Search input or trace ID"
+          aria-label="Search runs"
+          className="h-7 pl-8 text-[12px]"
+        />
+      </div>
+      <Select
+        items={serviceItems}
+        value={service}
+        onValueChange={(value: string | null) => value !== null && onServiceChange(value)}
+      >
+        <SelectTrigger size="sm" className="h-7 min-w-[130px] text-[12px]" aria-label="Filter by service">
+          <SelectValue />
+        </SelectTrigger>
+        <SelectContent>
+          {serviceItems.map((item) => (
+            <SelectItem key={item.value} value={item.value}>
+              {item.label}
+            </SelectItem>
+          ))}
+        </SelectContent>
+      </Select>
+      <Select
+        items={STATUS_ITEMS}
+        value={status}
+        onValueChange={(value: RunStatusFilter | null) => value !== null && onStatusChange(value)}
+      >
+        <SelectTrigger size="sm" className="h-7 min-w-[110px] text-[12px]" aria-label="Filter by status">
+          <SelectValue />
+        </SelectTrigger>
+        <SelectContent>
+          {STATUS_ITEMS.map((item) => (
+            <SelectItem key={item.value} value={item.value}>
+              {item.label}
+            </SelectItem>
+          ))}
+        </SelectContent>
+      </Select>
+      {children && <div className="ml-auto flex items-center gap-2">{children}</div>}
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx

@@ -0,0 +1,267 @@
+"use client";
+
+import {
+  Bot,
+  BrainCircuit,
+  ChevronDown,
+  ChevronRight,
+  CircleDot,
+  CornerDownRight,
+  MoreHorizontal,
+  Network,
+  Wrench,
+} from "lucide-react";
+import { useEffect, useRef } from "react";
+
+import { Switch } from "@/components/ui/switch";
+import { cn } from "@/lib/cva.config";
+
+import { DurationBar } from "./DurationBar";
+import type { TreeRow } from "./traceTree";
+import type { SpanType } from "./traceTypes";
+import { fmtMs } from "./traceUtils";
+
+interface SpanTreeProps {
+  rows: TreeRow[];
+  spanCount: number;
+  totalMs: number;
+  selectedId: string;
+  hideFramework: boolean;
+  onSelect: (id: string) => void;
+  onToggleHideFramework: (checked: boolean) => void;
+  onToggleSpan: (id: string) => void;
+  onToggleGroup: (id: string) => void;
+  onLoadMore: (groupId: string) => void;
+}
+
+const ROW_GRID = "grid h-8 w-full grid-cols-[minmax(275px,1fr)_minmax(94px,27%)_64px] items-center px-2";
+const INDENT_PX = 15;
+
+const rowClass = (selected: boolean): string =>
+  cn(
+    ROW_GRID,
+    "border-b border-border/60 text-left outline-none focus-visible:ring-1 focus-visible:ring-ring focus-visible:ring-inset",
+    selected ? "bg-accent shadow-[inset_2px_0_0_var(--foreground)]" : "hover:bg-muted/60",
+  );
+
+/** Span list with a framework toggle, a timeline column and keyboard hints. */
+export function SpanTree({
+  rows,
+  spanCount,
+  totalMs,
+  selectedId,
+  hideFramework,
+  onSelect,
+  onToggleHideFramework,
+  onToggleSpan,
+  onToggleGroup,
+  onLoadMore,
+}: SpanTreeProps) {
+  const scrollRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    scrollRef.current
+      ?.querySelector(`[data-row-id="${CSS.escape(selectedId)}"]`)
+      ?.scrollIntoView?.({ block: "nearest" });
+  }, [selectedId]);
+
+  return (
+    <section className="flex h-full min-h-0 min-w-0 flex-col border-r border-border bg-card" aria-label="Run spans">
+      <div className="flex h-10 shrink-0 items-center gap-2 border-b border-border bg-muted/40 px-2.5">
+        <span className="shrink-0 font-mono text-[9px] text-muted-foreground tabular-nums">
+          {`${spanCount.toLocaleString()} spans`}
+        </span>
+        <label className="ml-auto flex shrink-0 cursor-pointer items-center gap-2 font-mono text-[9px] text-muted-foreground">
+          hide framework
+          <Switch
+            size="sm"
+            checked={hideFramework}
+            onCheckedChange={(checked) => onToggleHideFramework(checked)}
+            aria-label="Hide framework spans"
+          />
+        </label>
+      </div>
+      <div
+        className={cn(
+          ROW_GRID,
+          "h-7 shrink-0 border-b border-border bg-muted/60 font-mono text-[8px] tracking-[0.1em] text-muted-foreground uppercase",
+        )}
+      >
+        <span className="pl-6">Span</span>
+        <span>Timeline</span>
+        <span className="text-right">Time</span>
+      </div>
+      <div ref={scrollRef} className="min-h-0 flex-1 overflow-auto" role="tree" aria-label="Spans in time order">
+        {rows.map((row) => (
+          <TreeRowItem
+            key={row.id}
+            row={row}
+            selected={selectedId === row.id}
+            totalMs={totalMs}
+            onSelect={onSelect}
+            onToggleSpan={onToggleSpan}
+            onToggleGroup={onToggleGroup}
+            onLoadMore={onLoadMore}
+          />
+        ))}
+      </div>
+      <div className="flex h-7 shrink-0 items-center gap-3 border-t border-border bg-muted/60 px-3 font-mono text-[8px] text-muted-foreground">
+        <span>
+          <Kbd>J</Kbd>/<Kbd>K</Kbd> move
+        </span>
+        <span>
+          <Kbd>←</Kbd>/<Kbd>→</Kbd> fold
+        </span>
+        <span>
+          <Kbd>Esc</Kbd> close
+        </span>
+      </div>
+    </section>
+  );
+}
+
+function Kbd({ children }: { children: React.ReactNode }) {
+  return (
+    <kbd className="rounded-[3px] border border-border border-b-2 bg-muted px-[3px] font-mono text-muted-foreground">
+      {children}
+    </kbd>
+  );
+}
+
+interface TreeRowItemProps {
+  row: TreeRow;
+  selected: boolean;
+  totalMs: number;
+  onSelect: (id: string) => void;
+  onToggleSpan: (id: string) => void;
+  onToggleGroup: (id: string) => void;
+  onLoadMore: (groupId: string) => void;
+}
+
+function TreeRowItem({ row, selected, totalMs, onSelect, onToggleSpan, onToggleGroup, onLoadMore }: TreeRowItemProps) {
+  if (row.kind === "load-more") {
+    return (
+      <button
+        type="button"
+        onClick={() => onLoadMore(row.groupId)}
+        className="flex h-8 w-full items-center gap-2 border-b border-border/60 px-2 font-mono text-[9px] text-foreground hover:bg-muted/60"
+        style={{ paddingLeft: `${18 + row.depth * INDENT_PX}px` }}
+      >
+        <MoreHorizontal className="size-3" /> load 20 more{" "}
+        <span className="text-muted-foreground">({row.remaining} remaining)</span>
+      </button>
+    );
+  }
+
+  if (row.kind === "group") {
+    const start = Math.min(...row.members.map((m) => m.start_offset_ms));
+    const end = Math.max(...row.members.map((m) => m.start_offset_ms + m.duration_ms));
+    return (
+      <button
+        type="button"
+        role="treeitem"
+        aria-expanded={row.expanded}
+        aria-selected={selected}
+        data-row-id={row.id}
+        onClick={() => {
+          onSelect(row.id);
+          onToggleGroup(row.id);
+        }}
+        className={rowClass(selected)}
+      >
+        <div className="flex min-w-0 items-center" style={{ paddingLeft: `${row.depth * INDENT_PX}px` }}>
+          {row.expanded ? (
+            <ChevronDown className="mr-0.5 size-3 shrink-0 text-muted-foreground" />
+          ) : (
+            <ChevronRight className="mr-0.5 size-3 shrink-0 text-muted-foreground" />
+          )}
+          <TypeIcon type={row.type} error={row.failedCount > 0} />
+          <span
+            className={cn(
+              "ml-1.5 truncate font-mono text-[10px]",
+              row.isFailureGroup ? "text-destructive" : "text-foreground",
+            )}
+          >
+            {row.name}
+          </span>
+          <span className="ml-1 font-mono text-[9px] text-muted-foreground">×{row.members.length}</span>
+          <span className="ml-2 hidden truncate font-mono text-[8px] text-muted-foreground xl:block">
+            p50={fmtMs(row.p50Duration)}
+          </span>
+          {row.failedCount > 0 && (
+            <span className="ml-2 shrink-0 font-mono text-[8px] text-destructive">{row.failedCount} failed</span>
+          )}
+        </div>
+        <DurationBar startMs={start} durationMs={end - start} totalMs={totalMs} error={row.isFailureGroup} />
+        <span className="text-right font-mono text-[9px] text-muted-foreground tabular-nums">
+          {fmtMs(row.p50Duration)}
+        </span>
+      </button>
+    );
+  }
+
+  const { span } = row;
+  const failed = span.status === "error";
+  return (
+    <button
+      type="button"
+      role="treeitem"
+      aria-expanded={row.hasChildren ? !row.collapsed : undefined}
+      aria-selected={selected}
+      data-row-id={row.id}
+      onClick={() => onSelect(row.id)}
+      onDoubleClick={() => row.hasChildren && onToggleSpan(row.id)}
+      className={rowClass(selected)}
+    >
+      <div className="flex min-w-0 items-center" style={{ paddingLeft: `${row.depth * INDENT_PX}px` }}>
+        {row.hasChildren ? (
+          <span
+            role="button"
+            tabIndex={-1}
+            aria-label={row.collapsed ? "Expand" : "Collapse"}
+            className="mr-0.5 shrink-0"
+            onClick={(event) => {
+              event.stopPropagation();
+              onToggleSpan(row.id);
+            }}
+          >
+            {row.collapsed ? (
+              <ChevronRight className="size-3 text-muted-foreground" />
+            ) : (
+              <ChevronDown className="size-3 text-muted-foreground" />
+            )}
+          </span>
+        ) : (
+          <CornerDownRight className="mr-0.5 size-3 shrink-0 text-muted-foreground/50" />
+        )}
+        <TypeIcon type={span.type} error={failed} />
+        <span className={cn("ml-1.5 truncate font-mono text-[10px]", failed ? "text-destructive" : "text-foreground")}>
+          {span.name}
+        </span>
+        {span.model && (
+          <span className="ml-2 hidden truncate font-mono text-[8px] text-muted-foreground 2xl:block">
+            {span.model.split("/").pop()}
+          </span>
+        )}
+      </div>
+      <DurationBar startMs={span.start_offset_ms} durationMs={span.duration_ms} totalMs={totalMs} error={failed} />
+      <span
+        className={cn(
+          "text-right font-mono text-[9px] tabular-nums",
+          failed ? "text-destructive" : "text-muted-foreground",
+        )}
+      >
+        {fmtMs(span.duration_ms)}
+      </span>
+    </button>
+  );
+}
+
+function TypeIcon({ type, error }: { type: SpanType; error: boolean }) {
+  const className = cn("size-3 shrink-0", error ? "text-destructive" : "text-muted-foreground");
+  if (type === "agent") return <Bot className={className} />;
+  if (type === "llm") return <BrainCircuit className={className} />;
+  if (type === "tool") return <Wrench className={className} />;
+  if (type === "framework") return <Network className={className} />;
+  return <CircleDot className={className} />;
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/StatusMark.tsx

@@ -0,0 +1,27 @@
+import { AlertCircle, Check, Circle } from "lucide-react";
+
+interface StatusMarkProps {
+  status: "ok" | "error" | "unset";
+  count?: number;
+  subtle?: boolean;
+}
+
+/** Run / span status glyph: red alert (+ count) for errors, a quiet muted dot or check otherwise. */
+export function StatusMark({ status, count, subtle = false }: StatusMarkProps) {
+  if (status === "error") {
+    return (
+      <span
+        className="inline-flex items-center gap-1 text-[11px] text-destructive"
+        aria-label={`${count ?? 1} error${count === 1 ? "" : "s"}`}
+      >
+        <AlertCircle className="size-3" />
+        {count !== undefined && <span className="font-mono tabular-nums">{count}</span>}
+      </span>
+    );
+  }
+  if (subtle)
+    return (
+      <Circle className="size-2 shrink-0 fill-muted-foreground/60 text-muted-foreground/60" aria-label="Success" />
+    );
+  return <Check className="size-3 text-muted-foreground" aria-label="Success" />;
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx

@@ -0,0 +1,101 @@
+"use client";
+
+import { ChevronDown, Pause, Play, RotateCcw } from "lucide-react";
+import moment from "moment";
+
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuRadioGroup,
+  DropdownMenuRadioItem,
+  DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import { cn } from "@/lib/cva.config";
+
+import type { TimeWindow } from "./TracesTimeline";
+
+export const RANGE_PRESETS = [
+  { hours: 1, label: "Last hour" },
+  { hours: 6, label: "Last 6 hours" },
+  { hours: 24, label: "Last 24 hours" },
+  { hours: 168, label: "Last 7 days" },
+  { hours: 720, label: "Last 30 days" },
+] as const;
+
+const RANGE_LABEL_FORMAT = "MMM D, h:mm A";
+
+export const rangeLabel = (range: TimeWindow): string =>
+  `${moment(range.startMs).format(RANGE_LABEL_FORMAT)} to ${moment(range.endMs).format(RANGE_LABEL_FORMAT)}`;
+
+const SEGMENT = "inline-flex h-7 items-center gap-1.5 px-2.5 text-[13px] outline-none focus-visible:bg-accent";
+
+interface TimeRangeControlsProps {
+  range: TimeWindow;
+  rangeHours: number;
+  onRangeHoursChange: (hours: number) => void;
+  live: boolean;
+  onLiveChange: (live: boolean) => void;
+  zoomed: boolean;
+  onResetZoom: () => void;
+}
+
+/** Joined control group: reset zoom, the actual time range (opens presets), and Live. */
+export function TimeRangeControls({
+  range,
+  rangeHours,
+  onRangeHoursChange,
+  live,
+  onLiveChange,
+  zoomed,
+  onResetZoom,
+}: TimeRangeControlsProps) {
+  return (
+    <div className="flex items-center gap-1.5">
+      <button
+        type="button"
+        onClick={onResetZoom}
+        disabled={!zoomed}
+        aria-label="Reset zoom"
+        className="inline-flex size-7 items-center justify-center rounded-md border border-border text-info hover:bg-info/10 disabled:text-muted-foreground disabled:opacity-40 disabled:hover:bg-transparent"
+      >
+        <RotateCcw className="size-3.5" />
+      </button>
+      <div className="flex items-center divide-x divide-border overflow-hidden rounded-md border border-border bg-card">
+        <DropdownMenu>
+          <DropdownMenuTrigger
+            aria-label="Time range"
+            className={cn(SEGMENT, "text-foreground hover:bg-muted/60")}
+            data-testid="time-range-trigger"
+          >
+            <span className="tabular-nums">{rangeLabel(range)}</span>
+            <ChevronDown className="size-3.5 text-muted-foreground" />
+          </DropdownMenuTrigger>
+          <DropdownMenuContent align="end" className="w-auto min-w-44">
+            <DropdownMenuRadioGroup
+              value={String(rangeHours)}
+              onValueChange={(value: string) => onRangeHoursChange(Number(value))}
+            >
+              {RANGE_PRESETS.map((preset) => (
+                <DropdownMenuRadioItem key={preset.hours} value={String(preset.hours)} className="text-[13px]">
+                  {preset.label}
+                </DropdownMenuRadioItem>
+              ))}
+            </DropdownMenuRadioGroup>
+          </DropdownMenuContent>
+        </DropdownMenu>
+        <button
+          type="button"
+          aria-pressed={live}
+          onClick={() => onLiveChange(!live)}
+          className={cn(
+            SEGMENT,
+            live ? "bg-info/10 text-info hover:bg-info/15" : "text-muted-foreground hover:text-foreground",
+          )}
+        >
+          {live ? <Pause className="size-3.5" /> : <Play className="size-3.5" />}
+          Live
+        </button>
+      </div>
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx

@@ -0,0 +1,146 @@
+import { screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils";
+import researchTrace from "./__fixtures__/research_trace.json";
+import swarmTrace from "./__fixtures__/swarm_trace.json";
+import { agentHandoffText, initialRunSelection, RunView } from "./TraceDrawer";
+import type { Span } from "./traceTypes";
+import type { Trace } from "./traceTypes";
+import { traceDisplayName } from "./traceUtils";
+
+vi.mock("../../networking", () => ({
+  agentTraceCall: vi.fn(),
+  agentTraceSpanCall: vi.fn(),
+  getProxyBaseUrl: () => "http://proxy.test/",
+}));
+
+// DetailPane is built separately; render a stub that exposes which row is selected.
+vi.mock("./DetailPane", () => ({
+  DetailPane: ({ row, onClose }: { row?: { id: string }; onClose: () => void }) => (
+    <div data-testid="detail-pane" data-row-id={row?.id}>
+      <button type="button" onClick={onClose}>
+        close detail
+      </button>
+    </div>
+  ),
+}));
+
+vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn().mockResolvedValue(true) }));
+
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import { agentTraceCall } from "../../networking";
+
+const swarm = swarmTrace as Trace;
+const research = researchTrace as Trace;
+
+const renderRun = (trace: Trace) => {
+  vi.mocked(agentTraceCall).mockResolvedValue(trace);
+  return renderWithProviders(<RunView traceId={trace.summary.trace_id} accessToken="sk-test" onBack={vi.fn()} />);
+};
+
+const rootSpanId = (trace: Trace): string => trace.spans.find((s) => s.parent_span_id === null)?.span_id ?? "";
+
+describe("RunView", () => {
+  beforeEach(() => {
+    testQueryClient.clear();
+    vi.mocked(copyToClipboard).mockClear();
+  });
+
+  it("shows a one-line run header: agent name, trace id, duration and steps", async () => {
+    renderRun(research);
+
+    const header = await screen.findByRole("banner");
+    expect(screen.getByRole("heading", { level: 1 })).toHaveTextContent(traceDisplayName(research.summary));
+    expect(header).toHaveTextContent(research.summary.trace_id);
+    expect(header).toHaveTextContent("duration 40.20s");
+    expect(header).toHaveTextContent(`steps ${research.summary.span_count}`);
+    expect(header).not.toHaveTextContent("failed");
+  });
+
+  it("folds researcher ×12 in the span tree", async () => {
+    renderRun(swarm);
+
+    const tree = await screen.findByRole("tree", { name: "Spans in time order" });
+    expect(tree).toHaveTextContent("researcher×12");
+    expect(screen.getByRole("banner")).toHaveTextContent(`failed ${swarm.summary.error_count}`);
+  });
+
+  it("opens a failed run on its first failed span", async () => {
+    renderRun(swarm);
+
+    const pane = await screen.findByTestId("detail-pane");
+    const { selectedId } = initialRunSelection(swarm);
+    expect(selectedId).not.toBe(rootSpanId(swarm));
+    expect(pane).toHaveAttribute("data-row-id", selectedId);
+    expect(swarm.spans.find((s) => s.span_id === selectedId)?.status).toBe("error");
+  });
+
+  it("opens a healthy run on the root span", async () => {
+    renderRun(research);
+    expect(await screen.findByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research));
+  });
+
+  it("moves the selection with J / K and closes the detail pane with Esc", async () => {
+    const user = userEvent.setup();
+    renderRun(research);
+
+    const pane = await screen.findByTestId("detail-pane");
+    const root = rootSpanId(research);
+    expect(pane).toHaveAttribute("data-row-id", root);
+    await user.keyboard("j");
+    expect(screen.getByTestId("detail-pane").getAttribute("data-row-id")).not.toBe(root);
+    await user.keyboard("k");
+    expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", root);
+    await user.keyboard("{Escape}");
+    expect(screen.queryByTestId("detail-pane")).not.toBeInTheDocument();
+  });
+
+  it("keeps a way back to the runs table when a run fails to load", async () => {
+    const user = userEvent.setup();
+    const onBack = vi.fn();
+    vi.mocked(agentTraceCall).mockRejectedValue(new Error("trace exceeds the 1000 span read limit"));
+    renderWithProviders(<RunView traceId="big" accessToken="sk-test" onBack={onBack} />);
+
+    expect(await screen.findByText("trace exceeds the 1000 span read limit")).toBeInTheDocument();
+    await user.click(screen.getByRole("button", { name: /back to traces/i }));
+    expect(onBack).toHaveBeenCalledTimes(1);
+  });
+
+  it("copies a curl one-liner for Claude / Codex", async () => {
+    const user = userEvent.setup();
+    renderRun(research);
+
+    await user.click(await screen.findByRole("button", { name: /copy for agent/i }));
+    expect(copyToClipboard).toHaveBeenCalledWith(agentHandoffText(research.summary.trace_id), "Command copied");
+    expect(agentHandoffText("t1")).toContain('"http://proxy.test/v1/traces/t1?format=md"');
+    expect(agentHandoffText("t1", "s1")).toContain("&span_id=s1");
+  });
+});
+
+describe("initialRunSelection", () => {
+  const base = research.spans.find((s) => s.parent_span_id === null) as Span;
+  const child = (over: Partial<Span>): Span => {
+    const defaults: Partial<Span> = { parent_span_id: base.span_id, status: "ok" };
+    return { ...base, ...defaults, ...over };
+  };
+
+  it("lands on a visible failure, never on a framework span the tree hides", () => {
+    const hiddenFields: Partial<Span> = { span_id: "mw", type: "framework", status: "error", start_offset_ms: 1 };
+    const toolFields: Partial<Span> = { span_id: "tool", type: "tool", status: "error", start_offset_ms: 5 };
+    const hiddenFailure = child(hiddenFields);
+    const toolFailure = child(toolFields);
+    const trace = { ...research, spans: [base, hiddenFailure, toolFailure] };
+    expect(initialRunSelection(trace).selectedId).toBe("tool");
+  });
+
+  it("falls back to the nearest visible ancestor when only a hidden span failed", () => {
+    const agent = child({ span_id: "agent", type: "agent", name: "researcher" });
+    const hiddenFields: Partial<Span> = { span_id: "mw", parent_span_id: "agent", type: "framework", status: "error" };
+    const hiddenFailure = child(hiddenFields);
+    const trace = { ...research, spans: [base, agent, hiddenFailure] };
+    expect(initialRunSelection(trace).selectedId).toBe("agent");
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx

@@ -0,0 +1,269 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { ArrowLeft, Check, Circle, Copy } from "lucide-react";
+import { useCallback, useEffect, useMemo, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import { agentTraceCall, getProxyBaseUrl } from "../../networking";
+import { DetailPane } from "./DetailPane";
+import { SpanTree } from "./SpanTree";
+import type { SpanTreeState, TreeRow } from "./traceTree";
+import type { Trace } from "./traceTypes";
+import {
+  buildTreeRows,
+  firstErrorSpan,
+  fmtMs,
+  GROUP_PAGE_SIZE,
+  isFrameworkSpan,
+  nearestVisibleSpanId,
+  revealSpanInState,
+  traceDisplayName,
+} from "./traceUtils";
+
+/** What "Copy for agent" puts on the clipboard: a one-liner Claude Code / Codex can run. */
+export const agentHandoffText = (traceId: string, spanId?: string | null): string => {
+  const url = `${getProxyBaseUrl().replace(/\/$/, "")}/v1/traces/${traceId}?format=md${spanId ? `&span_id=${spanId}` : ""}`;
+  const what = spanId ? "this step of a LiteLLM agent trace" : "this LiteLLM agent trace";
+  return `Read ${what} and explain what happened and why it failed:\ncurl -s -H "Authorization: Bearer $LITELLM_API_KEY" "${url}"`;
+};
+
+const INITIAL_STATE: SpanTreeState = {
+  hideFramework: true,
+  collapsedSpanIds: new Set(),
+  expandedGroupIds: new Set(),
+  groupRevealCounts: {},
+};
+
+/** First failed span if the run has errors (with its tree path opened), otherwise the root agent. */
+export function initialRunSelection(trace: Trace): { selectedId: string; state: SpanTreeState } {
+  const failed = firstErrorSpan(trace.spans);
+  if (!failed || failed.parent_span_id === null) {
+    const root = trace.spans.find((s) => s.parent_span_id === null);
+    return { selectedId: root?.span_id ?? "", state: INITIAL_STATE };
+  }
+  const visibleFailure = trace.spans
+    .filter((s) => s.status === "error" && s.parent_span_id !== null && !isFrameworkSpan(s))
+    .sort((a, b) => a.start_offset_ms - b.start_offset_ms)[0];
+  const selectedId = visibleFailure?.span_id ?? nearestVisibleSpanId(trace.spans, failed.span_id, true);
+  return { selectedId, state: revealSpanInState(trace.spans, INITIAL_STATE, selectedId) };
+}
+
+const toggle = (set: ReadonlySet<string>, id: string): Set<string> => {
+  const next = new Set(set);
+  if (next.has(id)) next.delete(id);
+  else next.add(id);
+  return next;
+};
+
+function CopyForAgent({ traceId }: { traceId: string }) {
+  const [copied, setCopied] = useState(false);
+  useEffect(() => {
+    if (!copied) return;
+    const timeout = window.setTimeout(() => setCopied(false), 1600);
+    return () => window.clearTimeout(timeout);
+  }, [copied]);
+  return (
+    <Button
+      variant="outline"
+      size="xs"
+      className="shrink-0 gap-1.5 rounded-[4px] font-mono text-[10px] shadow-none"
+      onClick={async () => setCopied(await copyToClipboard(agentHandoffText(traceId), "Command copied"))}
+    >
+      {copied ? <Check className="size-3" /> : <Copy className="size-3" />}
+      {copied ? "Command copied" : "Copy for agent"}
+    </Button>
+  );
+}
+
+function Stat({ label, value, error = false }: { label: string; value: string; error?: boolean }) {
+  return (
+    <span className={cn("shrink-0", error && "text-destructive")}>
+      <span className="text-muted-foreground/70">{label} </span>
+      {value}
+    </span>
+  );
+}
+
+function RunHeader({ trace, onBack }: { trace: Trace; onBack: () => void }) {
+  const { summary } = trace;
+  const failed = summary.error_count > 0;
+  return (
+    <header className="flex h-11 shrink-0 items-center gap-3 border-b border-border bg-card px-2">
+      <button
+        type="button"
+        onClick={onBack}
+        className="grid size-7 shrink-0 place-items-center rounded-[4px] text-muted-foreground hover:bg-muted hover:text-foreground"
+        aria-label="Back to runs"
+      >
+        <ArrowLeft className="size-4" />
+      </button>
+      <Circle className={cn("size-2 shrink-0 fill-current", failed ? "text-destructive" : "text-muted-foreground")} />
+      <h1 className="shrink-0 truncate text-[14px] font-medium text-foreground">{traceDisplayName(summary)}</h1>
+      <span className="flex min-w-0 items-center gap-1 font-mono text-[11px] text-muted-foreground">
+        <span className="truncate">{summary.trace_id}</span>
+        <button
+          type="button"
+          aria-label="Copy trace ID"
+          className="shrink-0 hover:text-foreground"
+          onClick={() => void copyToClipboard(summary.trace_id, "Trace ID copied")}
+        >
+          <Copy className="size-3" />
+        </button>
+      </span>
+      <div className="flex min-w-0 items-center gap-4 font-mono text-[11px] text-foreground tabular-nums">
+        <Stat label="duration" value={fmtMs(summary.duration_ms)} />
+        <Stat label="steps" value={summary.span_count.toLocaleString()} />
+        {failed && <Stat label="failed" value={summary.error_count.toLocaleString()} error />}
+      </div>
+      <div className="ml-auto">
+        <CopyForAgent traceId={summary.trace_id} />
+      </div>
+    </header>
+  );
+}
+
+/** Tree + detail pane for one loaded run, with J/K/arrow keyboard navigation. */
+function RunBody({ trace, accessToken }: { trace: Trace; accessToken: string }) {
+  const initial = useMemo(() => initialRunSelection(trace), [trace]);
+  const [state, setState] = useState<SpanTreeState>(initial.state);
+  const [selectedId, setSelectedId] = useState<string>(initial.selectedId);
+  const [detailOpen, setDetailOpen] = useState(true);
+
+  const rows = useMemo(() => buildTreeRows(trace.spans, state), [trace, state]);
+  const selectedRow: TreeRow | undefined = rows.find((row) => row.id === selectedId) ?? rows[0];
+
+  const select = useCallback((id: string) => {
+    setSelectedId(id);
+    setDetailOpen(true);
+  }, []);
+  const toggleSpan = useCallback(
+    (id: string) => setState((prev) => ({ ...prev, collapsedSpanIds: toggle(prev.collapsedSpanIds, id) })),
+    [],
+  );
+  const toggleGroup = useCallback(
+    (id: string) => setState((prev) => ({ ...prev, expandedGroupIds: toggle(prev.expandedGroupIds, id) })),
+    [],
+  );
+  const loadMore = useCallback(
+    (groupId: string) =>
+      setState((prev) => ({
+        ...prev,
+        groupRevealCounts: {
+          ...prev.groupRevealCounts,
+          [groupId]: (prev.groupRevealCounts[groupId] ?? GROUP_PAGE_SIZE) + GROUP_PAGE_SIZE,
+        },
+      })),
+    [],
+  );
+
+  useEffect(() => {
+    const onKeyDown = (event: KeyboardEvent) => {
+      if ((event.target as HTMLElement | null)?.matches("input, textarea, [role='combobox']")) return;
+      const index = rows.findIndex((row) => row.id === selectedRow?.id);
+      const row = rows[index];
+      if (event.key === "Escape" && detailOpen) {
+        event.preventDefault();
+        event.stopPropagation();
+        setDetailOpen(false);
+        return;
+      }
+      if (["j", "J", "ArrowDown"].includes(event.key)) {
+        event.preventDefault();
+        const next = rows[Math.min(rows.length - 1, index + 1)];
+        if (next) select(next.id);
+      } else if (["k", "K", "ArrowUp"].includes(event.key)) {
+        event.preventDefault();
+        const next = rows[Math.max(0, index - 1)];
+        if (next) select(next.id);
+      } else if (event.key === "ArrowLeft" && row) {
+        if (row.kind === "span" && row.hasChildren && !row.collapsed) toggleSpan(row.id);
+        if (row.kind === "group" && row.expanded) toggleGroup(row.id);
+      } else if (event.key === "ArrowRight" && row) {
+        if (row.kind === "span" && row.hasChildren && row.collapsed) toggleSpan(row.id);
+        if (row.kind === "group" && !row.expanded) toggleGroup(row.id);
+      }
+    };
+    window.addEventListener("keydown", onKeyDown, true);
+    return () => window.removeEventListener("keydown", onKeyDown, true);
+  }, [rows, selectedRow, detailOpen, select, toggleSpan, toggleGroup]);
+
+  return (
+    <div
+      className={cn(
+        "grid min-h-0 flex-1",
+        detailOpen
+          ? "grid-cols-1 grid-rows-2 lg:grid-cols-[minmax(0,3fr)_minmax(360px,2fr)] lg:grid-rows-1 2xl:grid-cols-[minmax(0,2fr)_minmax(420px,1fr)]"
+          : "grid-cols-1",
+      )}
+    >
+      <SpanTree
+        rows={rows}
+        spanCount={trace.summary.span_count}
+        totalMs={trace.summary.duration_ms}
+        selectedId={selectedRow?.id ?? selectedId}
+        hideFramework={state.hideFramework}
+        onSelect={select}
+        onToggleHideFramework={(hideFramework) => setState((prev) => ({ ...prev, hideFramework }))}
+        onToggleSpan={toggleSpan}
+        onToggleGroup={toggleGroup}
+        onLoadMore={loadMore}
+      />
+      {detailOpen && (
+        <DetailPane trace={trace} row={selectedRow} accessToken={accessToken} onClose={() => setDetailOpen(false)} />
+      )}
+    </div>
+  );
+}
+
+interface RunViewProps {
+  traceId: string;
+  accessToken: string;
+  onBack: () => void;
+}
+
+/** One agent run: header with totals and "Copy for agent", span tree on the left, span details on the right. */
+export function RunView({ traceId, accessToken, onBack }: RunViewProps) {
+  const traceQuery = useQuery({
+    queryKey: ["agentTrace", traceId, accessToken],
+    queryFn: () => agentTraceCall(accessToken, traceId),
+    staleTime: 30_000,
+  });
+  const trace = traceQuery.data;
+
+  if (traceQuery.isLoading) {
+    return (
+      <div role="status" aria-label="Loading trace" className="flex h-[60vh] items-center justify-center">
+        <UiLoadingSpinner className="size-6 text-muted-foreground" />
+      </div>
+    );
+  }
+  if (traceQuery.isError || !trace) {
+    return (
+      <div className="p-6 text-[12px]" data-testid="run-view-error">
+        <button
+          type="button"
+          onClick={onBack}
+          className="mb-4 inline-flex items-center gap-1.5 text-[12px] text-muted-foreground hover:text-foreground"
+        >
+          <ArrowLeft className="size-3.5" /> Back to traces
+        </button>
+        <h1 className="mb-2 text-[13px] font-medium">Could not load trace</h1>
+        <span className="text-muted-foreground">{traceQuery.error?.message ?? "Unknown error"}</span>
+      </div>
+    );
+  }
+  return (
+    <div
+      className="flex min-h-[560px] flex-1 flex-col overflow-hidden border-y border-border bg-background"
+      data-testid="run-view"
+    >
+      <RunHeader trace={trace} onBack={onBack} />
+      <RunBody key={trace.summary.trace_id} trace={trace} accessToken={accessToken} />
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+
+import { bandForWindow, bucketRuns, dragUpdate, formatSpan } from "./TracesTimeline";
+import type { TraceSummary } from "./traceTypes";
+
+const HOUR = 3600 * 1000;
+const START = Date.UTC(2026, 8, 30, 0, 0, 0);
+const range = { startMs: START, endMs: START + 10 * HOUR };
+
+const run = (offsetMs: number, errorCount = 0): TraceSummary =>
+  ({
+    trace_id: `t${offsetMs}`,
+    start_time: new Date(START + offsetMs).toISOString(),
+    error_count: errorCount,
+  }) as TraceSummary;
+
+describe("bucketRuns", () => {
+  it("splits the window into equal buckets that tile it exactly", () => {
+    const buckets = bucketRuns([], range, 10);
+    expect(buckets).toHaveLength(10);
+    expect(buckets[0].startMs).toBe(range.startMs);
+    expect(buckets.at(-1)?.endMs).toBe(range.endMs);
+    const fourthBucket = { startMs: START + 3 * HOUR, endMs: START + 4 * HOUR, runs: 0, failed: 0 };
+    expect(buckets[3]).toMatchObject(fourthBucket);
+  });
+
+  it("puts each run in the bucket covering its start time", () => {
+    const buckets = bucketRuns([run(0), run(30 * 60 * 1000), run(2.5 * HOUR), run(10 * HOUR - 1)], range, 10);
+    expect(buckets.map((b) => b.runs)).toEqual([2, 0, 1, 0, 0, 0, 0, 0, 0, 1]);
+  });
+
+  it("drops runs that start before or at/after the window", () => {
+    const buckets = bucketRuns([run(-1), run(10 * HOUR), run(20 * HOUR), run(HOUR)], range, 10);
+    expect(buckets.reduce((sum, b) => sum + b.runs, 0)).toBe(1);
+    expect(buckets[1].runs).toBe(1);
+  });
+
+  it("counts runs with any errors as failed", () => {
+    const buckets = bucketRuns([run(HOUR, 3), run(HOUR + 1), run(HOUR + 2, 1), run(5 * HOUR)], range, 10);
+    expect(buckets[1]).toMatchObject({ runs: 3, failed: 2 });
+    expect(buckets[5]).toMatchObject({ runs: 1, failed: 0 });
+  });
+});
+
+describe("formatSpan", () => {
+  it("prints the largest two units, dropping zero parts", () => {
+    expect(formatSpan(45 * 60 * 1000)).toBe("45m");
+    expect(formatSpan(6 * HOUR + 12 * 60 * 1000)).toBe("6h 12m");
+    expect(formatSpan(24 * HOUR)).toBe("1d");
+    expect(formatSpan(152 * 24 * HOUR + 23 * HOUR)).toBe("152d 23h");
+    expect(formatSpan(-5)).toBe("0m");
+  });
+});
+
+describe("dragUpdate", () => {
+  const band = { lo: 10, hi: 14 };
+
+  it("selects between the press point and the pointer, in either direction", () => {
+    expect(dragUpdate({ mode: "select", origin: 20, band: { lo: 20, hi: 20 } }, 25)).toEqual({ lo: 20, hi: 25 });
+    expect(dragUpdate({ mode: "select", origin: 20, band: { lo: 20, hi: 20 } }, 12)).toEqual({ lo: 12, hi: 20 });
+  });
+
+  it("resizes one edge without letting it cross the other", () => {
+    expect(dragUpdate({ mode: "resize-lo", origin: 10, band }, 4)).toEqual({ lo: 4, hi: 14 });
+    expect(dragUpdate({ mode: "resize-lo", origin: 10, band }, 30)).toEqual({ lo: 14, hi: 14 });
+    expect(dragUpdate({ mode: "resize-hi", origin: 14, band }, 40)).toEqual({ lo: 10, hi: 40 });
+    expect(dragUpdate({ mode: "resize-hi", origin: 14, band }, 2)).toEqual({ lo: 10, hi: 10 });
+  });
+
+  it("pans the band keeping its width, clamped to the strip", () => {
+    expect(dragUpdate({ mode: "move", origin: 12, band }, 20)).toEqual({ lo: 18, hi: 22 });
+    expect(dragUpdate({ mode: "move", origin: 12, band }, -50)).toEqual({ lo: 0, hi: 4 });
+    expect(dragUpdate({ mode: "move", origin: 12, band }, 500, 60)).toEqual({ lo: 55, hi: 59 });
+  });
+});
+
+describe("bandForWindow", () => {
+  it("maps a selected window back to the buckets it covers", () => {
+    const buckets = bucketRuns([], range, 10);
+    expect(bandForWindow(buckets, { startMs: START + 2 * HOUR, endMs: START + 5 * HOUR })).toEqual({ lo: 2, hi: 4 });
+    expect(bandForWindow(buckets, null)).toBeNull();
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/TracesTimeline.tsx

@@ -0,0 +1,365 @@
+"use client";
+
+import moment from "moment";
+import { useEffect, useMemo, useRef, useState } from "react";
+
+import { cn } from "@/lib/cva.config";
+
+import type { TraceSummary } from "./traceTypes";
+
+const BUCKETS = 60;
+const TICKS = 6;
+const MINUTE_MS = 60 * 1000;
+const HOUR_MS = 60 * MINUTE_MS;
+const DAY_MS = 24 * HOUR_MS;
+const EDGE_FORMAT = "MMM DD, HH:mm";
+
+export interface TimeWindow {
+  startMs: number;
+  endMs: number;
+}
+
+export interface Bucket {
+  startMs: number;
+  endMs: number;
+  runs: number;
+  failed: number;
+}
+
+/** Run counts per equal-width time bucket across the window; runs outside it are dropped. */
+export function bucketRuns(runs: readonly TraceSummary[], range: TimeWindow, buckets = BUCKETS): Bucket[] {
+  const width = (range.endMs - range.startMs) / buckets;
+  const placed = runs.map((run) => ({
+    index: Math.floor((moment(run.start_time).valueOf() - range.startMs) / width),
+    failed: run.error_count > 0,
+  }));
+  return Array.from({ length: buckets }, (_, i) => {
+    const hits = placed.filter((p) => p.index === i);
+    return {
+      startMs: range.startMs + i * width,
+      endMs: range.startMs + (i + 1) * width,
+      runs: hits.length,
+      failed: hits.filter((p) => p.failed).length,
+    };
+  });
+}
+
+/** Compact window length, Logfire-style: "45m", "6h 12m", "7d", "152d 23h". */
+export function formatSpan(ms: number): string {
+  const totalMinutes = Math.max(0, Math.round(ms / MINUTE_MS));
+  const days = Math.floor(totalMinutes / (24 * 60));
+  const hours = Math.floor((totalMinutes % (24 * 60)) / 60);
+  const minutes = totalMinutes % 60;
+  if (days > 0) return hours > 0 ? `${days}d ${hours}h` : `${days}d`;
+  if (hours > 0) return minutes > 0 ? `${hours}h ${minutes}m` : `${hours}h`;
+  return `${minutes}m`;
+}
+
+const tickFormat = (range: TimeWindow): string => (range.endMs - range.startMs > 2 * DAY_MS ? EDGE_FORMAT : "HH:mm");
+
+const pct = (value: number): string => `${value * 100}%`;
+
+/** Keep the first / last tick label inside the strip; center the rest on their tick. */
+const tickShift = (t: number): string => {
+  if (t === 0) return "translateX(0)";
+  if (t === 1) return "translateX(-100%)";
+  return "translateX(-50%)";
+};
+
+interface TracesTimelineProps {
+  runs: readonly TraceSummary[];
+  range: TimeWindow;
+  selection: TimeWindow | null;
+  onSelect: (selection: TimeWindow | null) => void;
+}
+
+function BucketBar({
+  bucket,
+  max,
+  dimmed,
+  hovered,
+}: {
+  bucket: Bucket;
+  max: number;
+  dimmed: boolean;
+  hovered: boolean;
+}) {
+  return (
+    <div
+      className="pointer-events-none flex h-full flex-1 flex-col justify-end px-px"
+      data-testid="timeline-bucket"
+      data-runs={bucket.runs}
+    >
+      {bucket.runs > 0 && (
+        <div
+          className={cn(
+            "mx-auto flex w-full max-w-[10px] flex-col overflow-hidden rounded-t-[2px]",
+            dimmed && "bg-muted-foreground/25",
+            !dimmed && (hovered ? "bg-info" : "bg-info/65"),
+          )}
+          style={{ height: `${Math.max(6, (bucket.runs / max) * 100)}%` }}
+        >
+          {bucket.failed > 0 && (
+            <div
+              className="w-full shrink-0 bg-destructive"
+              style={{ height: `${Math.max(2, Math.min(25, (bucket.failed / bucket.runs) * 100))}%` }}
+            />
+          )}
+        </div>
+      )}
+    </div>
+  );
+}
+
+function BucketTooltip({ bucket, index }: { bucket: Bucket; index: number }) {
+  return (
+    <div
+      className="pointer-events-none absolute top-full z-floating mt-1 rounded-md border border-border bg-popover px-2.5 py-1.5 font-mono text-[11px] text-popover-foreground shadow-md"
+      style={{ left: pct(Math.min(0.8, index / BUCKETS)) }}
+      role="tooltip"
+    >
+      <div>
+        {moment(bucket.startMs).format(EDGE_FORMAT)} to {moment(bucket.endMs).format("HH:mm")}
+      </div>
+      <div>
+        {bucket.runs} {bucket.runs === 1 ? "run" : "runs"}
+        {bucket.failed > 0 && `, ${bucket.failed} failed`}
+      </div>
+      <div className="text-info">drag to zoom</div>
+    </div>
+  );
+}
+
+const MIN_DURATION_LABEL_PX = 120;
+
+/** Bucket-index band [lo, hi], inclusive. */
+export interface Band {
+  lo: number;
+  hi: number;
+}
+
+export type DragMode = "select" | "move" | "resize-lo" | "resize-hi";
+
+export interface DragState {
+  mode: DragMode;
+  origin: number;
+  band: Band;
+}
+
+/** The band a drag produces when the pointer is over bucket `at`; always within [0, buckets - 1]. */
+export function dragUpdate(drag: DragState, at: number, buckets = BUCKETS): Band {
+  const last = buckets - 1;
+  const clamp = (i: number) => Math.min(last, Math.max(0, i));
+  const { band, origin } = drag;
+  if (drag.mode === "select") return { lo: Math.min(origin, clamp(at)), hi: Math.max(origin, clamp(at)) };
+  if (drag.mode === "resize-lo") return { lo: Math.min(clamp(at), band.hi), hi: band.hi };
+  if (drag.mode === "resize-hi") return { lo: band.lo, hi: Math.max(clamp(at), band.lo) };
+  const width = band.hi - band.lo;
+  const lo = Math.min(last - width, Math.max(0, band.lo + (at - origin)));
+  return { lo, hi: lo + width };
+}
+
+/** Bucket band covered by a selected window, or null when nothing is selected. */
+export function bandForWindow(buckets: readonly Bucket[], selection: TimeWindow | null): Band | null {
+  if (selection === null) return null;
+  const inside = buckets.flatMap((b, i) => (b.startMs >= selection.startMs && b.endMs <= selection.endMs ? [i] : []));
+  return inside.length > 0 ? { lo: inside[0], hi: inside[inside.length - 1] } : null;
+}
+
+const edgeFormat = (range: TimeWindow): string => (range.endMs - range.startMs < DAY_MS ? "HH:mm" : EDGE_FORMAT);
+
+/** The selected window drawn as a flat bracket: resize handles on each edge, times outside, span centered. */
+function SelectionBracket({
+  band,
+  window,
+  format,
+  stripWidth,
+  onHandleDown,
+}: {
+  band: Band;
+  window: TimeWindow;
+  format: string;
+  stripWidth: number;
+  onHandleDown: (mode: DragMode) => (e: React.PointerEvent) => void;
+}) {
+  const leftFrac = band.lo / BUCKETS;
+  const rightFrac = (band.hi + 1) / BUCKETS;
+  const widthPx = (rightFrac - leftFrac) * stripWidth;
+  const handle = "absolute inset-y-0 w-[3px] cursor-ew-resize bg-info";
+  const edgeLabel =
+    "pointer-events-none absolute -top-4 font-mono text-[10px] whitespace-nowrap text-info tabular-nums";
+  return (
+    <>
+      <div
+        className="absolute inset-y-0 cursor-grab rounded-[2px] border border-info bg-info/10 active:cursor-grabbing"
+        style={{ left: pct(leftFrac), width: pct(rightFrac - leftFrac) }}
+        data-testid="timeline-selection"
+        onPointerDown={onHandleDown("move")}
+      >
+        <span
+          className={cn(handle, "-left-px")}
+          data-testid="timeline-handle-lo"
+          onPointerDown={onHandleDown("resize-lo")}
+        />
+        <span
+          className={cn(handle, "-right-px")}
+          data-testid="timeline-handle-hi"
+          onPointerDown={onHandleDown("resize-hi")}
+        />
+        {widthPx >= MIN_DURATION_LABEL_PX && (
+          <span className="pointer-events-none absolute inset-x-0 top-1/2 -translate-y-1/2 text-center font-mono text-[11px] font-medium text-info">
+            {formatSpan(window.endMs - window.startMs)}
+          </span>
+        )}
+      </div>
+      <span className={cn(edgeLabel, leftFrac < 0.12 ? "" : "-translate-x-full pr-1")} style={{ left: pct(leftFrac) }}>
+        {moment(window.startMs).format(format)}
+      </span>
+      <span className={cn(edgeLabel, rightFrac > 0.88 ? "-translate-x-full" : "pl-1")} style={{ left: pct(rightFrac) }}>
+        {moment(window.endMs).format(format)}
+      </span>
+    </>
+  );
+}
+
+function TickAxis({ range }: { range: TimeWindow }) {
+  const format = tickFormat(range);
+  const ticks = Array.from({ length: TICKS }, (_, i) => i / (TICKS - 1));
+  return (
+    <div className="relative mt-0.5 h-5">
+      {ticks.map((t) => (
+        <div
+          key={t}
+          className={cn(
+            "absolute top-0 flex flex-col",
+            t === 0 && "items-start",
+            t === 1 && "items-end",
+            t > 0 && t < 1 && "items-center",
+          )}
+          style={{ left: pct(t), transform: tickShift(t) }}
+        >
+          <span className="h-1 w-px bg-border" />
+          <span className="font-mono text-[10px] whitespace-nowrap text-muted-foreground tabular-nums">
+            {moment(range.startMs + (range.endMs - range.startMs) * t).format(format)}
+          </span>
+        </div>
+      ))}
+    </div>
+  );
+}
+
+/** Histogram of runs over the window. Drag to select; drag the bracket or its edges to adjust; Esc clears. */
+export function TracesTimeline({ runs, range, selection, onSelect }: TracesTimelineProps) {
+  const buckets = useMemo(() => bucketRuns(runs, range), [runs, range]);
+  const max = Math.max(1, ...buckets.map((b) => b.runs));
+  const [hover, setHover] = useState<number | null>(null);
+  const [drag, setDrag] = useState<DragState | null>(null);
+  const [draft, setDraft] = useState<Band | null>(null);
+  const areaRef = useRef<HTMLDivElement>(null);
+  const [stripWidth, setStripWidth] = useState(0);
+
+  useEffect(() => {
+    const el = areaRef.current;
+    if (!el) return;
+    const measure = () => setStripWidth(el.getBoundingClientRect().width);
+    measure();
+    if (typeof ResizeObserver === "undefined") return;
+    const observer = new ResizeObserver(measure);
+    observer.observe(el);
+    return () => observer.disconnect();
+  }, []);
+
+  const committed = bandForWindow(buckets, selection);
+  const band = drag ? draft : committed;
+  const windowOf = (b: Band): TimeWindow => ({ startMs: buckets[b.lo].startMs, endMs: buckets[b.hi].endMs });
+
+  const indexAt = (clientX: number): number => {
+    const rect = areaRef.current?.getBoundingClientRect();
+    if (!rect || rect.width === 0) return 0;
+    return Math.min(BUCKETS - 1, Math.max(0, Math.floor(((clientX - rect.left) / rect.width) * BUCKETS)));
+  };
+  const begin = (mode: DragMode, e: React.PointerEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    areaRef.current?.setPointerCapture?.(e.pointerId);
+    const at = indexAt(e.clientX);
+    const start = mode === "select" || committed === null ? { lo: at, hi: at } : committed;
+    setDrag({ mode: committed === null ? "select" : mode, origin: at, band: start });
+    setDraft(start);
+  };
+  const onHandleDown = (mode: DragMode) => (e: React.PointerEvent) => begin(mode, e);
+  const onMove = (e: React.PointerEvent) => {
+    const at = indexAt(e.clientX);
+    setHover(at);
+    if (drag) setDraft(dragUpdate(drag, at));
+  };
+  const onUp = (e: React.PointerEvent) => {
+    areaRef.current?.releasePointerCapture?.(e.pointerId);
+    if (!drag || !draft) return;
+    const finished = draft;
+    const wasSelect = drag.mode === "select";
+    setDrag(null);
+    setDraft(null);
+    if (wasSelect && finished.lo === finished.hi) {
+      onSelect(null);
+      return;
+    }
+    onSelect(windowOf(finished));
+  };
+  const onKeyDown = (e: React.KeyboardEvent) => {
+    if (e.key !== "Escape" || selection === null) return;
+    e.preventDefault();
+    onSelect(null);
+  };
+
+  const isDimmed = (i: number): boolean => band !== null && (i < band.lo || i > band.hi);
+  const labelFormat = edgeFormat(range);
+
+  return (
+    <div
+      className="relative shrink-0 border-b border-border bg-card px-3 pt-2 pb-1 outline-none select-none"
+      data-testid="traces-timeline"
+      tabIndex={0}
+      onKeyDown={onKeyDown}
+    >
+      <div className="mb-4 flex items-center">
+        <span className="rounded border border-border bg-muted/50 px-1.5 py-px font-mono text-[10px] text-muted-foreground">
+          Total {formatSpan(range.endMs - range.startMs)}
+        </span>
+      </div>
+      <div
+        ref={areaRef}
+        role="presentation"
+        data-testid="timeline-area"
+        className="relative flex h-14 cursor-crosshair touch-none items-end border-b border-border"
+        onPointerDown={(e) => begin("select", e)}
+        onPointerMove={onMove}
+        onPointerUp={onUp}
+        onPointerLeave={() => setHover(null)}
+        onDoubleClick={() => onSelect(null)}
+      >
+        {hover !== null && !drag && (
+          <div
+            className="pointer-events-none absolute inset-y-0 w-px bg-info"
+            style={{ left: pct((hover + 0.5) / BUCKETS) }}
+            data-testid="timeline-cursor"
+          />
+        )}
+        {buckets.map((b, i) => (
+          <BucketBar key={b.startMs} bucket={b} max={max} dimmed={isDimmed(i)} hovered={hover === i} />
+        ))}
+        {band && (
+          <SelectionBracket
+            band={band}
+            window={windowOf(band)}
+            format={labelFormat}
+            stripWidth={stripWidth}
+            onHandleDown={onHandleDown}
+          />
+        )}
+      </div>
+      <TickAxis range={range} />
+      {hover !== null && !drag && <BucketTooltip bucket={buckets[hover]} index={hover} />}
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx

@@ -0,0 +1,57 @@
+import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { describe, expect, it, vi } from "vitest";
+
+import { codingAgentPrompt, tracingEnvSnippet, TracingSetupCard } from "./TracingSetupCard";
+
+vi.mock("../../networking", () => ({ getProxyBaseUrl: () => "http://proxy.test/" }));
+vi.mock("@/utils/dataUtils", () => ({ copyToClipboard: vi.fn().mockResolvedValue(true) }));
+
+describe("TracingSetupCard", () => {
+  it("shows the waiting state and OTEL-only setup when tracing is on", () => {
+    render(<TracingSetupCard detail={null} />);
+
+    const card = screen.getByTestId("tracing-setup-card");
+    expect(card).toHaveTextContent("Waiting for traces");
+    expect(card).toHaveTextContent("No traces detected yet. Follow our guide to start tracing your application.");
+    expect(card).toHaveTextContent("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test");
+    expect(card).not.toHaveTextContent(/langsmith/i);
+    expect(card).not.toHaveTextContent("store: clickhouse");
+  });
+
+  it("swaps the install command and prompt when another framework is picked", async () => {
+    const user = userEvent.setup();
+    render(<TracingSetupCard detail={null} />);
+
+    expect(screen.getByTestId("tracing-setup-card")).toHaveTextContent("openinference-instrumentation-langchain");
+    await user.click(screen.getByRole("radio", { name: "CrewAI" }));
+
+    const card = screen.getByTestId("tracing-setup-card");
+    expect(screen.getByRole("radio", { name: "CrewAI" })).toBeChecked();
+    expect(card).toHaveTextContent("pip install -U opentelemetry-distro");
+    expect(card).toHaveTextContent("crewai openinference-instrumentation-crewai");
+    expect(card).toHaveTextContent("Send this CrewAI project's OpenTelemetry traces to LiteLLM.");
+    expect(card).not.toHaveTextContent("openinference-instrumentation-langchain");
+  });
+
+  it("shows the proxy config step only when tracing is not enabled", () => {
+    render(<TracingSetupCard detail="Agent tracing is not enabled" />);
+    const card = screen.getByTestId("tracing-setup-card");
+    expect(card).toHaveTextContent("Tracing is not enabled");
+    expect(card).toHaveTextContent("store: clickhouse");
+  });
+});
+
+describe("setup snippets", () => {
+  it("points OTLP at the proxy base URL and reads the key from the environment", () => {
+    const env = tracingEnvSnippet("http://proxy.test");
+    expect(env).toContain("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test\n");
+    expect(env).not.toContain("/v1/traces");
+    expect(env).toContain("Bearer $LITELLM_API_KEY");
+
+    const prompt = codingAgentPrompt("http://proxy.test", { label: "LangChain", packages: "langchain" });
+    expect(prompt).toContain("base_url=http://proxy.test/v1");
+    expect(prompt).toContain("opentelemetry-distro opentelemetry-exporter-otlp-proto-http langchain");
+    expect(prompt).not.toMatch(/langsmith/i);
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx

@@ -0,0 +1,390 @@
+"use client";
+
+import { ArrowUpRight, Check, Copy, Loader2 } from "lucide-react";
+import { useState } from "react";
+
+import { cn } from "@/lib/cva.config";
+import { copyToClipboard } from "@/utils/dataUtils";
+
+import previewImg from "../../../../public/assets/agent-traces-preview.png";
+import crewaiLogo from "../../../../public/assets/logos/crewai-color.svg";
+import langchainLogo from "../../../../public/assets/logos/langchain.svg";
+import langgraphLogo from "../../../../public/assets/logos/langgraph-color.svg";
+import llamaindexLogo from "../../../../public/assets/logos/llamaindex-color.svg";
+import openaiAgentsLogo from "../../../../public/assets/logos/openai-agents.svg";
+import otelLogo from "../../../../public/assets/logos/opentelemetry.svg";
+import pydanticAiLogo from "../../../../public/assets/logos/pydantic-ai-color.svg";
+import { getProxyBaseUrl } from "../../networking";
+
+const COPIED_RESET_MS = 1500;
+const DOCS_URL = "https://docs.litellm.ai";
+const OTEL_BASE_PACKAGES = "opentelemetry-distro opentelemetry-exporter-otlp-proto-http";
+const RUN_SNIPPET = "opentelemetry-instrument python my_agent.py";
+
+type Installer = "pip" | "uv";
+
+interface FrameworkGuide {
+  id: string;
+  label: string;
+  logo: string;
+  packages: string;
+  quickstart: string;
+}
+
+const FRAMEWORKS: readonly FrameworkGuide[] = [
+  {
+    id: "langgraph",
+    label: "LangGraph / Deep Agents",
+    logo: langgraphLogo.src,
+    packages: "langgraph langchain-openai openinference-instrumentation-langchain",
+    quickstart: `from langchain.agents import create_agent
+from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = create_agent(model=llm, tools=[], name="my_agent")
+agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`,
+  },
+  {
+    id: "langchain",
+    label: "LangChain",
+    logo: langchainLogo.src,
+    packages: "langchain langchain-openai openinference-instrumentation-langchain",
+    quickstart: `from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+llm.invoke("What is LiteLLM?")`,
+  },
+  {
+    id: "openai-agents",
+    label: "OpenAI Agents SDK",
+    logo: openaiAgentsLogo.src,
+    packages: "openai-agents openinference-instrumentation-openai-agents",
+    quickstart: `from agents import Agent, OpenAIChatCompletionsModel, Runner
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(name="my_agent", model=OpenAIChatCompletionsModel(model="claude-sonnet-4-5", openai_client=client))
+print(Runner.run_sync(agent, "What is LiteLLM?").final_output)`,
+  },
+  {
+    id: "crewai",
+    label: "CrewAI",
+    logo: crewaiLogo.src,
+    packages: "crewai openinference-instrumentation-crewai",
+    quickstart: `from crewai import LLM, Agent, Crew, Task
+
+llm = LLM(model="openai/claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(role="Researcher", goal="Answer questions", backstory="", llm=llm)
+task = Task(description="What is LiteLLM?", expected_output="A short answer", agent=agent)
+Crew(agents=[agent], tasks=[task]).kickoff()`,
+  },
+  {
+    id: "pydantic-ai",
+    label: "Pydantic AI",
+    logo: pydanticAiLogo.src,
+    packages: "pydantic-ai openinference-instrumentation-pydantic-ai",
+    quickstart: `from pydantic_ai import Agent
+from pydantic_ai.models.openai import OpenAIModel
+from pydantic_ai.providers.openai import OpenAIProvider
+
+provider = OpenAIProvider(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"])
+agent = Agent(OpenAIModel("claude-sonnet-4-5", provider=provider), name="my_agent", instrument=True)
+print(agent.run_sync("What is LiteLLM?").output)`,
+  },
+  {
+    id: "llamaindex",
+    label: "LlamaIndex",
+    logo: llamaindexLogo.src,
+    packages: "llama-index llama-index-llms-openai-like openinference-instrumentation-llama-index",
+    quickstart: `from llama_index.llms.openai_like import OpenAILike
+
+llm = OpenAILike(model="claude-sonnet-4-5", api_base="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"], is_chat_model=True)
+print(llm.complete("What is LiteLLM?"))`,
+  },
+  {
+    id: "otel",
+    label: "OpenTelemetry",
+    logo: otelLogo.src,
+    packages: "",
+    quickstart: `# Any OTEL SDK works. Use the gen_ai.* semantic conventions:
+# gen_ai.operation.name, gen_ai.agent.name, gen_ai.response.id, gen_ai.usage.*
+from opentelemetry import trace
+
+tracer = trace.get_tracer("my-agent")
+attrs = {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "my_agent"}
+with tracer.start_as_current_span("my_agent", attributes=attrs):
+    ...`,
+  },
+];
+
+const HIGHLIGHTS = [
+  ["Input and output", "What the agent was asked and what it answered, at the top of every run."],
+  ["Every step, nested", "LLM calls, tool calls and subagents in one tree, with timing."],
+  ["Failures pinpointed", "See whether the tool, the model or LiteLLM broke."],
+  ["Hand off to Claude / Codex", "Copy one command and your coding agent debugs the run."],
+] as const;
+
+const installPackages = (guide: Pick<FrameworkGuide, "packages">): string =>
+  [OTEL_BASE_PACKAGES, guide.packages].filter(Boolean).join(" ");
+
+/** The endpoint is the proxy base URL: OTLP exporters append /v1/traces themselves. */
+export const tracingEnvSnippet = (proxyUrl: string): string =>
+  [
+    `export OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`,
+    'export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"',
+    "export OTEL_SERVICE_NAME=my-agent",
+  ].join("\n");
+
+export const codingAgentPrompt = (proxyUrl: string, guide: Pick<FrameworkGuide, "label" | "packages">): string =>
+  [
+    `Send this ${guide.label} project's OpenTelemetry traces to LiteLLM.`,
+    "",
+    `1. Add these dependencies: ${installPackages(guide)}`,
+    "2. Set these env vars wherever the project loads config (.env, settings, deployment manifests):",
+    `   OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`,
+    '   OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"',
+    "   OTEL_SERVICE_NAME=<a short name for this agent>",
+    "3. Start the app through OTEL auto-instrumentation: opentelemetry-instrument <existing start command>.",
+    `4. Point every LLM client at LiteLLM: base_url=${proxyUrl}/v1, api key from LITELLM_API_KEY.`,
+    "5. Give each agent and subagent a name so runs are easy to read.",
+    "6. Run the agent once and confirm the run shows up in the LiteLLM UI under Logs > Agent Traces.",
+    "",
+    "Never hardcode the key. Read it from LITELLM_API_KEY.",
+  ].join("\n");
+
+export const PROXY_CONFIG_SNIPPET = [
+  "general_settings:",
+  "  tracing:",
+  "    store: clickhouse",
+  "",
+  "# env: CLICKHOUSE_URL (writer) and CLICKHOUSE_READER_URL (read-only user)",
+].join("\n");
+
+function CodeBlock({ code, tabs, wrap = false }: { code: string; tabs?: React.ReactNode; wrap?: boolean }) {
+  const [copied, setCopied] = useState(false);
+  const copy = async () => {
+    await copyToClipboard(code);
+    setCopied(true);
+    window.setTimeout(() => setCopied(false), COPIED_RESET_MS);
+  };
+  return (
+    <div className="overflow-hidden rounded-md border border-border bg-muted/30">
+      <div className="flex h-9 items-center border-b border-border px-3">
+        {tabs}
+        <button
+          type="button"
+          onClick={() => void copy()}
+          aria-label="Copy"
+          className="ml-auto text-muted-foreground hover:text-foreground"
+        >
+          {copied ? <Check className="size-3.5" /> : <Copy className="size-3.5" />}
+        </button>
+      </div>
+      <pre
+        className={cn(
+          "px-4 py-3 font-mono text-[12.5px] leading-6 text-foreground",
+          wrap ? "whitespace-pre-wrap" : "overflow-x-auto",
+        )}
+      >
+        <code>{code}</code>
+      </pre>
+    </div>
+  );
+}
+
+function FileLabel({ children }: { children: React.ReactNode }) {
+  return <span className="text-[12.5px] text-foreground">{children}</span>;
+}
+
+function LineTabs<T extends string>({
+  value,
+  options,
+  onChange,
+}: {
+  value: T;
+  options: T[];
+  onChange: (v: T) => void;
+}) {
+  return (
+    <div role="tablist" className="flex h-9 items-end gap-3">
+      {options.map((option) => (
+        <button
+          key={option}
+          type="button"
+          role="tab"
+          aria-selected={value === option}
+          onClick={() => onChange(option)}
+          className={cn(
+            "-mb-px h-9 border-b-2 text-[12.5px]",
+            value === option
+              ? "border-foreground text-foreground"
+              : "border-transparent text-muted-foreground hover:text-foreground",
+          )}
+        >
+          {option}
+        </button>
+      ))}
+    </div>
+  );
+}
+
+function Step({ title, children }: { title: string; children: React.ReactNode }) {
+  return (
+    <section className="mt-6">
+      <h3 className="mb-2 text-[13px] font-medium text-foreground">{title}</h3>
+      {children}
+    </section>
+  );
+}
+
+function SetupStatus({ detail, connected }: { detail: string | null; connected: boolean }) {
+  const pill = "inline-flex items-center gap-1.5 rounded-full bg-muted px-2.5 py-1 text-[12px] text-foreground";
+  const hint = "text-[13px] text-muted-foreground";
+  if (connected) {
+    return (
+      <>
+        <span className={pill}>
+          <Check className="size-3" /> Receiving traces
+        </span>
+        <span className={hint}>Add another agent: point its OpenTelemetry exporter at LiteLLM.</span>
+      </>
+    );
+  }
+  if (detail === null) {
+    return (
+      <>
+        <span className={pill}>
+          <Loader2 className="size-3 animate-spin" /> Waiting for traces…
+        </span>
+        <span className={hint}>No traces detected yet. Follow our guide to start tracing your application.</span>
+      </>
+    );
+  }
+  return (
+    <>
+      <span className={pill}>Tracing is not enabled</span>
+      <span className={hint}>Turn on tracing in the proxy config, then point your agent at LiteLLM.</span>
+    </>
+  );
+}
+
+function WhatYoullSee() {
+  return (
+    <section className="mt-6">
+      <h2 className="text-[15px] font-medium text-foreground">What you&apos;ll see</h2>
+      <div className="mt-3 overflow-hidden rounded-lg border border-border bg-background shadow-sm">
+        <img src={previewImg.src} alt="Preview of an agent trace in LiteLLM" className="block w-full" />
+      </div>
+      <div className="mt-4 grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
+        {HIGHLIGHTS.map(([title, body]) => (
+          <div key={title}>
+            <div className="text-[13px] font-medium text-foreground">{title}</div>
+            <div className="mt-0.5 text-[13px] text-muted-foreground">{body}</div>
+          </div>
+        ))}
+      </div>
+    </section>
+  );
+}
+
+function FrameworkPicker({ value, onChange }: { value: string; onChange: (id: string) => void }) {
+  return (
+    <div className="mt-4 flex flex-wrap gap-2" role="radiogroup" aria-label="Framework">
+      {FRAMEWORKS.map((f) => (
+        <button
+          key={f.id}
+          type="button"
+          role="radio"
+          aria-checked={value === f.id}
+          onClick={() => onChange(f.id)}
+          className={cn(
+            "inline-flex items-center gap-2 rounded-md border px-3 py-2 text-[13px]",
+            value === f.id
+              ? "border-foreground bg-accent text-foreground"
+              : "border-border text-muted-foreground hover:text-foreground",
+          )}
+        >
+          <img src={f.logo} alt="" className="size-4" />
+          {f.label}
+        </button>
+      ))}
+    </div>
+  );
+}
+
+/**
+ * Agent Traces onboarding: shown until the first trace arrives (and when tracing isn't enabled on the proxy).
+ * `detail` is the proxy's 501 message when tracing is off; null means tracing is on and we're waiting.
+ */
+export function TracingSetupCard({ detail, connected = false }: { detail: string | null; connected?: boolean }) {
+  const proxyUrl = getProxyBaseUrl().replace(/\/$/, "");
+  const [framework, setFramework] = useState(FRAMEWORKS[0].id);
+  const [installer, setInstaller] = useState<Installer>("pip");
+  const guide = FRAMEWORKS.find((f) => f.id === framework) ?? FRAMEWORKS[0];
+  const packages = installPackages(guide);
+  const install = installer === "pip" ? `pip install -U ${packages}` : `uv add ${packages}`;
+
+  return (
+    <div className="pb-12" data-testid="tracing-setup-card">
+      <div className="w-full rounded-lg border border-border bg-card p-6">
+        <div className="flex flex-wrap items-center gap-3">
+          <SetupStatus detail={detail} connected={connected} />
+          <a
+            href={DOCS_URL}
+            target="_blank"
+            rel="noreferrer"
+            className="ml-auto inline-flex items-center gap-1 text-[13px] text-foreground hover:underline"
+          >
+            View docs <ArrowUpRight className="size-3.5" />
+          </a>
+        </div>
+
+        <WhatYoullSee />
+
+        <div className="mt-10 max-w-[860px]">
+          <h2 className="text-[15px] font-medium text-foreground">
+            Send your agent&apos;s OpenTelemetry traces to LiteLLM
+          </h2>
+          <p className="mt-1 text-[13px] text-muted-foreground">
+            Standard OTLP. Pick your framework, set three env vars, and run your agent as usual.
+          </p>
+          <FrameworkPicker value={framework} onChange={setFramework} />
+
+          {detail !== null && (
+            <Step title="Enable tracing on the proxy">
+              <CodeBlock code={PROXY_CONFIG_SNIPPET} tabs={<FileLabel>config.yaml</FileLabel>} />
+            </Step>
+          )}
+
+          <Step title="Let Claude Code or Codex set it up">
+            <p className="mb-2 text-[13px] text-muted-foreground">
+              Paste this into your coding agent from the project root, or follow the steps below by hand.
+            </p>
+            <CodeBlock code={codingAgentPrompt(proxyUrl, guide)} wrap tabs={<FileLabel>Prompt</FileLabel>} />
+          </Step>
+
+          <Step title="Install dependencies">
+            <CodeBlock
+              code={install}
+              tabs={<LineTabs value={installer} options={["pip", "uv"]} onChange={setInstaller} />}
+            />
+          </Step>
+
+          <Step title="Configure environment">
+            <CodeBlock code={tracingEnvSnippet(proxyUrl)} tabs={<FileLabel>Shell</FileLabel>} />
+          </Step>
+
+          <Step title="Run your agent">
+            <CodeBlock
+              code={`import os\n\n${guide.quickstart.replace("{PROXY}", proxyUrl)}`}
+              tabs={<FileLabel>my_agent.py</FileLabel>}
+            />
+            <div className="mt-3">
+              <CodeBlock code={RUN_SNIPPET} tabs={<FileLabel>Shell</FileLabel>} />
+            </div>
+          </Step>
+        </div>
+      </div>
+    </div>
+  );
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json

@@ -0,0 +1,59 @@
+{
+  "data": [
+    {
+      "trace_id": "e309a123963901e74c29cd2d3c86ff9e",
+      "name": "research_lead",
+      "service": "research-agent",
+      "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]",
+      "start_time": "2026-09-30T06:43:54.291000+00:00",
+      "duration_ms": 40198.0,
+      "status": "ok",
+      "span_count": 216,
+      "agent_count": 6,
+      "llm_calls": 21,
+      "tool_calls": 25,
+      "error_count": 0,
+      "input_tokens": 69506,
+      "output_tokens": 2960,
+      "models": ["claude-sonnet-4-5"],
+      "agent_invocations": 6
+    },
+    {
+      "trace_id": "f78f6df35480060fafadac887e234241",
+      "name": "support_triage_agent",
+      "service": "research-agent",
+      "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-404 says billing is wrong. What plan are they on?\"}]",
+      "start_time": "2026-09-30T06:43:52.928000+00:00",
+      "duration_ms": 1315.0,
+      "status": "ok",
+      "span_count": 5,
+      "agent_count": 1,
+      "llm_calls": 1,
+      "tool_calls": 1,
+      "error_count": 2,
+      "input_tokens": 659,
+      "output_tokens": 60,
+      "models": ["claude-sonnet-4-5"],
+      "agent_invocations": 1
+    },
+    {
+      "trace_id": "71498cec128bbea430f01de04b972f36",
+      "name": "support_triage_agent",
+      "service": "research-agent",
+      "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-42 gets 429s after upgrading. What plan are they on and what should they check?\"}]",
+      "start_time": "2026-09-30T06:43:47.373000+00:00",
+      "duration_ms": 5551.0,
+      "status": "ok",
+      "span_count": 9,
+      "agent_count": 1,
+      "llm_calls": 2,
+      "tool_calls": 2,
+      "error_count": 0,
+      "input_tokens": 1552,
+      "output_tokens": 202,
+      "models": ["claude-sonnet-4-5"],
+      "agent_invocations": 1
+    }
+  ],
+  "next_cursor": null
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/traceTree.ts

@@ -0,0 +1,48 @@
+/**
+ * Shared contract for the agent run view (span tree + detail pane).
+ * Rows are produced by `buildTreeRows` in traceUtils.ts and rendered by SpanTree / DetailPane.
+ */
+import type { Span, SpanType } from "./traceTypes";
+
+export type ErrorSource = "model" | "tool" | "litellm";
+
+export interface SpanRowData {
+  kind: "span";
+  id: string;
+  span: Span;
+  depth: number;
+  hasChildren: boolean;
+  collapsed: boolean;
+}
+
+export interface GroupRowData {
+  kind: "group";
+  id: string;
+  depth: number;
+  name: string;
+  type: SpanType;
+  agent: string;
+  members: Span[];
+  failedCount: number;
+  p50Duration: number;
+  /** Every member failed (e.g. a tool that failed ×12). */
+  isFailureGroup: boolean;
+  expanded: boolean;
+}
+
+export interface LoadMoreRowData {
+  kind: "load-more";
+  id: string;
+  depth: number;
+  groupId: string;
+  remaining: number;
+}
+
+export type TreeRow = SpanRowData | GroupRowData | LoadMoreRowData;
+
+export interface SpanTreeState {
+  hideFramework: boolean;
+  collapsedSpanIds: ReadonlySet<string>;
+  expandedGroupIds: ReadonlySet<string>;
+  groupRevealCounts: Readonly<Record<string, number>>;
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts

@@ -0,0 +1,89 @@
+/**
+ * Agent tracing types. Mirrors `litellm/tracing/types.py` exactly.
+ *
+ * A trace is one agent run made of spans (agent / llm / tool / chain / framework).
+ */
+
+export type SpanType = "agent" | "llm" | "tool" | "chain" | "framework";
+export type SpanStatus = "ok" | "error" | "unset";
+
+export interface Span {
+  span_id: string;
+  parent_span_id: string | null;
+  name: string;
+  type: SpanType;
+  /** The agent this span runs inside, e.g. "researcher". */
+  agent: string;
+  /** Relative to trace start. */
+  start_offset_ms: number;
+  duration_ms: number;
+  status: SpanStatus;
+  /** Exception message when status is "error". */
+  error?: string | null;
+  input_preview: string;
+  model: string | null;
+  input_tokens: number;
+  output_tokens: number;
+  litellm_request_id: string | null;
+}
+
+/** One distinct agent in a trace. 200 invocations of `researcher` = one node. */
+export interface AgentNode {
+  name: string;
+  parent_agent: string | null;
+  invocations: number;
+  llm_calls: number;
+  tool_calls: number;
+  duration_ms: number;
+}
+
+export interface TraceSummary {
+  trace_id: string;
+  name: string;
+  service: string;
+  input_preview: string;
+  /** ISO 8601 */
+  start_time: string;
+  duration_ms: number;
+  status: SpanStatus;
+  span_count: number;
+  agent_count: number;
+  llm_calls: number;
+  tool_calls: number;
+  /** Spans with an error status; > 0 means the run shows as failed. */
+  error_count: number;
+  input_tokens: number;
+  output_tokens: number;
+  models: string[];
+}
+
+export interface Trace {
+  summary: TraceSummary;
+  agents: AgentNode[];
+  spans: Span[];
+}
+
+export interface TracePage {
+  data: TraceSummary[];
+  next_cursor: string | null;
+}
+
+/** `input` / `output` are JSON strings (messages for llm spans, raw args / result for tools). */
+export interface SpanDetail {
+  span_id: string;
+  input: string;
+  output: string;
+  attributes: Record<string, string>;
+}
+
+export interface TraceToolCall {
+  name: string;
+  args: unknown;
+}
+
+export interface TraceMessage {
+  role: string;
+  content: string;
+  name?: string;
+  tool_calls?: TraceToolCall[];
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts

@@ -0,0 +1,269 @@
+import { describe, expect, it } from "vitest";
+
+import deepAgentTrace from "./__fixtures__/deep_agent_trace.json";
+import researchTrace from "./__fixtures__/research_trace.json";
+import swarmTrace from "./__fixtures__/swarm_trace.json";
+import { type SpanTreeState, type TreeRow } from "./traceTree";
+import type { Span, Trace } from "./traceTypes";
+import {
+  buildTreeRows,
+  buildVisibleTree,
+  errorSource,
+  firstErrorSpan,
+  fmtMs,
+  GROUP_PAGE_SIZE,
+  groupRowId,
+  isFrameworkSpan,
+  median,
+  parseMessages,
+  previewText,
+  revealSpanInState,
+  ROOT_KEY,
+} from "./traceUtils";
+
+const swarm = swarmTrace as Trace;
+const research = researchTrace as Trace;
+const deepAgent = deepAgentTrace as Trace;
+
+const STATE: SpanTreeState = {
+  hideFramework: true,
+  collapsedSpanIds: new Set(),
+  expandedGroupIds: new Set(),
+  groupRevealCounts: {},
+};
+
+type SpanOverrides = Partial<Span> & Pick<Span, "span_id">;
+
+const span = (overrides: SpanOverrides): Span => ({
+  parent_span_id: null,
+  name: overrides.span_id,
+  type: "chain",
+  agent: "root",
+  start_offset_ms: 0,
+  duration_ms: 1,
+  status: "ok",
+  error: null,
+  input_preview: "",
+  model: null,
+  input_tokens: 0,
+  output_tokens: 0,
+  litellm_request_id: null,
+  ...overrides,
+});
+
+const groups = (rows: TreeRow[]) => rows.filter((r): r is Extract<TreeRow, { kind: "group" }> => r.kind === "group");
+const spanRows = (rows: TreeRow[]) => rows.filter((r): r is Extract<TreeRow, { kind: "span" }> => r.kind === "span");
+
+describe("formatting", () => {
+  it("formats durations", () => {
+    expect(fmtMs(4.25)).toBe("4.3ms");
+    expect(fmtMs(950)).toBe("950ms");
+    expect(fmtMs(51386)).toBe("51.39s");
+  });
+
+  it("pulls the user message out of a truncated JSON preview", () => {
+    expect(previewText('[{"role": "user", "content": "Customer acme-404 says billing is wrong."}]')).toBe(
+      "Customer acme-404 says billing is wrong.",
+    );
+    expect(previewText('[{"role": "system", "content": "sys"}, {"role": "user", "content": "Compare ingest thr')).toBe(
+      "Compare ingest thr",
+    );
+    expect(previewText("plain text")).toBe("plain text");
+  });
+
+  it("pulls the user message out of an OpenInference LangChain input", () => {
+    const input =
+      '{"messages": [{"type": "human", "data": {"content": "Customer acme-7 keeps hitting 429s", "type": "human"';
+    expect(previewText(input)).toBe("Customer acme-7 keeps hitting 429s");
+  });
+
+  it("takes the upper median", () => {
+    expect(median([3, 1, 2])).toBe(2);
+    expect(median([4, 1, 3, 2])).toBe(3);
+    expect(median([])).toBe(0);
+  });
+});
+
+describe("buildVisibleTree / isFrameworkSpan", () => {
+  it("hides framework spans and re-parents their children", () => {
+    const middleware: SpanOverrides = {
+      span_id: "mw",
+      parent_span_id: "root",
+      type: "framework",
+      name: "X.wrap_model_call",
+    };
+    const modelNode: SpanOverrides = { span_id: "model", parent_span_id: "mw", type: "chain", name: "model" };
+    const llmCall: SpanOverrides = { span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 };
+    const planner: SpanOverrides = {
+      span_id: "step",
+      parent_span_id: "root",
+      type: "chain",
+      name: "planner",
+      start_offset_ms: 1,
+    };
+    const spans = [
+      span({ span_id: "root", type: "agent" }),
+      span(middleware),
+      span(modelNode),
+      span(llmCall),
+      span(planner),
+    ];
+    const compact = buildVisibleTree(spans, false);
+    expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]);
+    expect(compact.children.get("root")?.map((s) => s.span_id)).toEqual(["step", "llm"]);
+    expect(buildVisibleTree(spans, true).visibleCount).toBe(5);
+  });
+
+  it("never hides the root span", () => {
+    expect(isFrameworkSpan(span({ span_id: "r", type: "framework" }))).toBe(false);
+  });
+});
+
+describe("buildTreeRows", () => {
+  it("starts the tree at the root span", () => {
+    const rows = buildTreeRows(research.spans, STATE);
+    expect(rows[0]).toMatchObject({
+      kind: "span",
+      depth: 0,
+      id: research.spans.find((s) => s.parent_span_id === null)?.span_id,
+    });
+  });
+
+  it("hides every framework / graph-node span of the real Deep Agents trace, keeping all LLM calls", () => {
+    const rows = buildTreeRows(research.spans, STATE);
+    const shown = new Set(spanRows(rows).map((r) => r.span.span_id));
+    expect(research.spans.filter(isFrameworkSpan).every((s) => !shown.has(s.span_id))).toBe(true);
+    const all = buildTreeRows(research.spans, { ...STATE, hideFramework: false });
+    expect(spanRows(all).length).toBeGreaterThan(spanRows(rows).length);
+  });
+
+  it("folds the swarm's 12 researcher invocations into one group row", () => {
+    const rows = buildTreeRows(swarm.spans, STATE);
+    const researcher = groups(rows).find((g) => g.name === "researcher" && g.type === "agent");
+    expect(researcher?.members).toHaveLength(12);
+    expect(researcher?.expanded).toBe(false);
+    expect(researcher?.p50Duration).toBeGreaterThan(0);
+    // folded members are not rendered until the group is expanded
+    expect(spanRows(rows).some((r) => r.span.name === "researcher")).toBe(false);
+  });
+
+  it("folds as few as 3 failed siblings of the same tool into a failure group", () => {
+    const parent = span({ span_id: "p", type: "agent" });
+    const failing = [0, 1, 2].map((i) => {
+      const failedGrep: SpanOverrides = {
+        span_id: `t${i}`,
+        parent_span_id: "p",
+        type: "tool",
+        name: "grep_code",
+        status: "error",
+        start_offset_ms: i,
+      };
+      return span(failedGrep);
+    });
+    const rows = buildTreeRows([parent, ...failing], STATE);
+    const group = groups(rows)[0];
+    expect(group).toMatchObject({ name: "grep_code", failedCount: 3, isFailureGroup: true });
+  });
+
+  it("never folds same-named calls from different agents into one group", () => {
+    const parent = span({ span_id: "p", type: "agent" });
+    const calls = [0, 1, 2, 3, 4, 5].map((i) => {
+      const call: SpanOverrides = {
+        span_id: `c${i}`,
+        parent_span_id: "p",
+        type: "llm",
+        name: "ChatOpenAI",
+        agent: i < 3 ? "planner" : "critic",
+      };
+      return span(call);
+    });
+    const rows = buildTreeRows([parent, ...calls], STATE);
+    expect(groups(rows)).toHaveLength(0);
+    expect(spanRows(rows).filter((r) => r.span.name === "ChatOpenAI")).toHaveLength(6);
+  });
+
+  it("leaves 5 healthy same-named siblings unfolded", () => {
+    const parent = span({ span_id: "p", type: "agent" });
+    const kids = [0, 1, 2, 3, 4].map((i) => {
+      const search: SpanOverrides = { span_id: `k${i}`, parent_span_id: "p", type: "tool", name: "search" };
+      return span(search);
+    });
+    expect(groups(buildTreeRows([parent, ...kids], STATE))).toHaveLength(0);
+  });
+
+  it("pages expanded groups 20 at a time with a load-more row", () => {
+    const parent = span({ span_id: "p", type: "agent" });
+    const kids = Array.from({ length: 45 }, (_, i) => {
+      const worker: SpanOverrides = {
+        span_id: `k${i}`,
+        parent_span_id: "p",
+        type: "agent",
+        name: "worker",
+        start_offset_ms: i,
+      };
+      return span(worker);
+    });
+    const all = [parent, ...kids];
+    const id = groupRowId("p", kids[0]);
+    const page1 = buildTreeRows(all, { ...STATE, expandedGroupIds: new Set([id]) });
+    expect(spanRows(page1).filter((r) => r.span.name === "worker")).toHaveLength(GROUP_PAGE_SIZE);
+    expect(page1.find((r) => r.kind === "load-more")).toMatchObject({ groupId: id, remaining: 25 });
+    const everything = buildTreeRows(all, {
+      ...STATE,
+      expandedGroupIds: new Set([id]),
+      groupRevealCounts: { [id]: 60 },
+    });
+    expect(spanRows(everything).filter((r) => r.span.name === "worker")).toHaveLength(45);
+    expect(everything.some((r) => r.kind === "load-more")).toBe(false);
+  });
+
+  it("hides the children of collapsed spans", () => {
+    const root = swarm.spans.find((s) => s.parent_span_id === null) as Span;
+    const rows = buildTreeRows(swarm.spans, { ...STATE, collapsedSpanIds: new Set([root.span_id]) });
+    expect(rows.map((r) => r.kind)).toEqual(["span"]);
+  });
+});
+
+describe("revealSpanInState", () => {
+  it("opens the path to a span nested in a folded group so the view can land on it", () => {
+    const failed = firstErrorSpan(swarm.spans) as Span;
+    const state = revealSpanInState(swarm.spans, STATE, failed.span_id);
+    const rows = buildTreeRows(swarm.spans, state);
+    expect(rows.some((r) => r.id === failed.span_id)).toBe(true);
+  });
+});
+
+describe("errorSource", () => {
+  it("blames the tool, LiteLLM, or the model", () => {
+    expect(errorSource(span({ span_id: "a", status: "ok" }))).toBeNull();
+    const failure = (span_id: string, type: Span["type"], error: string): Span => {
+      const failed: SpanOverrides = { span_id, type, error, status: "error" };
+      return span(failed);
+    };
+    expect(errorSource(failure("t", "tool", "boom"))).toBe("tool");
+    expect(errorSource(failure("l", "llm", "429 Rate limit exceeded"))).toBe("litellm");
+    expect(errorSource(failure("g", "llm", "Blocked by guardrail"))).toBe("litellm");
+    expect(errorSource(failure("m", "llm", "context length exceeded"))).toBe("model");
+  });
+
+  it("classifies the swarm's failing lookup_benchmark calls as tool errors", () => {
+    const failedTool = swarm.spans.find((s) => s.status === "error" && s.type === "tool") as Span;
+    expect(failedTool.name).toBe("lookup_benchmark");
+    expect(errorSource(failedTool)).toBe("tool");
+  });
+});
+
+describe("payload helpers", () => {
+  it("finds the earliest failing non-root span", () => {
+    const failed = firstErrorSpan(swarm.spans);
+    expect(failed?.status).toBe("error");
+    expect(failed?.parent_span_id).not.toBeNull();
+    expect(firstErrorSpan(deepAgent.spans)).toBeNull();
+  });
+
+  it("parses llm message payloads and rejects non-message JSON", () => {
+    expect(parseMessages('[{"role":"user","content":"hi"}]')).toEqual([{ role: "user", content: "hi" }]);
+    expect(parseMessages('{"file_path":"/tmp/x"}')).toBeNull();
+    expect(parseMessages("not json")).toBeNull();
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts

@@ -0,0 +1,314 @@
+/**
+ * Pure helpers for the agent trace views. No React in here: everything the span tree
+ * computes lives here so it can be unit-tested directly.
+ */
+import { type ErrorSource, type SpanTreeState, type TreeRow } from "./traceTree";
+import type { Span, TraceMessage, TraceSummary } from "./traceTypes";
+
+/* ------------------------------------------------------------------ */
+/*  Formatting                                                         */
+/* ------------------------------------------------------------------ */
+
+export const fmtMs = (ms: number): string => {
+  if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`;
+  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+  return `${Math.max(ms, 0).toFixed(ms < 10 ? 1 : 0)}ms`;
+};
+
+export const fmtTok = (n: number): string => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n));
+
+export const shortId = (id: string, length = 16): string => (id.length > length ? `${id.slice(0, length)}…` : id);
+
+/* ------------------------------------------------------------------ */
+/*  Visible tree (framework spans hidden + children re-parented)       */
+/* ------------------------------------------------------------------ */
+
+/** Root-level key in the children map. */
+export const ROOT_KEY = "__root__";
+
+export type ChildrenMap = Map<string, Span[]>;
+
+export interface VisibleTree {
+  children: ChildrenMap;
+  visibleCount: number;
+}
+
+/**
+ * Framework plumbing: middleware wrappers plus LangGraph's generic "model" / "tools"
+ * graph nodes. The root span is never hidden.
+ */
+const GRAPH_NODE_NAMES = new Set(["model", "tools"]);
+
+export const isFrameworkSpan = (span: Span): boolean => {
+  const isGraphNode = span.type === "chain" && GRAPH_NODE_NAMES.has(span.name);
+  const isPlumbing = span.type === "framework" || isGraphNode;
+  return span.parent_span_id !== null && isPlumbing;
+};
+
+const byStart = (a: Span, b: Span): number => a.start_offset_ms - b.start_offset_ms;
+
+export const indexSpans = (spans: readonly Span[]): Map<string, Span> => new Map(spans.map((s) => [s.span_id, s]));
+
+const pushChild = (children: ChildrenMap, key: string, span: Span): void => {
+  const list = children.get(key);
+  if (list) list.push(span);
+  else children.set(key, [span]);
+};
+
+/**
+ * Children map for the waterfall. With `showFramework` off, framework spans are
+ * dropped and their children attach to the nearest visible ancestor. Spans whose
+ * parent is missing from the trace attach to the root level.
+ */
+export function buildVisibleTree(spans: readonly Span[], showFramework: boolean): VisibleTree {
+  const byId = indexSpans(spans);
+  const hidden = (span: Span) => !showFramework && isFrameworkSpan(span);
+  const visibleParentKey = (span: Span): string => {
+    let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined;
+    while (parent && hidden(parent)) {
+      parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined;
+    }
+    return parent ? parent.span_id : ROOT_KEY;
+  };
+  const children: ChildrenMap = new Map();
+  let visibleCount = 0;
+  for (const span of spans) {
+    if (hidden(span)) continue;
+    visibleCount++;
+    pushChild(children, visibleParentKey(span), span);
+  }
+  children.forEach((list) => list.sort(byStart));
+  return { children, visibleCount };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Run view rows (Input, span tree with grouping, Output)             */
+/* ------------------------------------------------------------------ */
+
+/** Same-named siblings fold into one row at this count ... */
+export const GROUP_THRESHOLD_OK = 6;
+/** ... or as soon as this many of them failed. */
+export const GROUP_THRESHOLD_ERROR = 3;
+/** Group rows reveal this many members at a time. */
+export const GROUP_PAGE_SIZE = 20;
+
+export const median = (values: readonly number[]): number => {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.floor(sorted.length / 2)];
+};
+
+const LITELLM_ERROR = /rate.?limit|429|guardrail|budget|litellm/i;
+
+/** Who failed: the tool, LiteLLM (rate limit / guardrail / budget), or the model. Null when the span is fine. */
+export function errorSource(span: Span): ErrorSource | null {
+  if (span.status !== "error") return null;
+  if (span.type === "tool") return "tool";
+  if (LITELLM_ERROR.test(span.error ?? "")) return "litellm";
+  return "model";
+}
+
+type GroupOrSpan = Span | { group: Span[] };
+
+const groupKey = (span: Pick<Span, "name" | "type" | "agent">): string => `${span.agent}|${span.name}|${span.type}`;
+
+/** Siblings sharing agent + name + type fold into one group once there are enough of them (or enough failures). */
+function groupChildren(children: readonly Span[]): GroupOrSpan[] {
+  const byKey = new Map<string, Span[]>();
+  for (const child of children) {
+    const key = groupKey(child);
+    const list = byKey.get(key);
+    if (list) list.push(child);
+    else byKey.set(key, [child]);
+  }
+  const emitted = new Set<string>();
+  const out: GroupOrSpan[] = [];
+  for (const child of children) {
+    const key = groupKey(child);
+    const group = byKey.get(key) ?? [child];
+    const failed = group.filter((s) => s.status === "error").length;
+    if (group.length >= GROUP_THRESHOLD_OK || failed >= GROUP_THRESHOLD_ERROR) {
+      if (!emitted.has(key)) {
+        emitted.add(key);
+        out.push({ group });
+      }
+    } else {
+      out.push(child);
+    }
+  }
+  return out;
+}
+
+export const groupRowId = (parentKey: string, span: Pick<Span, "name" | "type" | "agent">): string =>
+  `grp::${parentKey}::${groupKey(span)}`;
+
+interface RowContext {
+  children: ChildrenMap;
+  state: SpanTreeState;
+  rows: TreeRow[];
+}
+
+function pushSpan(ctx: RowContext, span: Span, depth: number): void {
+  const hasChildren = (ctx.children.get(span.span_id)?.length ?? 0) > 0;
+  const collapsed = ctx.state.collapsedSpanIds.has(span.span_id);
+  const row: TreeRow = { kind: "span", id: span.span_id, span, depth, hasChildren, collapsed };
+  ctx.rows.push(row);
+  if (hasChildren && !collapsed) pushLevel(ctx, span.span_id, depth + 1);
+}
+
+function pushGroup(ctx: RowContext, parentKey: string, members: Span[], depth: number): void {
+  const first = members[0];
+  const id = groupRowId(parentKey, first);
+  const failedCount = members.filter((s) => s.status === "error").length;
+  const expanded = ctx.state.expandedGroupIds.has(id);
+  const groupRow: TreeRow = {
+    kind: "group",
+    id,
+    depth,
+    name: first.name,
+    type: first.type,
+    agent: first.agent,
+    members,
+    failedCount,
+    p50Duration: median(members.map((s) => s.duration_ms)),
+    isFailureGroup: failedCount === members.length,
+    expanded,
+  };
+  ctx.rows.push(groupRow);
+  if (!expanded) return;
+  const reveal = Math.min(ctx.state.groupRevealCounts[id] ?? GROUP_PAGE_SIZE, members.length);
+  members.slice(0, reveal).forEach((member) => pushSpan(ctx, member, depth + 1));
+  if (reveal < members.length) {
+    const moreRow: TreeRow = {
+      kind: "load-more",
+      id: `${id}::more`,
+      depth: depth + 1,
+      groupId: id,
+      remaining: members.length - reveal,
+    };
+    ctx.rows.push(moreRow);
+  }
+}
+
+function pushLevel(ctx: RowContext, parentKey: string, depth: number): void {
+  for (const item of groupChildren(ctx.children.get(parentKey) ?? [])) {
+    if ("group" in item) pushGroup(ctx, parentKey, item.group, depth);
+    else pushSpan(ctx, item, depth);
+  }
+}
+
+/**
+ * Rows for the run view: the span tree, framework spans optionally hidden with their children lifted
+ * and same-named siblings folded into paged groups.
+ */
+export function buildTreeRows(spans: readonly Span[], state: SpanTreeState): TreeRow[] {
+  const ctx: RowContext = { children: buildVisibleTree(spans, !state.hideFramework).children, state, rows: [] };
+  pushLevel(ctx, ROOT_KEY, 0);
+  return ctx.rows;
+}
+
+/** Tree state with every visible ancestor of `spanId` expanded and any group holding it paged far enough. */
+export function revealSpanInState(spans: readonly Span[], state: SpanTreeState, spanId: string): SpanTreeState {
+  const { children } = buildVisibleTree(spans, !state.hideFramework);
+  const parentOf = new Map<string, string>();
+  children.forEach((list, key) => list.forEach((s) => parentOf.set(s.span_id, key)));
+  if (!parentOf.has(spanId)) return state;
+  const collapsed = new Set(state.collapsedSpanIds);
+  const expanded = new Set(state.expandedGroupIds);
+  const reveal = { ...state.groupRevealCounts };
+  let current = spanId;
+  while (parentOf.has(current)) {
+    const parentKey = parentOf.get(current) as string;
+    collapsed.delete(parentKey);
+    for (const item of groupChildren(children.get(parentKey) ?? [])) {
+      if (!("group" in item)) continue;
+      const index = item.group.findIndex((s) => s.span_id === current);
+      if (index < 0) continue;
+      const id = groupRowId(parentKey, item.group[0]);
+      expanded.add(id);
+      reveal[id] = Math.max(reveal[id] ?? GROUP_PAGE_SIZE, Math.ceil((index + 1) / GROUP_PAGE_SIZE) * GROUP_PAGE_SIZE);
+    }
+    current = parentKey;
+  }
+  return { ...state, collapsedSpanIds: collapsed, expandedGroupIds: expanded, groupRevealCounts: reveal };
+}
+
+/** `spanId` if it shows in the tree, else its nearest ancestor that does (framework spans can be hidden). */
+export function nearestVisibleSpanId(spans: readonly Span[], spanId: string, hideFramework: boolean): string {
+  if (!hideFramework) return spanId;
+  const byId = new Map(spans.map((s) => [s.span_id, s]));
+  let current = byId.get(spanId);
+  while (current && isFrameworkSpan(current) && current.parent_span_id !== null) {
+    current = byId.get(current.parent_span_id);
+  }
+  return current?.span_id ?? spanId;
+}
+
+/* ------------------------------------------------------------------ */
+/*  Trace-level rollups                                                */
+/* ------------------------------------------------------------------ */
+
+/** Earliest failing non-root span (the root just echoes its children), else the root. */
+export function firstErrorSpan(spans: readonly Span[]): Span | null {
+  const failed = spans.filter((s) => s.status === "error").sort(byStart);
+  return failed.find((s) => s.parent_span_id !== null) ?? failed[0] ?? null;
+}
+
+/* ------------------------------------------------------------------ */
+/*  Span detail payloads                                               */
+/* ------------------------------------------------------------------ */
+
+export const parseJson = (value: string): unknown => {
+  if (!value) return null;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+};
+
+const isMessage = (value: unknown): value is TraceMessage => {
+  const isObject = typeof value === "object" && value !== null;
+  return isObject && "role" in value && typeof (value as TraceMessage).role === "string";
+};
+
+/** An llm span's input (array of messages) or output (one message); null when it isn't one. */
+export function parseMessages(value: string): TraceMessage[] | null {
+  const parsed = parseJson(value);
+  if (Array.isArray(parsed)) return parsed.every(isMessage) ? parsed : null;
+  return isMessage(parsed) ? [parsed] : null;
+}
+
+/** Pretty JSON when the payload is JSON, else the raw string. */
+export const prettyPayload = (value: string): string => {
+  const parsed = parseJson(value);
+  if (parsed === null || typeof parsed === "string") return typeof parsed === "string" ? parsed : value;
+  return JSON.stringify(parsed, null, 2);
+};
+
+/* ------------------------------------------------------------------ */
+/*  List view helpers                                                  */
+/* ------------------------------------------------------------------ */
+
+const PREVIEW_USER_CONTENT =
+  /"(?:role|type)":\s*"(?:user|human)",\s*"(?:content|data)":\s*(?:\{"content":\s*)?"((?:[^"\\]|\\.)*)/;
+
+/**
+ * Human text for an input preview. Previews are often a (possibly truncated) JSON
+ * message array; show the last user/tool message's content when we can find it.
+ */
+export function previewText(preview: string): string {
+  if (!preview) return "";
+  const messages = parseMessages(preview);
+  if (messages) {
+    const last = [...messages].reverse().find((m) => m.role === "user" || m.role === "tool") ?? messages.at(-1);
+    return last?.content || preview;
+  }
+  if (!/^\s*[[{]/.test(preview)) return preview;
+  const match = PREVIEW_USER_CONTENT.exec(preview);
+  return match ? match[1].replace(/\\n/g, " ").replace(/\\"/g, '"') : preview;
+}
+
+/** Trace display name; root spans without a name fall back to the service. */
+export const traceDisplayName = (summary: Pick<TraceSummary, "name" | "service">): string =>
+  summary.name || summary.service || "(unnamed trace)";

ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+
+import { traceWindowStartMs } from "./useAgentTraces";
+import { spanLogWindow } from "./useSpanRequestLog";
+
+const HOUR = 3600 * 1000;
+
+describe("traceWindowStartMs", () => {
+  it("rolls a preset range forward with now so live tail keeps a fixed-length window", () => {
+    const start = "2026-09-29T10:00";
+    const end = "2026-09-30T10:00";
+    const mountedAt = Date.parse("2026-09-30T10:00:00");
+    const tenHoursLater = mountedAt + 10 * HOUR;
+    expect(tenHoursLater - traceWindowStartMs(start, end, false, tenHoursLater)).toBe(24 * HOUR);
+    expect(traceWindowStartMs(start, end, false, tenHoursLater)).toBeGreaterThan(
+      traceWindowStartMs(start, end, false, mountedAt),
+    );
+  });
+
+  it("keeps a custom range pinned to what the user picked", () => {
+    const start = "2026-09-01T00:00";
+    expect(traceWindowStartMs(start, "2026-09-02T00:00", true, Date.parse("2026-09-30T00:00:00"))).toBe(
+      Date.parse(start),
+    );
+  });
+});
+
+describe("spanLogWindow", () => {
+  it("looks up the request log around the span's own time, not the logs tab window", () => {
+    const spanStart = Date.parse("2026-08-01T12:00:00Z");
+    expect(spanLogWindow(spanStart)).toEqual({ start_date: "2026-08-01 11:30:00", end_date: "2026-08-01 12:30:00" });
+  });
+});

ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts

@@ -0,0 +1,91 @@
+import { useInfiniteQuery } from "@tanstack/react-query";
+import moment from "moment";
+import { useMemo } from "react";
+
+import { ApiError } from "@/lib/http/client";
+
+import { agentTraceListCall } from "../../networking";
+import { LIVE_TAIL_INTERVAL_MS } from "../log_filter_logic";
+import type { TracePage, TraceSummary } from "./traceTypes";
+
+export const TRACING_NOT_ENABLED_STATUS = 501;
+/** A proxy without the tracing routes at all answers 404; treat it like tracing being off. */
+const TRACING_ROUTE_MISSING_STATUS = 404;
+
+export const isTracingNotEnabled = (error: unknown): error is ApiError =>
+  error instanceof ApiError &&
+  (error.status === TRACING_NOT_ENABLED_STATUS || error.status === TRACING_ROUTE_MISSING_STATUS);
+
+interface UseAgentTracesOptions {
+  accessToken: string;
+  startTime: string;
+  endTime: string;
+  isCustomDate: boolean;
+  isLiveTail: boolean;
+  enabled: boolean;
+}
+
+export interface AgentTracesResult {
+  traces: TraceSummary[];
+  isLoading: boolean;
+  isFetching: boolean;
+  /** Set when the proxy answered 501: tracing isn't configured. */
+  notEnabledDetail: string | null;
+  error: Error | null;
+  hasMore: boolean;
+  loadMore: () => void;
+  refetch: () => void;
+}
+
+/** Start of the fetch window: a preset range rolls with "now", so live tail keeps a fixed-length window. */
+export const traceWindowStartMs = (startTime: string, endTime: string, isCustomDate: boolean, nowMs: number): number =>
+  isCustomDate ? moment(startTime).valueOf() : nowMs - (moment(endTime).valueOf() - moment(startTime).valueOf());
+
+/**
+ * GET /v1/traces for the Logs page time range, cursor-paginated ("Load more").
+ * Preset ranges re-read "now" on every fetch, moving both bounds so the window keeps its length.
+ */
+export function useAgentTraces({
+  accessToken,
+  startTime,
+  endTime,
+  isCustomDate,
+  isLiveTail,
+  enabled,
+}: UseAgentTracesOptions): AgentTracesResult {
+  const fetchPage = (pageParam: unknown): Promise<TracePage> => {
+    const nowMs = Date.now();
+    const listOptions: Parameters<typeof agentTraceListCall>[0] = {
+      accessToken,
+      startMs: traceWindowStartMs(startTime, endTime, isCustomDate, nowMs),
+      endMs: isCustomDate ? moment(endTime).valueOf() : nowMs,
+      cursor: pageParam as string | null,
+    };
+    return agentTraceListCall(listOptions);
+  };
+  const queryOptions: Parameters<typeof useInfiniteQuery<TracePage, Error>>[0] = {
+    queryKey: ["agentTraces", accessToken, startTime, endTime, isCustomDate],
+    queryFn: ({ pageParam }) => fetchPage(pageParam),
+    initialPageParam: null,
+    getNextPageParam: (lastPage) => lastPage.next_cursor ?? undefined,
+    enabled,
+    retry: (failureCount, error) => !isTracingNotEnabled(error) && failureCount < 1,
+    refetchInterval: (q) => (isLiveTail && !isTracingNotEnabled(q.state.error) ? LIVE_TAIL_INTERVAL_MS : false),
+    refetchIntervalInBackground: false,
+  };
+  const query = useInfiniteQuery<TracePage, Error>(queryOptions);
+
+  const traces = useMemo(() => query.data?.pages.flatMap((page) => page.data) ?? [], [query.data]);
+  const notEnabled = isTracingNotEnabled(query.error);
+
+  return {
+    traces,
+    isLoading: query.isLoading,
+    isFetching: query.isFetching,
+    notEnabledDetail: notEnabled ? query.error?.message || "Agent tracing is not enabled" : null,
+    error: notEnabled ? null : query.error,
+    hasMore: query.hasNextPage,
+    loadMore: () => void query.fetchNextPage(),
+    refetch: () => void query.refetch(),
+  };
+}

ui/litellm-dashboard/src/components/view_logs/TraceView/useSpanRequestLog.ts

@@ -0,0 +1,47 @@
+"use client";
+
+import { useQuery, type UseQueryOptions } from "@tanstack/react-query";
+import moment from "moment";
+
+import { uiSpendLogsCall } from "../../networking";
+import type { LogEntry } from "../columns";
+
+/** Spend-log timestamps are written when the call finishes, so pad the span start on both sides. */
+const LOOKUP_PAD_MINUTES = 30;
+const SPEND_LOG_TIME_FORMAT = "YYYY-MM-DD HH:mm:ss";
+
+export const spanLogWindow = (spanStartMs: number): { start_date: string; end_date: string } => ({
+  start_date: moment.utc(spanStartMs).subtract(LOOKUP_PAD_MINUTES, "minutes").format(SPEND_LOG_TIME_FORMAT),
+  end_date: moment.utc(spanStartMs).add(LOOKUP_PAD_MINUTES, "minutes").format(SPEND_LOG_TIME_FORMAT),
+});
+
+/**
+ * The LiteLLM request log behind an LLM span, looked up around the span's own time rather than
+ * the Request Logs tab's window, so runs older than that window still resolve.
+ */
+export function useSpanRequestLog(
+  accessToken: string,
+  requestId: string | null,
+  spanStartMs: number,
+  enabled: boolean,
+) {
+  const fetchLog = async (): Promise<LogEntry | null> => {
+    if (requestId === null) return null;
+    const logsOptions: Parameters<typeof uiSpendLogsCall>[0] = {
+      accessToken,
+      ...spanLogWindow(spanStartMs),
+      page: 1,
+      page_size: 1,
+      params: { request_id: requestId },
+    };
+    const response = await uiSpendLogsCall(logsOptions);
+    return response.data.find((log: LogEntry) => log.request_id === requestId) ?? null;
+  };
+  const queryOptions: UseQueryOptions<LogEntry | null> = {
+    queryKey: ["logs", "spanRequest", requestId, spanStartMs, accessToken],
+    queryFn: fetchLog,
+    enabled: enabled && requestId !== null,
+    staleTime: Infinity,
+  };
+  return useQuery(queryOptions);
+}

ui/litellm-dashboard/src/components/view_logs/index.test.tsx

@@ -65,12 +65,14 @@ describe("SpendLogsTable", () => {
     useOrganizationsMock.mockReturnValue({ data: [] });
   });
 
-  it("renders the four log tabs", () => {
+  it("renders the log tabs, with Agent Traces marked new and Request Logs selected", () => {
     renderAs("Admin");
 
     for (const label of ["Request Logs", "Audit Logs", "Deleted Keys", "Deleted Teams"]) {
       expect(screen.getByRole("tab", { name: label })).toBeInTheDocument();
     }
+    expect(screen.getByRole("tab", { name: /Agent Traces/ })).toHaveTextContent("New");
+    expect(screen.getByRole("tab", { name: "Request Logs" })).toHaveAttribute("aria-selected", "true");
   });
 
   it("marks only the visible tab's panel active so background tabs do not query", async () => {
@@ -115,7 +117,7 @@ describe("SpendLogsTable", () => {
     it("does not hand an org admin the Audit Logs tab, which the backend still refuses them", () => {
       renderAs("Internal User", ORG_ADMIN_MEMBERSHIPS);
 
-      expect(tabNames()).toEqual(["Request Logs", "Deleted Keys", "Deleted Teams"]);
+      expect(tabNames()).toEqual(["Request Logs", "Agent TracesNew", "Deleted Keys", "Deleted Teams"]);
       expect(screen.queryByTestId("audit-logs-panel")).not.toBeInTheDocument();
     });
 
@@ -124,7 +126,7 @@ describe("SpendLogsTable", () => {
         { organization_id: "org-1", members: [{ user_id: "user-1", user_role: "internal_user" }] },
       ]);
 
-      expect(tabNames()).toEqual(["Request Logs", "Deleted Keys"]);
+      expect(tabNames()).toEqual(["Request Logs", "Agent TracesNew", "Deleted Keys"]);
     });
 
     it("activates the org admin's selected tab rather than the one at the four-tab index", async () => {

ui/litellm-dashboard/src/components/view_logs/index.tsx

@@ -4,6 +4,7 @@ import DeletedKeysPage from "../DeletedKeysPage/DeletedKeysPage";
 import DeletedTeamsPage from "../DeletedTeamsPage/DeletedTeamsPage";
 import AuditLogsPanel from "./AuditLogsPanel";
 import RequestLogsPanel from "./RequestLogsPanel";
+import AgentTracesPage from "./TraceView/AgentTracesPage";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
 
@@ -15,20 +16,24 @@ interface SpendLogsTableProps {
   premiumUser: boolean;
 }
 
-type LogsTabId = "request logs" | "audit logs" | "deleted keys" | "deleted teams";
+type LogsTabId = "request logs" | "agent traces" | "audit logs" | "deleted keys" | "deleted teams";
 
 interface LogsTab {
   id: LogsTabId;
   label: string;
+  isNew?: boolean;
 }
 
 const REQUEST_LOGS_TAB: LogsTab = { id: "request logs", label: "Request Logs" };
+const AGENT_TRACES_TAB: LogsTab = { id: "agent traces", label: "Agent Traces", isNew: true };
 const AUDIT_LOGS_TAB: LogsTab = { id: "audit logs", label: "Audit Logs" };
 const DELETED_KEYS_TAB: LogsTab = { id: "deleted keys", label: "Deleted Keys" };
 const DELETED_TEAMS_TAB: LogsTab = { id: "deleted teams", label: "Deleted Teams" };
 
 const tabContentClassName = (tabId: LogsTabId): string =>
-  tabId === REQUEST_LOGS_TAB.id ? "flex min-h-0 flex-1 flex-col" : "min-h-0 flex-1 overflow-y-auto";
+  tabId === REQUEST_LOGS_TAB.id || tabId === AGENT_TRACES_TAB.id
+    ? "flex min-h-0 flex-1 flex-col"
+    : "min-h-0 flex-1 overflow-y-auto";
 
 export default function SpendLogsTable({ accessToken, token, userRole, userID, premiumUser }: SpendLogsTableProps) {
   const [activeTab, setActiveTab] = useState<LogsTabId>(REQUEST_LOGS_TAB.id);
@@ -45,6 +50,7 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
 
   const tabs: LogsTab[] = [
     REQUEST_LOGS_TAB,
+    AGENT_TRACES_TAB,
     ...(canViewAuditLogs ? [AUDIT_LOGS_TAB] : []),
     DELETED_KEYS_TAB,
     ...(canViewDeletedTeams ? [DELETED_TEAMS_TAB] : []),
@@ -62,6 +68,8 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
             isActive={activeTab === "request logs"}
           />
         );
+      case "agent traces":
+        return activeTab === "agent traces" ? <AgentTracesPage accessToken={accessToken} /> : null;
       case "audit logs":
         return (
           <AuditLogsPanel
@@ -81,12 +89,21 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p
   };
 
   return (
-    <div className="flex h-full w-full flex-col p-6">
-      <Tabs value={activeTab} onValueChange={(value) => setActiveTab(value as LogsTabId)} className="min-h-0 flex-1">
-        <TabsList variant="line">
+    <div className="flex h-full w-full flex-col px-4 pb-4">
+      <Tabs
+        value={activeTab}
+        onValueChange={(value) => setActiveTab(value as LogsTabId)}
+        className="min-h-0 flex-1 gap-0"
+      >
+        <TabsList variant="line" className="-mx-4 h-10 w-auto justify-start gap-4 border-b border-border px-4">
           {tabs.map((tab) => (
-            <TabsTrigger key={tab.id} value={tab.id} className="flex-none">
+            <TabsTrigger key={tab.id} value={tab.id} className="flex-none gap-1.5 px-0 text-[13px] font-normal">
               {tab.label}
+              {tab.isNew && (
+                <span className="rounded-sm border border-border px-1 text-[10px] leading-4 text-muted-foreground">
+                  New
+                </span>
+              )}
             </TabsTrigger>
           ))}
         </TabsList>