Merge 9928ec608f into 6fc9555f83

litellm/proxy/_types.py

@@ -3785,6 +3785,7 @@ class SpendLogsRouterMetadata(TypedDict):
 
 
 class SpendLogsMetadata(TypedDict):
+    agent_name: ReadOnly[NotRequired[str | None]]
     actor_agent_id: ReadOnly[NotRequired[str | None]]
     target_agent_id: ReadOnly[NotRequired[str | None]]
     billing_agent_id: ReadOnly[NotRequired[str | None]]
@@ -4583,6 +4584,7 @@ class LitellmMetadataFromRequestHeaders(TypedDict, total=False):
     """
 
     spend_logs_metadata: dict | None
+    agent_name: ReadOnly[str]
     agent_id: str | None
     trace_id: str | None
     session_id: str | None

litellm/proxy/litellm_pre_call_utils.py

@@ -1938,7 +1938,15 @@ class LiteLLMProxyRequestSetup:
         """
         from litellm.proxy._types import LitellmMetadataFromRequestHeaders
 
-        metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
+        normalized_headers: Final = MappingProxyType(
+            {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)}
+        )
+        agent_name_from_header: Final = normalized_headers.get("x-litellm-agent-name", "").strip()
+        metadata_from_headers: Final = (
+            LitellmMetadataFromRequestHeaders(agent_name=agent_name_from_header)
+            if agent_name_from_header
+            else LitellmMetadataFromRequestHeaders()
+        )
         request_body: Final = cast(Mapping[str, object], data)  # cast-ok: request body is a str-keyed JSON object
         _drop_client_session_metadata(request_body)
         spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
@@ -1949,12 +1957,9 @@ class LiteLLMProxyRequestSetup:
         # Finally update the requests metadata with the `metadata_from_headers`
         #########################################################################################
 
-        normalized_headers: Final = MappingProxyType(
-            {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)}
-        )
         agent_id_from_header: Final = headers.get("x-litellm-agent-id")
         # Explicit litellm headers take precedence; fall back to any x-*-session-id header.
-        chain_id: Final = get_chain_id_from_headers(dict(headers))
+        chain_id: Final = get_chain_id_from_headers(dict(normalized_headers))
         child_tier_metadata: Final = _child_tier_session_metadata(
             normalized_headers, request_body, _metadata_variable_name
         )

litellm/proxy/proxy_server.py

@@ -649,6 +649,7 @@ from litellm.proxy.config_resolvers.sso import SSO_FIELD_ENV_VARS
 from litellm.proxy.container_endpoints.endpoints import router as container_router
 from litellm.proxy.credential_endpoints.endpoints import router as credential_router
 from litellm.proxy.db.create_views import SupportsRawQueries
+from litellm.proxy.db.db_span import db_span
 from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
 from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
 from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
@@ -1041,6 +1042,7 @@ from litellm.proxy.utils import (  # noqa: F401, RUF100  # legacy module exports
     migrate_passwords_to_scrypt_async,
     model_dump_with_preserved_fields,
     prefetch_config_params,
+    query_raw_within_statement_timeout,
     update_spend,
 )
 from litellm.proxy.video_endpoints.endpoints import router as video_router
@@ -1067,7 +1069,9 @@ from litellm.secret_managers.main import (
     secret_manager_would_be_consulted,
     str_to_bool,
 )
+from litellm.tracing import TraceReceiver
 from litellm.tracing.config import is_lens_tracing_enabled
+from litellm.tracing.spend_logs import SpendLogTraceStorage
 from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs
 from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
 from litellm.types.llms.anthropic import (
@@ -1578,6 +1582,22 @@ def _forget_prisma_client() -> None:
     prisma_client = None  # rebind-ok: a closed client must not survive into the next lifespan entry
 
 
+@dataclass(frozen=True, slots=True)
+class _SpendLogReads:
+    prisma: PrismaClient
+
+    async def fetch(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
+        async with db_span("lens_spend_log_traces", "LiteLLM_SpendLogs", "select"):
+            return await query_raw_within_statement_timeout(self.prisma, query, *args, what="Lens traces")
+
+
+def _spend_log_trace_receiver(database: database_runtime.Database | None) -> TraceReceiver | None:
+    if database is None:
+        return None
+    reads: Final = _SpendLogReads(database.prisma)
+    return TraceReceiver(storage=SpendLogTraceStorage(lambda: reads))
+
+
 @asynccontextmanager
 async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
     with GracefulShutdownManager.capture_sigterm():
@@ -1951,11 +1971,11 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener
         TypeAdapter(dict[str, object] | None).validate_python(general_settings.get("tracing")),
     )
     tracing_enabled: Final = is_lens_tracing_enabled(tracing_settings)
-    async with manage_tracing(
-        enabled=tracing_enabled,
-        settings=tracing_settings,
-    ) as receiver:
-        state: Final[ProxyLifespanState] = {"tracing_receiver": receiver, "database": database}
+    async with manage_tracing(enabled=tracing_enabled, settings=tracing_settings):
+        state: Final[ProxyLifespanState] = {
+            "tracing_receiver": _spend_log_trace_receiver(database),
+            "database": database,
+        }
         from litellm.proxy.admin_mcp import admin_mcp_lifespan
 
         telemetry_settings: Final = load_telemetry_settings()

litellm/proxy/tracing_endpoints.py

@@ -27,8 +27,9 @@ from litellm.constants import (
     TRACE_READ_RETRY_AFTER_SECONDS,
 )
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_trace_read_scope
+from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_spend_log_read_scope, resolve_trace_read_scope
 from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency
+from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.http_parsing_utils import is_otlp_trace_request
 from litellm.proxy.tracing_runtime import provide_receiver, require_receiver
@@ -54,6 +55,7 @@ from litellm.tracing.generated.types import (
     TraceScope,
 )
 from litellm.tracing.otlp_http import encode_otlp_response
+from litellm.tracing.spend_logs import SpendLogTraceStorage
 from litellm.tracing.storage import LensTraceStorage
 from litellm.tracing.types import TraceAgentList
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -61,6 +63,7 @@ from litellm.types.llms.base import LiteLLMBaseModel
 router = APIRouter(tags=["agent tracing"])
 
 MS_PER_DAY: Final = 24 * 60 * 60 * 1000
+_SQL_NEEDS_LENS_DETAIL: Final = "Trace SQL queries need the Lens service. Configure LITELLM_LENS_URL."
 
 
 async def current_time_ms() -> int:
@@ -84,6 +87,10 @@ async def provide_trace_access(
     tracing: Annotated[TraceReceiver | None, Depends(provide_receiver)],
     log_team_lookup: LogTeamLookupDependency,
 ) -> TraceAccessContext:
+    if tracing is not None and isinstance(tracing.storage, SpendLogTraceStorage):
+        RouteChecks.should_call_route(route="/spend/logs/ui", valid_token=auth)
+        spend_scope: Final = await resolve_spend_log_read_scope(auth, log_team_lookup)
+        return TraceAccessContext(tracing, spend_scope if auth.user_id or isinstance(spend_scope, AllRows) else None)
     read_scope: Final = await resolve_trace_read_scope(auth, partial(log_team_lookup, auth))
     return TraceAccessContext(tracing, read_scope)
 
@@ -250,6 +257,8 @@ async def provide_trace_query_access(
     log_team_lookup: LogTeamLookupDependency,
 ) -> TraceQueryAccess:
     storage: Final = require_receiver(tracing).storage
+    if isinstance(storage, SpendLogTraceStorage):
+        raise HTTPException(status_code=501, detail=_SQL_NEEDS_LENS_DETAIL)
     scope: Final = await resolve_trace_read_scope(auth, partial(log_team_lookup, auth))
     if scope is None:
         raise HTTPException(status_code=403, detail="Not allowed to view logs")

litellm/proxy/tracing_runtime.py

@@ -31,7 +31,7 @@ async def provide_receiver(request: Request) -> TraceReceiver | None:
 
 async def provide_storage(request: Request) -> LensTraceStorage | None:
     tracing: Final = await provide_receiver(request)
-    return tracing.storage if tracing is not None else None
+    return tracing.storage if tracing is not None and isinstance(tracing.storage, LensTraceStorage) else None
 
 
 @asynccontextmanager

litellm/tracing/AGENTS.md

@@ -1,7 +1,7 @@
 # Lens gateway boundary
 
-- Lens owns trace normalization, schemas, storage, graph assembly and querying in `BerriAI/lens`
-- This package owns the HTTP client, gateway response validation and asynchronous gateway spend export. Keep the trace reader remote-only
+- Lens owns the canonical trace schemas and its service's normalization, storage, graph assembly and querying in `BerriAI/lens`
+- This package owns the HTTP client, gateway response validation, asynchronous gateway spend export and the read-only Postgres spend-log adapter. Trace reads always come from spend logs, even when a Lens service is configured, using existing spend-log permissions. It does not ingest OTLP or implement Lens analysis
 - Gateway endpoints preserve authenticated user/team scope, trace references, pagination and public error categories
 - `generated/` comes from `scripts/generate_trace_types.py` using the Lens schemas pinned in `scripts/lens_assets/source.json`. Update the canonical Lens Rust contracts and import their schema outputs before regenerating. Never edit generated Python
 - Keep the pinned schema and fixture checksums current. CI verifies assets and regenerated Python without a Rust trace implementation in this repository

litellm/tracing/receiver.py

@@ -4,12 +4,13 @@ from typing import Final
 from litellm.constants import AGENT_TRACING_AGENT_LIST_LIMIT, AGENT_TRACING_LIST_PAGE_SIZE
 from litellm.tracing.generated.models import TraceAgentsParams
 from litellm.tracing.generated.types import SpanDetail, SpanErrorPage, Trace, TracePage, TraceScope
+from litellm.tracing.spend_logs import SpendLogTraceStorage
 from litellm.tracing.storage import LensTraceStorage
 from litellm.tracing.types import TraceAgent, TraceAgentList
 
 
 class TraceReceiver:
-    def __init__(self, storage: LensTraceStorage) -> None:
+    def __init__(self, storage: LensTraceStorage | SpendLogTraceStorage) -> None:
         self.storage: Final = storage
 
     async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage:

litellm/tracing/spend_logs.py

@@ -0,0 +1,837 @@
+"""Agent traces read from LiteLLM_SpendLogs, so Lens works without its own trace store.
+
+A trace is every spend log row sharing a session id. A row's agent comes from an `agent:<name>` tag (with an optional
+`parent_agent:<name>` tag), the row's agent_id, a Claude Agent SDK subagent whose first prompt matches a parent's
+Task/Agent tool call, or the calling framework read from the user agent."""
+
+import asyncio
+import base64
+import hashlib
+import json
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from itertools import chain, groupby, islice
+from typing import Final, Protocol, TypeAlias
+
+from pydantic import TypeAdapter
+from typing_extensions import NotRequired, ReadOnly, TypedDict
+
+from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE
+from litellm.litellm_core_utils.prompt_templates.factory import (
+    get_tool_calls_from_response,
+    resolve_structured_messages,
+)
+from litellm.tracing.errors import TraceChanged
+from litellm.tracing.generated.models import TraceAgentRow, TraceAgentsParams
+from litellm.tracing.generated.types import (
+    AgentNode,
+    ChatRole,
+    Span,
+    SpanDetail,
+    SpanErrorPage,
+    SpanStatus,
+    Trace,
+    TracePage,
+    TraceScope,
+    TraceSummary,
+    UIFields,
+    UIMessage,
+    UIMessages,
+    UIText,
+    UIToolCall,
+)
+
+AGENT_TAG_PREFIX: Final = "agent:"
+PARENT_AGENT_TAG_PREFIX: Final = "parent_agent:"
+SPEND_LOGS_SERVICE: Final = "litellm"
+TRACE_ROW_LIMIT: Final = 10_000
+TRACE_SPAN_PAGE_SIZE: Final = 500
+AGENT_LIST_TRACE_PAGE_SIZE: Final = 200
+PREVIEW_CHARS: Final = 300
+ERROR_PREVIEW_CHARS: Final = 500
+TRACE_KEY_SQL: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
+FRAMEWORK_USER_AGENTS: Final = (
+    ("agent-sdk", "claude-agent-sdk"),
+    ("claude-cli", "claude-code"),
+    ("pydantic-ai", "pydantic-ai"),
+    ("openai-agents", "openai-agents"),
+    ("langchain", "langchain"),
+    ("langgraph", "langgraph"),
+    ("crewai", "crewai"),
+    ("codex", "codex"),
+)
+AGENT_USER_AGENT_SQL: Final = "|".join(pattern for pattern, _ in FRAMEWORK_USER_AGENTS)
+DEFAULT_FRAMEWORK: Final = "litellm"
+SUBAGENT_TOOL_NAMES: Final = frozenset(("Task", "Agent"))
+
+
+class SpendLogReader(Protocol):
+    async def fetch(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
+
+
+class _TraceKeyRow(TypedDict):
+    trace_key: ReadOnly[str]
+    last_activity: ReadOnly[str]
+
+
+class SpendLogRow(TypedDict):
+    request_id: ReadOnly[str]
+    trace_key: ReadOnly[str]
+    call_type: ReadOnly[str]
+    model: ReadOnly[str]
+    status: ReadOnly[str | None]
+    spend: ReadOnly[float]
+    prompt_tokens: ReadOnly[int]
+    completion_tokens: ReadOnly[int]
+    start_time: ReadOnly[str]
+    end_time: ReadOnly[str]
+    agent_id: ReadOnly[str | None]
+    agent_name: ReadOnly[NotRequired[str | None]]
+    request_tags: ReadOnly[str | None]
+    metadata: ReadOnly[str | None]
+    messages: ReadOnly[str | None]
+    proxy_server_request: ReadOnly[str | None]
+    response: ReadOnly[str | None]
+
+
+_TRACE_KEY_ROWS: Final = TypeAdapter(tuple[_TraceKeyRow, ...])
+_SPEND_LOG_ROWS: Final = TypeAdapter(tuple[SpendLogRow, ...])
+_JSON: Final = TypeAdapter(object)
+_OBJECT_MAPPING: Final = TypeAdapter(dict[str, object])
+_OBJECT_SEQUENCE: Final = TypeAdapter(tuple[object, ...])
+
+
+@dataclass(frozen=True, slots=True)
+class _ToolCall:
+    id: str | None
+    name: str
+    arguments: str
+    subagent: str | None
+    subagent_prompt: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class _Call:
+    row: SpendLogRow
+    start_ms: float
+    end_ms: float
+    framework: str
+    explicit_agent: str | None
+    explicit_parent: str | None
+    prompts: frozenset[str]
+    input_messages: tuple[UIMessage, ...]
+    output_text: str
+    tool_calls: tuple[_ToolCall, ...]
+    error: str | None
+
+    @property
+    def failed(self) -> bool:
+        return self.row["status"] == "failure" or self.error is not None
+
+
+@dataclass(frozen=True, slots=True)
+class _Placed:
+    call: _Call
+    agent: str
+    parent_agent: str | None
+
+
+def _loads(raw: str | None) -> object:
+    if not raw:
+        return None
+    try:
+        return _JSON.validate_python(json.loads(raw))
+    except ValueError:
+        return None
+
+
+def _mapping(value: object) -> Mapping[str, object]:
+    return _OBJECT_MAPPING.validate_python(value) if isinstance(value, dict) else {}
+
+
+def _sequence(value: object) -> Sequence[object]:
+    return _OBJECT_SEQUENCE.validate_python(value) if isinstance(value, (list, tuple)) else ()
+
+
+def _text(value: object) -> str:
+    return value if isinstance(value, str) else ""
+
+
+def _parse_ms(timestamp: str) -> float:
+    parsed: Final = datetime.fromisoformat(timestamp)
+    return (parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)).timestamp() * 1000
+
+
+def _iso(ms: float) -> str:
+    return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _framework(user_agent: str) -> str:
+    lowered: Final = user_agent.lower()
+    return next((name for pattern, name in FRAMEWORK_USER_AGENTS if pattern in lowered), DEFAULT_FRAMEWORK)
+
+
+def _tag_value(tags: Sequence[object], prefix: str) -> str | None:
+    return next(
+        (tag[len(prefix) :].strip() for tag in tags if isinstance(tag, str) and tag.startswith(prefix)),
+        None,
+    )
+
+
+def _text_blocks(content: object) -> tuple[str, ...]:
+    if isinstance(content, str):
+        return (content,)
+    return tuple(
+        _text(block.get("text"))
+        for block in map(_mapping, _sequence(content))
+        if block.get("type") in ("text", "input_text", "output_text")
+    )
+
+
+def _block_texts(block: Mapping[str, object]) -> tuple[str, ...]:
+    if block.get("type") == "tool_result":
+        return _text_blocks(block.get("content"))
+    return _text_blocks((block,))
+
+
+def _content_texts(content: object) -> tuple[str, ...]:
+    if isinstance(content, str):
+        return (content,)
+    blocks: Final = chain.from_iterable(_block_texts(block) for block in map(_mapping, _sequence(content)))
+    return tuple(text for text in blocks if text)
+
+
+def _visible_text(content: object) -> str:
+    return "\n".join(text for text in _content_texts(content) if not text.lstrip().startswith("<system-reminder>"))
+
+
+def _role(value: object) -> ChatRole:
+    match value:
+        case "system":
+            return "system"
+        case "assistant":
+            return "assistant"
+        case "tool":
+            return "tool"
+        case _:
+            return "user"
+
+
+def _message_tool_calls(message: Mapping[str, object]) -> tuple[UIToolCall, ...]:
+    response: Final = {"content": message.get("content"), "choices": [{"message": message}]}
+    return tuple(UIToolCall(name=call.name, arguments=call.arguments) for call in _response_tool_calls(response))
+
+
+def _ui_messages(messages: object) -> tuple[UIMessage, ...]:
+    return tuple(
+        UIMessage(
+            role=_role(_mapping(message).get("role")),
+            content=_visible_text(_mapping(message).get("content")),
+            tool_calls=_message_tool_calls(_mapping(message)),
+        )
+        for message in _sequence(messages)
+    )
+
+
+def _first_user_prompts(messages: object) -> frozenset[str]:
+    first_user: Final = next(
+        (_mapping(message) for message in _sequence(messages) if _mapping(message).get("role") == "user"), None
+    )
+    if first_user is None:
+        return frozenset()
+    return frozenset(text.strip() for text in _content_texts(first_user.get("content")) if text.strip())
+
+
+def _response_tool_calls(response: Mapping[str, object]) -> tuple[_ToolCall, ...]:
+    return tuple(
+        _ToolCall(
+            id=call["id"],
+            name=call["name"],
+            arguments=json.dumps(call["arguments"]),
+            subagent=_text(call["arguments"].get("subagent_type")) or None
+            if call["name"] in SUBAGENT_TOOL_NAMES
+            else None,
+            subagent_prompt=_text(call["arguments"].get("prompt")).strip() or None
+            if call["name"] in SUBAGENT_TOOL_NAMES
+            else None,
+        )
+        for call in get_tool_calls_from_response(response)
+        if call["name"]
+    )
+
+
+def _response_text(response: Mapping[str, object]) -> str:
+    anthropic: Final = _visible_text(response.get("content"))
+    openai: Final = "\n".join(
+        _visible_text(_mapping(choice.get("message")).get("content"))
+        for choice in map(_mapping, _sequence(response.get("choices")))
+    )
+    responses_api: Final = "\n".join(
+        _visible_text(item.get("content")) for item in map(_mapping, _sequence(response.get("output")))
+    )
+    return next((text for text in (anthropic, openai.strip(), responses_api.strip()) if text), "")
+
+
+def _request_messages(row: SpendLogRow) -> object:
+    stored: Final = _loads(row["messages"])
+    if _sequence(stored):
+        return stored
+    request: Final = _mapping(_loads(row["proxy_server_request"]))
+    body: Final = _mapping(request.get("body")) if "body" in request else request
+    messages: Final = _sequence(body.get("messages"))
+    if messages:
+        return messages
+    return resolve_structured_messages(None, dict(body)) if body.get("input") else messages
+
+
+def _call(row: SpendLogRow) -> _Call:
+    metadata: Final = _mapping(_loads(row["metadata"]))
+    tags: Final = _sequence(_loads(row["request_tags"]))
+    messages: Final = _request_messages(row)
+    response: Final = _mapping(_loads(row["response"]))
+    error_message: Final = _text(_mapping(metadata.get("error_information")).get("error_message"))
+    return _Call(
+        row=row,
+        start_ms=_parse_ms(row["start_time"]),
+        end_ms=_parse_ms(row["end_time"]),
+        framework=_framework(_text(metadata.get("user_agent"))),
+        explicit_agent=_text(metadata.get("agent_name")).strip()
+        or _tag_value(tags, AGENT_TAG_PREFIX)
+        or row.get("agent_name")
+        or row["agent_id"]
+        or None,
+        explicit_parent=_tag_value(tags, PARENT_AGENT_TAG_PREFIX),
+        prompts=_first_user_prompts(messages),
+        input_messages=_ui_messages(messages),
+        output_text=_response_text(response),
+        tool_calls=_response_tool_calls(response),
+        error=error_message or ("Request failed" if row["status"] == "failure" else None),
+    )
+
+
+def _root_agent(calls: Sequence[_Call]) -> str:
+    return next((call.framework for call in calls if call.framework != DEFAULT_FRAMEWORK), "agent")
+
+
+_Spawner: TypeAlias = tuple[int, _ToolCall, _Placed]
+
+
+def _place_call(call: _Call, spawners: Mapping[str, _Spawner], root: str) -> _Placed:
+    matches: Final = tuple(spawners[prompt] for prompt in call.prompts if prompt in spawners)
+    spawner: Final = max(matches, key=lambda match: match[0]) if matches else None
+    parent: Final = call.explicit_parent if call.explicit_parent != call.explicit_agent else None
+    inferred: Final = spawner is not None and call.explicit_agent in (None, spawner[2].call.explicit_agent)
+    return _Placed(
+        call=call,
+        agent=(spawner[1].subagent or root) if inferred and spawner else (call.explicit_agent or root),
+        parent_agent=spawner[2].agent if inferred and spawner else parent,
+    )
+
+
+def _placements(calls: Sequence[_Call]) -> Iterator[_Placed]:
+    root: Final = _root_agent(calls)
+    spawners: Final[dict[str, _Spawner]] = {}  # mutable-ok: rolling index avoids quadratic scans
+    for index, placed in ((index, _place_call(call, spawners, root)) for index, call in enumerate(calls)):
+        yield placed
+        for tool in reversed(placed.call.tool_calls):
+            if tool.subagent and tool.subagent_prompt:
+                spawners[tool.subagent_prompt] = (index, tool, placed)
+
+
+def _place(calls: Sequence[_Call]) -> tuple[_Placed, ...]:
+    return tuple(_placements(calls))
+
+
+def _agent_span_id(agent: str) -> str:
+    return f"agent:{agent}"
+
+
+def _tool_span_id(call: _Call, index: int) -> str:
+    return f"{call.row['request_id']}:tool:{index}"
+
+
+def _preview(messages: Sequence[UIMessage]) -> str:
+    return next(
+        (message["content"] for message in reversed(messages) if message["role"] == "user" and message["content"]), ""
+    )[:PREVIEW_CHARS]
+
+
+@dataclass(frozen=True, slots=True)
+class _AgentRun:
+    name: str
+    parent: str | None
+    placed: tuple[_Placed, ...]
+    start_ms: float
+    end_ms: float
+
+
+def _agent_run(name: str, own: tuple[_Placed, ...]) -> _AgentRun:
+    return _AgentRun(
+        name=name,
+        parent=next((item.parent_agent for item in own if item.parent_agent), None),
+        placed=own,
+        start_ms=min(item.call.start_ms for item in own),
+        end_ms=max(item.call.end_ms for item in own),
+    )
+
+
+def _agent_runs(placed: Sequence[_Placed]) -> tuple[_AgentRun, ...]:
+    groups: Final = {
+        name: tuple(items)
+        for name, items in groupby(sorted(placed, key=lambda item: item.agent), key=lambda item: item.agent)
+    }
+    names: Final = tuple(dict.fromkeys(item.agent for item in placed))
+    return tuple(_agent_run(name, groups[name]) for name in names)
+
+
+def _next_start(placed: Sequence[_Placed], index: int) -> float | None:
+    current: Final = placed[index]
+    return next(
+        (item.call.start_ms for item in islice(placed, index + 1, None) if item.agent == current.agent),
+        None,
+    )
+
+
+def _status(failed: bool) -> SpanStatus:
+    return "error" if failed else "ok"
+
+
+def _llm_span(item: _Placed, trace_start: float) -> Span:
+    call: Final = item.call
+    error: Final = call.error
+    return Span(
+        span_id=call.row["request_id"],
+        parent_span_id=_agent_span_id(item.agent),
+        name=call.row["model"] or call.row["call_type"],
+        type="llm",
+        agent=item.agent,
+        framework=call.framework,
+        model=call.row["model"] or None,
+        start_offset_ms=call.start_ms - trace_start,
+        duration_ms=max(call.end_ms - call.start_ms, 0.0),
+        input_preview=_preview(call.input_messages),
+        input_tokens=call.row["prompt_tokens"],
+        output_tokens=call.row["completion_tokens"],
+        spend=call.row["spend"],
+        status=_status(call.failed),
+        error=error[:ERROR_PREVIEW_CHARS] if error else None,
+        error_truncated=error is not None and len(error) > ERROR_PREVIEW_CHARS,
+        litellm_request_id=call.row["request_id"],
+        spend_log_request_id=call.row["request_id"],
+        spend_match="matched",
+    )
+
+
+def _tool_spans(placed: Sequence[_Placed], index: int, trace_start: float) -> tuple[Span, ...]:
+    item: Final = placed[index]
+    call: Final = item.call
+    if not call.tool_calls:
+        return ()
+    resumed_at: Final = _next_start(placed, index)
+    duration: Final = max((resumed_at if resumed_at is not None else call.end_ms) - call.end_ms, 0.0)
+    return tuple(
+        Span(
+            span_id=_tool_span_id(call, position),
+            parent_span_id=call.row["request_id"],
+            name=f"{tool.name} ({tool.subagent})" if tool.subagent else tool.name,
+            type="agent" if tool.subagent else "tool",
+            agent=item.agent,
+            framework=call.framework,
+            model=None,
+            start_offset_ms=call.end_ms - trace_start,
+            duration_ms=duration,
+            input_preview=tool.arguments[:PREVIEW_CHARS],
+            input_tokens=0,
+            output_tokens=0,
+            spend=None,
+            status="ok",
+            error=None,
+            error_truncated=False,
+            litellm_request_id=None,
+            spend_log_request_id=None,
+            spend_match=None,
+        )
+        for position, tool in enumerate(call.tool_calls)
+    )
+
+
+def _agent_span(run: _AgentRun, trace_start: float) -> Span:
+    calls: Final = tuple(item.call for item in run.placed)
+    return Span(
+        span_id=_agent_span_id(run.name),
+        parent_span_id=_agent_span_id(run.parent) if run.parent else None,
+        name=run.name,
+        type="agent",
+        agent=run.name,
+        framework=calls[0].framework,
+        model=None,
+        start_offset_ms=run.start_ms - trace_start,
+        duration_ms=run.end_ms - run.start_ms,
+        input_preview=_preview(calls[0].input_messages),
+        input_tokens=sum(call.row["prompt_tokens"] for call in calls),
+        output_tokens=sum(call.row["completion_tokens"] for call in calls),
+        spend=sum(call.row["spend"] for call in calls),
+        status=_status(any(call.failed for call in calls)),
+        error=None,
+        error_truncated=False,
+        litellm_request_id=None,
+        spend_log_request_id=None,
+        spend_match=None,
+    )
+
+
+def _agent_node(run: _AgentRun) -> AgentNode:
+    calls: Final = tuple(item.call for item in run.placed)
+    return AgentNode(
+        name=run.name,
+        parent_agent=run.parent,
+        duration_ms=run.end_ms - run.start_ms,
+        invocations=1,
+        llm_calls=len(calls),
+        priced_calls=sum(1 for call in calls if call.row["spend"] > 0),
+        spend=sum(call.row["spend"] for call in calls),
+        tool_calls=sum(len(call.tool_calls) for call in calls),
+    )
+
+
+def build_trace(rows: Sequence[SpendLogRow]) -> Trace:
+    calls: Final = tuple(sorted(map(_call, rows), key=lambda call: (call.start_ms, call.row["request_id"])))
+    placed: Final = _place(calls)
+    runs: Final = _agent_runs(placed)
+    trace_start: Final = calls[0].start_ms
+    trace_end: Final = max(call.end_ms for call in calls)
+    spans: Final = (
+        *(_agent_span(run, trace_start) for run in runs),
+        *chain.from_iterable(
+            (_llm_span(item, trace_start), *_tool_spans(placed, index, trace_start))
+            for index, item in enumerate(placed)
+        ),
+    )
+    errors: Final = sum(1 for call in calls if call.failed)
+    root: Final = next((run for run in runs if run.parent is None), runs[0])
+    return Trace(
+        agents=tuple(_agent_node(run) for run in runs),
+        spans=spans,
+        next_cursor=None,
+        summary=TraceSummary(
+            trace_id=calls[0].row["trace_key"],
+            name=root.name,
+            service=SPEND_LOGS_SERVICE,
+            start_time=_iso(trace_start),
+            duration_ms=trace_end - trace_start,
+            status=_status(errors > 0),
+            error_count=errors,
+            span_count=len(spans),
+            llm_calls=len(calls),
+            tool_calls=sum(len(call.tool_calls) for call in calls),
+            priced_calls=sum(1 for call in calls if call.row["spend"] > 0),
+            spend=sum(call.row["spend"] for call in calls),
+            input_tokens=sum(call.row["prompt_tokens"] for call in calls),
+            output_tokens=sum(call.row["completion_tokens"] for call in calls),
+            models=tuple(dict.fromkeys(call.row["model"] for call in calls if call.row["model"])),
+            agent_count=len(runs),
+            agent_invocations=len(runs),
+            agent_names=tuple(run.name for run in runs),
+            frameworks=tuple(dict.fromkeys(call.framework for call in calls)),
+            input_preview=_preview(calls[0].input_messages),
+        ),
+    )
+
+
+def _tool_result(messages: object, tool_id: str) -> str | None:
+    for message in map(_mapping, _sequence(messages)):
+        if message.get("role") == "tool" and message.get("tool_call_id") == tool_id:
+            return _visible_text(message.get("content"))
+        for block in map(_mapping, _sequence(message.get("content"))):
+            if block.get("type") == "tool_result" and block.get("tool_use_id") == tool_id:
+                return _visible_text(block.get("content"))
+    return None
+
+
+def _tool_output(calls: Sequence[_Call], tool: _ToolCall) -> str:
+    if tool.id is None:
+        return "Tool result was not recorded in the spend logs."
+    return next(
+        (result for call in calls if (result := _tool_result(_request_messages(call.row), tool.id)) is not None),
+        "Tool result was not recorded in the spend logs.",
+    )
+
+
+def _span_detail(trace_rows: Sequence[SpendLogRow], span_id: str) -> SpanDetail | None:
+    calls: Final = tuple(sorted(map(_call, trace_rows), key=lambda call: (call.start_ms, call.row["request_id"])))
+    llm: Final = next((call for call in calls if call.row["request_id"] == span_id), None)
+    if llm is not None:
+        return SpanDetail(
+            span_id=span_id,
+            attributes={
+                "litellm.request_id": llm.row["request_id"],
+                "litellm.call_type": llm.row["call_type"],
+                "litellm.status": llm.row["status"] or "success",
+                "gen_ai.request.model": llm.row["model"],
+                "agent.framework": llm.framework,
+            },
+            input=json.dumps(_request_messages(llm.row)),
+            input_ui=UIMessages(kind="messages", messages=llm.input_messages),
+            output=llm.output_text,
+            output_ui=UIText(kind="text", text=llm.output_text or (llm.error or "")),
+        )
+    tool: Final = next(
+        (
+            tool
+            for call, position, tool in chain.from_iterable(
+                ((call, position, tool) for position, tool in enumerate(call.tool_calls)) for call in calls
+            )
+            if _tool_span_id(call, position) == span_id
+        ),
+        None,
+    )
+    if tool is not None:
+        return SpanDetail(
+            span_id=span_id,
+            attributes={"tool.name": tool.name},
+            input=tool.arguments,
+            input_ui=UIText(kind="text", text=tool.arguments),
+            output=_tool_output(calls, tool),
+            output_ui=UIText(kind="text", text=_tool_output(calls, tool)),
+        )
+    trace: Final = build_trace(trace_rows)
+    agent: Final = next((span for span in trace["spans"] if span["span_id"] == span_id), None)
+    if agent is None:
+        return None
+    return SpanDetail(
+        span_id=span_id,
+        attributes={"agent.name": agent["agent"], "agent.framework": agent["framework"]},
+        input=agent["input_preview"],
+        input_ui=UIText(kind="text", text=agent["input_preview"]),
+        output="",
+        output_ui=UIFields(
+            kind="fields",
+            fields=(
+                {
+                    "key": "LLM calls",
+                    "value": str(
+                        sum(1 for span in trace["spans"] if span["agent"] == agent["agent"] and span["type"] == "llm")
+                    ),
+                },
+            ),
+        ),
+    )
+
+
+def _trace_page(trace: Trace, cursor: str | None, page_size: int | None) -> Trace:
+    snapshot: Final = hashlib.sha256(json.dumps(trace, sort_keys=True).encode()).hexdigest()
+    decoded: Final = TypeAdapter(tuple[str, int]).validate_json(base64.urlsafe_b64decode(cursor)) if cursor else None
+    if decoded is not None and decoded[0] != snapshot:
+        raise TraceChanged("The trace changed. Reload it to continue.")
+    offset: Final = decoded[1] if decoded else 0
+    if offset < 0 or offset > len(trace["spans"]):
+        raise ValueError("Invalid span cursor")
+    size: Final = page_size if page_size is not None else TRACE_SPAN_PAGE_SIZE
+    if size < 1 or size > TRACE_SPAN_PAGE_SIZE:
+        raise ValueError("Invalid span page size")
+    end: Final = offset + size
+    next_cursor: Final = (
+        base64.urlsafe_b64encode(json.dumps((snapshot, end)).encode()).decode() if end < len(trace["spans"]) else None
+    )
+    return Trace(
+        summary=trace["summary"], agents=trace["agents"], spans=trace["spans"][offset:end], next_cursor=next_cursor
+    )
+
+
+def _encode_cursor(row: _TraceKeyRow) -> str:
+    return base64.urlsafe_b64encode(json.dumps((row["last_activity"], row["trace_key"])).encode()).decode()
+
+
+def _decode_cursor(cursor: str | None) -> tuple[str, str] | None:
+    if not cursor:
+        return None
+    decoded: Final = TypeAdapter(tuple[str, str]).validate_json(base64.urlsafe_b64decode(cursor.encode()))
+    return decoded
+
+
+def _scope_sql(scope: TraceScope, first_index: int) -> tuple[str, tuple[object, ...]]:
+    if scope["all_teams"] == 1:
+        return "", ()
+    return (
+        f'AND ("user" = ${first_index} OR team_id = ANY(${first_index + 1}::text[]))',
+        (scope["user_id"], list(scope["team_ids"])),
+    )
+
+
+class SpendLogTraceStorage:
+    def __init__(self, database: Callable[[], SpendLogReader | None]) -> None:
+        self._database: Final = database
+
+    async def _query(self, sql: str, *args: object) -> Sequence[Mapping[str, object]]:
+        db: Final = self._database()
+        if db is None:
+            return ()
+        return await db.fetch(sql, *args)
+
+    async def _trace_keys(
+        self, scope: TraceScope, start_ms: int, end_ms: int, cursor: tuple[str, str] | None, limit: int
+    ) -> tuple[_TraceKeyRow, ...]:
+        scope_sql, scope_args = _scope_sql(scope, 3)
+        cursor_index: Final = 3 + len(scope_args)
+        after: Final = (
+            f"WHERE (last_activity, trace_key) < (${cursor_index}, ${cursor_index + 1})" if cursor is not None else ""
+        )
+        rows: Final = await self._query(
+            f"""
+            SELECT trace_key, last_activity FROM (
+                SELECT {TRACE_KEY_SQL} AS trace_key, MAX("startTime")::text AS last_activity
+                FROM "LiteLLM_SpendLogs"
+                WHERE "startTime" >= to_timestamp($1 / 1000.0) AT TIME ZONE 'UTC'
+                  AND "startTime" < to_timestamp($2 / 1000.0) AT TIME ZONE 'UTC'
+                  {scope_sql}
+                GROUP BY 1
+                HAVING COUNT(*) > 1
+                    OR bool_or(COALESCE(metadata->>'agent_name', '') <> '')
+                    OR bool_or(COALESCE(agent_id, '') <> '')
+                    OR bool_or(COALESCE(request_tags::text, '') LIKE '%"{AGENT_TAG_PREFIX}%')
+                    OR bool_or(COALESCE(metadata->>'user_agent', '') ~* '{AGENT_USER_AGENT_SQL}')
+            ) AS traces
+            {after}
+            ORDER BY last_activity DESC, trace_key DESC
+            LIMIT {limit}
+            """,
+            start_ms,
+            end_ms,
+            *scope_args,
+            *(cursor if cursor is not None else ()),
+        )
+        return _TRACE_KEY_ROWS.validate_python(rows)
+
+    async def _rows(self, trace_keys: Sequence[str], scope: TraceScope) -> tuple[SpendLogRow, ...]:
+        if not trace_keys:
+            return ()
+        scope_sql, scope_args = _scope_sql(scope, 2)
+        rows: Final = await self._query(
+            f"""
+            SELECT request_id, {TRACE_KEY_SQL} AS trace_key, call_type, model, status, spend, prompt_tokens,
+                   completion_tokens, "startTime"::text AS start_time, "endTime"::text AS end_time, agent_id,
+                   (SELECT agent_name FROM "LiteLLM_AgentsTable" a
+                    WHERE a.agent_id = "LiteLLM_SpendLogs".agent_id) AS agent_name,
+                   request_tags::text AS request_tags, metadata::text AS metadata, messages::text AS messages,
+                   proxy_server_request::text AS proxy_server_request, response::text AS response
+            FROM "LiteLLM_SpendLogs"
+            WHERE (session_id = ANY($1::text[]) OR request_id = ANY($1::text[]))
+              AND {TRACE_KEY_SQL} = ANY($1::text[])
+              {scope_sql}
+            ORDER BY "startTime", request_id
+            LIMIT {TRACE_ROW_LIMIT + 1}
+            """,
+            list(trace_keys),
+            *scope_args,
+        )
+        if len(rows) > TRACE_ROW_LIMIT:
+            raise OverflowError("Spend-log trace read exceeds its row budget")
+        return _SPEND_LOG_ROWS.validate_python(rows)
+
+    async def _traces(self, keys: Sequence[str], scope: TraceScope) -> tuple[Trace, ...]:
+        rows: Final = await self._rows(keys, scope)
+        return await asyncio.to_thread(_build_traces, keys, rows)
+
+    async def list_traces(
+        self,
+        scope: TraceScope,
+        start_ms: int,
+        end_ms: int,
+        cursor: str | None = None,
+        limit: int = AGENT_TRACING_LIST_PAGE_SIZE,
+    ) -> TracePage:
+        keys: Final = await self._trace_keys(scope, start_ms, end_ms, _decode_cursor(cursor), limit + 1)
+        page: Final = keys[:limit]
+        traces: Final = await self._traces(tuple(row["trace_key"] for row in page), scope)
+        return TracePage(
+            data=tuple(trace["summary"] for trace in traces),
+            next_cursor=_encode_cursor(page[-1]) if len(keys) > limit else None,
+        )
+
+    async def get_trace(
+        self,
+        trace_id: str,
+        scope: TraceScope,
+        trace_ref: str = "",
+        cursor: str | None = None,
+        page_size: int | None = None,
+    ) -> Trace | None:
+        traces: Final = await self._traces((trace_id,), scope)
+        if not traces:
+            return None
+        return await asyncio.to_thread(_trace_page, traces[0], cursor, page_size)
+
+    async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None:
+        rows: Final = await self._rows((trace_id,), scope)
+        return await asyncio.to_thread(_span_detail, rows, span_id) if rows else None
+
+    async def get_span_error(
+        self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "", cursor: str | None = None
+    ) -> SpanErrorPage | None:
+        rows: Final = await self._rows((trace_id,), scope)
+        call: Final = next((_call(row) for row in rows if row["request_id"] == span_id), None)
+        if call is None or call.error is None:
+            return None
+        return SpanErrorPage(span_id=span_id, message=call.error, next_cursor=None, total_chars=len(call.error))
+
+    async def trace_agents(self, parameters: TraceAgentsParams) -> tuple[TraceAgentRow, ...]:
+        scope: Final = TraceScope(
+            all_teams=parameters.all_teams, user_id=parameters.user_id, team_ids=parameters.team_ids
+        )
+        agents: Final[dict[str, TraceAgentRow]] = {}  # mutable-ok: aggregate pages without retaining graphs
+        cursor: tuple[str, str] | None = None  # rebind-ok: advance through database pages
+        while True:
+            keys = await self._trace_keys(
+                scope, parameters.start_ms, parameters.end_ms, cursor, AGENT_LIST_TRACE_PAGE_SIZE
+            )
+            traces = await self._traces(tuple(row["trace_key"] for row in keys), scope)
+            page_agents = await asyncio.to_thread(_agent_rows, traces)
+            for agent in page_agents:
+                previous = agents.get(agent.agent_name)
+                agents[agent.agent_name] = TraceAgentRow(
+                    agent_name=agent.agent_name,
+                    runs=agent.runs + (previous.runs if previous else 0),
+                    failed_runs=agent.failed_runs + (previous.failed_runs if previous else 0),
+                    last_seen_ms=max(agent.last_seen_ms, previous.last_seen_ms if previous else 0),
+                    frameworks=tuple(dict.fromkeys((*agent.frameworks, *(previous.frameworks if previous else ())))),
+                )
+            if len(keys) < AGENT_LIST_TRACE_PAGE_SIZE:
+                return tuple(sorted(agents.values(), key=lambda agent: (-agent.last_seen_ms, agent.agent_name)))[
+                    : parameters.limit
+                ]
+            cursor = (
+                keys[-1]["last_activity"],
+                keys[-1]["trace_key"],
+            )
+
+
+def _build_traces(keys: Sequence[str], rows: Sequence[SpendLogRow]) -> tuple[Trace, ...]:
+    groups: Final = {
+        key: tuple(own)
+        for key, own in groupby(sorted(rows, key=lambda row: row["trace_key"]), key=lambda row: row["trace_key"])
+    }
+    return tuple(build_trace(groups[key]) for key in keys if key in groups)
+
+
+def _agent_rows(traces: tuple[Trace, ...]) -> tuple[TraceAgentRow, ...]:
+    names: Final = tuple(
+        dict.fromkeys(chain.from_iterable(trace["summary"].get("agent_names", ()) for trace in traces))
+    )
+    return tuple(
+        _agent_row(name, tuple(trace for trace in traces if name in trace["summary"].get("agent_names", ())))
+        for name in names
+    )
+
+
+def _agent_row(name: str, own: tuple[Trace, ...]) -> TraceAgentRow:
+    spans: Final = tuple(span for span in chain.from_iterable(trace["spans"] for trace in own) if span["agent"] == name)
+    return TraceAgentRow(
+        agent_name=name,
+        runs=len(own),
+        failed_runs=sum(
+            1 for trace in own if any(span["status"] == "error" and span["agent"] == name for span in trace["spans"])
+        ),
+        last_seen_ms=int(max(_parse_ms(trace["summary"]["start_time"].replace("Z", "")) for trace in own)),
+        frameworks=tuple(dict.fromkeys(span["framework"] for span in spans)),
+    )

litellm/types/utils.py

@@ -3499,6 +3499,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
     Specific metadata k,v pairs logged to integration for easier cost tracking and prompt management
     """
 
+    agent_name: ReadOnly[NotRequired[str | None]]
     spend_logs_metadata: dict | None  # special param to log k,v pairs to spendlogs for a call
     requester_ip_address: str | None
     user_agent: str | None

tests/unit/integrations/gcs_pubsub/spend_logs_payload.json

@@ -11,7 +11,7 @@
     "user": "",
     "team_id": "",
     "organization_id": "",
-    "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
+    "metadata": "{\"agent_name\": null, \"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
     "cache_key": "Cache OFF",
     "spend": 0.00022500000000000002,
     "total_tokens": 30,

tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py

@@ -3838,7 +3838,7 @@ class TestSpendLogsPayload:
                     "model": "gpt-4o",
                     "user": "",
                     "team_id": "",
-                    "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
+                    "metadata": '{"user_api_key_user_email": null, "agent_name": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
                     "cache_key": "Cache OFF",
                     "spend": 0.00022500000000000002,
                     "total_tokens": 30,

tests/unit/proxy/test_litellm_pre_call_utils.py

@@ -11443,3 +11443,18 @@ def test_caller_tags_reach_litellm_metadata_whether_metadata_is_a_dict_or_a_json
     LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data)
     assert set(data["litellm_metadata"]["tags"]) == {"key-tag", "client-tag"}
     assert data["metadata"] == caller_metadata
+
+
+@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
+def test_agent_name_header_survives_spend_logging_without_an_agent_registration(metadata_field: str) -> None:
+    from litellm.proxy.spend_tracking.spend_tracking_utils import _get_spend_logs_metadata
+
+    data = {metadata_field: {"agent_name": "body-default"}}
+    LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
+        headers={"X-LiteLLM-Agent-Name": " Support agent "},
+        data=data,
+        _metadata_variable_name=metadata_field,
+    )
+    stored = _get_spend_logs_metadata(data[metadata_field])
+    assert stored["agent_name"] == "Support agent"
+    assert data[metadata_field].get("agent_id") is None

tests/unit/proxy/test_tracing_endpoints.py

@@ -7,7 +7,6 @@ import json
 from collections.abc import AsyncGenerator, Mapping
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
-from types import ModuleType
 from typing import Final, Literal, TypedDict
 from unittest.mock import AsyncMock, MagicMock, call
 
@@ -29,14 +28,15 @@ from litellm.proxy._types import LitellmUserRoles, ProxyLifespanState, UserAPIKe
 from litellm.proxy.auth.authorization import OwnedRows, ReadScope
 from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.tracing_runtime import manage_tracing, provide_storage
+from litellm.proxy.tracing_runtime import manage_tracing
+from litellm.tracing import TraceReceiver
 from litellm.tracing.errors import TraceChanged
 from litellm.tracing.generated.models import TraceQueryHelp
 from litellm.tracing.generated.responses import TraceSQLResponse
-from litellm.tracing.generated.types import AllQueryScope, TraceScope
-from litellm.tracing.storage import LensTraceStorage
-from litellm.tracing import TraceReceiver
+from litellm.tracing.generated.types import TraceScope
 from litellm.tracing.remote import RemoteTraceStore
+from litellm.tracing.spend_logs import SpendLogTraceStorage
+from litellm.tracing.storage import LensTraceStorage
 from litellm.tracing.types import TraceAgent, TraceAgentList
 
 SQL_ROWS: Final[tuple[Mapping[str, JsonValue], ...]] = (
@@ -692,6 +692,32 @@ def test_key_without_user_cannot_read_traces(client: TestClient, auth: UserAPIKe
     storage.query_help.assert_not_called()
 
 
+@pytest.mark.parametrize("request_", (("post", "/v1/traces/query"), ("get", "/v1/traces/query/help")))
+def test_sql_routes_need_the_lens_service_when_traces_come_from_spend_logs(
+    client: TestClient, request_: tuple[str, str]
+) -> None:
+    spend_logs: Final = TraceReceiver(storage=SpendLogTraceStorage(lambda: None))
+    client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: spend_logs
+    client.app.dependency_overrides[tracing_endpoints.provide_trace_query_secret] = lambda: "test-secret"
+    method, path = request_
+    response: Final = client.request(method, path, json={"sql": "SELECT 1"} if method == "post" else None)
+    assert response.status_code == 501, response.text
+    assert "Lens service" in response.json()["detail"]
+
+
+@pytest.mark.parametrize(("allowed_routes", "status_code"), ((["llm_api_routes"], 403), (None, 200)))
+def test_spend_log_traces_need_the_same_route_access_as_the_logs_page(
+    client: TestClient, allowed_routes: list[str] | None, status_code: int
+) -> None:
+    spend_logs: Final = TraceReceiver(storage=SpendLogTraceStorage(lambda: None))
+    client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: spend_logs
+    client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="user", token="hashed-key", allowed_routes=allowed_routes
+    )
+    response: Final = client.get("/v1/traces")
+    assert response.status_code == status_code, response.text
+
+
 def test_disabled_receiver_precedes_read_scope_rejection(client: TestClient) -> None:
     client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
         user_role=LitellmUserRoles.INTERNAL_USER
@@ -1009,8 +1035,12 @@ async def test_storage_preserves_page_cursor_and_normalizes_remote_trace_data(
 
     def accept(request: httpx.Request) -> httpx.Response:
         assert json.loads(request.content) == {
-            "operation": "trace", "scope": {**scope, "team_ids": []}, "trace_id": "t1", "trace_ref": "run",
-            "cursor": cursor, "page_size": page_size,
+            "operation": "trace",
+            "scope": {**scope, "team_ids": []},
+            "trace_id": "t1",
+            "trace_ref": "run",
+            "cursor": cursor,
+            "page_size": page_size,
         }
         return httpx.Response(200, json={**TRACE_RESPONSE, "next_cursor": "more"})
 
@@ -1026,7 +1056,9 @@ async def test_storage_validates_the_remote_query_help_value() -> None:
     async with httpx.AsyncClient(
         base_url="http://lens", transport=httpx.MockTransport(lambda request: httpx.Response(200, json=QUERY_HELP))
     ) as client:
-        assert await LensTraceStorage(RemoteTraceStore(client)).query_help({"kind": "all"}, "secret") == TraceQueryHelp.model_validate(QUERY_HELP)
+        assert await LensTraceStorage(RemoteTraceStore(client)).query_help(
+            {"kind": "all"}, "secret"
+        ) == TraceQueryHelp.model_validate(QUERY_HELP)
 
 
 @pytest.mark.parametrize(
@@ -1050,7 +1082,33 @@ async def test_storage_validates_the_remote_query_help_value() -> None:
 )
 async def test_storage_rejects_remote_query_help_that_drifts_from_the_contract(drift: Mapping[str, object]) -> None:
     async with httpx.AsyncClient(
-        base_url="http://lens", transport=httpx.MockTransport(lambda request: httpx.Response(200, json={**QUERY_HELP, **drift}))
+        base_url="http://lens",
+        transport=httpx.MockTransport(lambda request: httpx.Response(200, json={**QUERY_HELP, **drift})),
     ) as client:
         with pytest.raises(RuntimeError, match="invalid response"):
             await LensTraceStorage(RemoteTraceStore(client)).query_help({"kind": "all"}, "secret")
+
+
+@pytest.mark.parametrize(
+    "path", ["/v1/traces", "/v1/traces/agents", "/v1/traces/session", "/v1/traces/session/spans/call"]
+)
+def test_spend_log_traces_require_spend_log_route_access(client: TestClient, path: str) -> None:
+    client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver(
+        storage=SpendLogTraceStorage(lambda: None)
+    )
+    client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=["llm_api_routes"]
+    )
+    response: Final = client.get(path)
+    assert response.status_code == 403
+
+
+def test_spend_log_traces_allow_a_dashboard_session(client: TestClient) -> None:
+    client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver(
+        storage=SpendLogTraceStorage(lambda: None)
+    )
+    client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
+        user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=["/spend/logs/ui", "/v1/traces"]
+    )
+    response: Final = client.get("/v1/traces")
+    assert response.status_code == 200 and response.json()["data"] == []

tests/unit/tracing/test_spend_logs.py

@@ -0,0 +1,413 @@
+import json
+import math
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+import pytest
+
+from litellm.tracing.generated.models import TraceAgentsParams
+from litellm.tracing.generated.types import TraceScope
+from litellm.tracing.spend_logs import SpendLogRow, SpendLogTraceStorage, build_trace
+
+ALL: Final = TraceScope(all_teams=1, user_id="", team_ids=())
+OWNED: Final = TraceScope(all_teams=0, user_id="u1", team_ids=("t1",))
+SDK_UA: Final = "claude-cli/2.1.318 (external, sdk-py, agent-sdk/0.2.178)"
+
+
+def row(
+    request_id: str,
+    start: str,
+    end: str,
+    *,
+    trace_key: str = "s1",
+    tags: Sequence[str] = (),
+    user_agent: str = "",
+    messages: object = (),
+    response: Mapping[str, object] | None = None,
+    status: str = "success",
+    error: str | None = None,
+    spend: float = 0.01,
+    agent_id: str | None = None,
+    request: Mapping[str, object] | None = None,
+) -> SpendLogRow:
+    metadata: Final = {
+        "user_agent": user_agent,
+        **({"error_information": {"error_message": error}} if error else {}),
+    }
+    return SpendLogRow(
+        request_id=request_id,
+        trace_key=trace_key,
+        call_type="acompletion",
+        model="gpt-4o-mini",
+        status=status,
+        spend=spend,
+        prompt_tokens=10,
+        completion_tokens=5,
+        start_time=f"2026-11-06 05:51:{start}",
+        end_time=f"2026-11-06 05:51:{end}",
+        agent_id=agent_id,
+        request_tags=json.dumps(list(tags)),
+        metadata=json.dumps(metadata),
+        messages="{}",
+        proxy_server_request=json.dumps(request or {"messages": messages, "tags": list(tags)}),
+        response=json.dumps(response or {}),
+    )
+
+
+class FakeDatabase:
+    def __init__(self, *responses: Sequence[Mapping[str, object]]) -> None:
+        self.responses: Final = responses
+        self.calls: tuple[tuple[str, tuple[object, ...]], ...] = ()
+
+    async def fetch(self, query: str, *args: object) -> Sequence[Mapping[str, object]]:
+        self.calls = (*self.calls, (query, args))
+        return self.responses[len(self.calls) - 1]
+
+
+def test_tagged_calls_become_an_agent_tree_with_tool_spans() -> None:
+    tool_response: Final = {
+        "choices": [{"message": {"tool_calls": [{"function": {"name": "classify_ticket", "arguments": '{"t":1}'}}]}}]
+    }
+    trace: Final = build_trace(
+        (
+            row(
+                "r1",
+                "00.000",
+                "01.000",
+                tags=("agent:triage",),
+                response=tool_response,
+                messages=[{"role": "user", "content": "refund me"}],
+            ),
+            row("r2", "01.500", "02.000", tags=("agent:classifier", "parent_agent:triage"), spend=0.02),
+            row("r3", "03.000", "04.000", tags=("agent:triage",), status="failure", error="boom"),
+        )
+    )
+
+    assert {(agent["name"], agent["parent_agent"], agent["llm_calls"]) for agent in trace["agents"]} == {
+        ("triage", None, 2),
+        ("classifier", "triage", 1),
+    }
+    spans: Final = {span["span_id"]: span for span in trace["spans"]}
+    assert spans["agent:classifier"]["parent_span_id"] == "agent:triage"
+    assert spans["r2"]["parent_span_id"] == "agent:classifier"
+    assert spans["r1:tool:0"]["parent_span_id"] == "r1"
+    assert spans["r1:tool:0"]["name"] == "classify_ticket"
+    assert math.isclose(spans["r1:tool:0"]["duration_ms"], 2000)
+    assert spans["r3"]["status"] == "error" and spans["r3"]["error"] == "boom"
+    summary: Final = trace["summary"]
+    assert summary["name"] == "triage"
+    assert summary["llm_calls"] == 3 and summary["tool_calls"] == 1 and summary["error_count"] == 1
+    assert summary["status"] == "error"
+    assert math.isclose(summary.get("spend") or 0, 0.04)
+    assert summary["input_tokens"] == 30 and summary["output_tokens"] == 15
+    assert summary["duration_ms"] == 4000
+    assert summary["input_preview"] == "refund me"
+
+
+def test_claude_agent_sdk_subagent_is_inferred_from_its_task_prompt() -> None:
+    task: Final = {
+        "content": [
+            {"type": "tool_use", "name": "Agent", "input": {"subagent_type": "researcher", "prompt": "find MVCC facts"}}
+        ]
+    }
+    reminder: Final = {"type": "text", "text": "<system-reminder>skills</system-reminder>"}
+    trace: Final = build_trace(
+        (
+            row(
+                "r1",
+                "00.000",
+                "01.000",
+                user_agent=SDK_UA,
+                response=task,
+                messages=[{"role": "user", "content": [reminder, {"type": "text", "text": "research MVCC"}]}],
+            ),
+            row(
+                "r2",
+                "01.100",
+                "02.000",
+                user_agent=SDK_UA,
+                messages=[{"role": "user", "content": [reminder, {"type": "text", "text": "find MVCC facts"}]}],
+            ),
+            row("r3", "02.100", "03.000", user_agent=SDK_UA, messages=[{"role": "user", "content": "research MVCC"}]),
+        )
+    )
+
+    assert {(agent["name"], agent["parent_agent"]) for agent in trace["agents"]} == {
+        ("claude-agent-sdk", None),
+        ("researcher", "claude-agent-sdk"),
+    }
+    agents_by_span: Final = {span["span_id"]: span["agent"] for span in trace["spans"]}
+    assert (agents_by_span["r1"], agents_by_span["r2"], agents_by_span["r3"]) == (
+        "claude-agent-sdk",
+        "researcher",
+        "claude-agent-sdk",
+    )
+    assert trace["summary"].get("frameworks") == ("claude-agent-sdk",)
+    assert trace["summary"]["input_preview"] == "research MVCC"
+
+
+def test_claude_agent_sdk_subagent_wins_over_a_tag_inherited_from_its_parent() -> None:
+    task: Final = {
+        "content": [{"type": "tool_use", "name": "Task", "input": {"subagent_type": "researcher", "prompt": "dig"}}]
+    }
+    tags: Final = ("agent:support",)
+    trace: Final = build_trace(
+        (
+            row(
+                "r1",
+                "00.000",
+                "01.000",
+                tags=tags,
+                user_agent=SDK_UA,
+                response=task,
+                messages=[{"role": "user", "content": "help"}],
+            ),
+            row("r2", "01.100", "02.000", tags=tags, user_agent=SDK_UA, messages=[{"role": "user", "content": "dig"}]),
+            row(
+                "r3",
+                "02.100",
+                "03.000",
+                tags=("agent:billing",),
+                user_agent=SDK_UA,
+                messages=[{"role": "user", "content": "dig"}],
+            ),
+        )
+    )
+
+    assert {(agent["name"], agent["parent_agent"]) for agent in trace["agents"]} == {
+        ("support", None),
+        ("researcher", "support"),
+        ("billing", None),
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("scope", "scope_args"),
+    (pytest.param(ALL, (), id="admin"), pytest.param(OWNED, ("u1", ["t1"]), id="owned")),
+)
+async def test_list_traces_pages_by_last_activity_inside_the_reader_scope(
+    scope: TraceScope, scope_args: tuple[object, ...]
+) -> None:
+    keys: Final = (
+        {"trace_key": "s2", "last_activity": "2026-11-06 05:52:00"},
+        {"trace_key": "s1", "last_activity": "2026-11-06 05:51:00"},
+    )
+    db: Final = FakeDatabase(
+        keys, (row("r9", "00.000", "01.000", trace_key="s2"),), keys[1:], (row("r1", "00.000", "01.000"),)
+    )
+    storage: Final = SpendLogTraceStorage(lambda: db)
+
+    first: Final = await storage.list_traces(scope, 0, 99, limit=1)
+    second: Final = await storage.list_traces(scope, 0, 99, cursor=first["next_cursor"], limit=1)
+
+    assert [summary["trace_id"] for summary in first["data"]] == ["s2"]
+    assert first["next_cursor"] is not None
+    assert [summary["trace_id"] for summary in second["data"]] == ["s1"]
+    assert second["next_cursor"] is None
+    assert db.calls[0][1] == (0, 99, *scope_args)
+    assert db.calls[1][1] == (["s2"], *scope_args)
+    assert db.calls[2][1] == (0, 99, *scope_args, "2026-11-06 05:52:00", "s2")
+    assert ("team_id = ANY" in db.calls[0][0]) == (scope["all_teams"] == 0)
+
+
+@pytest.mark.asyncio
+async def test_span_reads_return_the_stored_request_and_error() -> None:
+    rows: Final = (
+        row(
+            "r1",
+            "00.000",
+            "01.000",
+            tags=("agent:triage",),
+            messages=[{"role": "user", "content": "hi"}],
+            response={"choices": [{"message": {"content": "hello"}}]},
+        ),
+        row("r2", "02.000", "03.000", tags=("agent:triage",), status="failure", error="rate limited"),
+    )
+    db: Final = FakeDatabase(rows, rows, rows)
+    storage: Final = SpendLogTraceStorage(lambda: db)
+
+    detail: Final = await storage.get_span("s1", "r1", ALL)
+    error: Final = await storage.get_span_error("s1", "r2", ALL)
+    missing: Final = await storage.get_span_error("s1", "r1", ALL)
+
+    assert detail is not None and detail["output"] == "hello"
+    assert detail["input_ui"] == {
+        "kind": "messages",
+        "messages": ({"role": "user", "content": "hi", "tool_calls": ()},),
+    }
+    assert error is not None and error["message"] == "rate limited" and error["total_chars"] == len("rate limited")
+    assert missing is None
+
+
+@pytest.mark.asyncio
+async def test_agents_list_counts_runs_and_failures_per_agent() -> None:
+    keys: Final = (
+        {"trace_key": "s1", "last_activity": "2026-11-06 05:51:03"},
+        {"trace_key": "s2", "last_activity": "2026-11-06 05:51:01"},
+    )
+    rows: Final = (
+        row("r1", "00.000", "01.000", tags=("agent:triage",)),
+        row("r2", "01.000", "02.000", tags=("agent:writer", "parent_agent:triage"), status="failure", error="x"),
+        row("r3", "00.000", "01.000", trace_key="s2", tags=("agent:triage",)),
+    )
+    db: Final = FakeDatabase(keys, rows)
+    storage: Final = SpendLogTraceStorage(lambda: db)
+
+    agents: Final = await storage.trace_agents(
+        TraceAgentsParams(all_teams=1, user_id="", team_ids=(), start_ms=0, end_ms=99, limit=10)
+    )
+
+    assert {(agent.agent_name, agent.runs, agent.failed_runs) for agent in agents} == {
+        ("triage", 2, 0),
+        ("writer", 1, 1),
+    }
+
+
+@pytest.mark.asyncio
+async def test_no_database_reads_as_no_traces() -> None:
+    page: Final = await SpendLogTraceStorage(lambda: None).list_traces(ALL, 0, 99)
+
+    assert page == {"data": (), "next_cursor": None}
+
+
+@pytest.mark.asyncio
+async def test_trace_pages_include_every_call_and_keep_complete_totals() -> None:
+    rows: Final = tuple(row(f"r{i:04d}", "00.000", "01.000", tags=("agent:triage",)) for i in range(601))
+    storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows))
+    first: Final = await storage.get_trace("s1", ALL, page_size=500)
+    assert first is not None and first["next_cursor"]
+    second: Final = await storage.get_trace("s1", ALL, cursor=first["next_cursor"], page_size=500)
+    assert second is not None and second["next_cursor"] is None
+    assert len(first["spans"]) == 500
+    assert {span["span_id"] for span in (*first["spans"], *second["spans"])} == {
+        "agent:triage",
+        *(item["request_id"] for item in rows),
+    }
+    assert first["summary"]["llm_calls"] == second["summary"]["llm_calls"] == 601
+    assert math.isclose(first["summary"]["spend"] or 0, 6.01)
+
+
+@pytest.mark.asyncio
+async def test_cursor_rejects_a_changed_run_and_invalid_offsets() -> None:
+    from litellm.tracing.errors import TraceChanged
+
+    rows: Final = (row("r1", "00.000", "01.000"), row("r2", "02.000", "03.000"))
+    db: Final = FakeDatabase(rows, (*rows, row("r3", "04.000", "05.000")))
+    storage: Final = SpendLogTraceStorage(lambda: db)
+    first: Final = await storage.get_trace("s1", ALL, page_size=1)
+    assert first is not None and first["next_cursor"]
+    with pytest.raises(TraceChanged):
+        await storage.get_trace("s1", ALL, cursor=first["next_cursor"], page_size=1)
+
+
+@pytest.mark.asyncio
+async def test_oversized_reads_fail_instead_of_returning_incomplete_runs() -> None:
+    from litellm.tracing.spend_logs import TRACE_ROW_LIMIT
+
+    rows: Final = (row("r1", "00.000", "01.000"),) * (TRACE_ROW_LIMIT + 1)
+    storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows))
+    with pytest.raises(OverflowError):
+        await storage.get_trace("s1", ALL)
+
+
+@pytest.mark.parametrize(
+    "body",
+    [
+        {"input": "Find the invoice"},
+        {"input": [{"role": "user", "content": [{"type": "input_text", "text": "Find the invoice"}]}]},
+    ],
+)
+@pytest.mark.asyncio
+async def test_responses_inputs_are_visible_in_the_run_and_step(body: Mapping[str, object]) -> None:
+    item: Final = SpendLogRow(**{**row("r1", "00.000", "01.000"), "proxy_server_request": json.dumps(body)})
+    storage: Final = SpendLogTraceStorage(lambda: FakeDatabase((item,)))
+    assert build_trace((item,))["summary"]["input_preview"] == "Find the invoice"
+    detail: Final = await storage.get_span("s1", "r1", ALL)
+    assert detail is not None
+    assert "Find the invoice" in detail["input"]
+
+
+@pytest.mark.parametrize(
+    ("response", "body"),
+    [
+        (
+            {
+                "choices": [
+                    {
+                        "message": {
+                            "tool_calls": [
+                                {"id": "call-1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+                            ]
+                        }
+                    }
+                ]
+            },
+            {"messages": [{"role": "tool", "tool_call_id": "call-1", "content": "Invoice paid"}]},
+        ),
+        (
+            {"content": [{"type": "tool_use", "id": "call-1", "name": "lookup", "input": {}}]},
+            {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [{"type": "tool_result", "tool_use_id": "call-1", "content": "Invoice paid"}],
+                    }
+                ]
+            },
+        ),
+        (
+            {"output": [{"type": "function_call", "call_id": "call-1", "name": "lookup", "arguments": "{}"}]},
+            {"input": [{"type": "function_call_output", "call_id": "call-1", "output": "Invoice paid"}]},
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_tool_details_use_the_result_recorded_in_the_next_request(
+    response: Mapping[str, object], body: Mapping[str, object]
+) -> None:
+    rows: Final = (
+        row("r1", "00.000", "01.000", response=response),
+        SpendLogRow(**{**row("r2", "02.000", "03.000"), "proxy_server_request": json.dumps(body)}),
+    )
+    storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows))
+    detail: Final = await storage.get_span("s1", "r1:tool:0", ALL)
+    assert detail is not None and detail["output"] == "Invoice paid"
+
+
+def test_registered_agent_names_are_readable_and_request_tags_override_them() -> None:
+    item: Final = SpendLogRow(**{**row("r1", "00.000", "01.000", agent_id="agent-uuid"), "agent_name": "Support"})
+    assert build_trace((item,))["summary"]["name"] == "Support"
+    tagged: Final = SpendLogRow(**{**item, "request_tags": '["agent:Research"]'})
+    assert build_trace((tagged,))["summary"]["name"] == "Research"
+
+
+def test_offset_timestamps_preserve_the_actual_instant() -> None:
+    item: Final = SpendLogRow(
+        **{
+            **row("r1", "00.000", "01.000"),
+            "start_time": "2026-11-06T13:00:00+08:00",
+            "end_time": "2026-11-06T13:00:01+08:00",
+        }
+    )
+    assert build_trace((item,))["summary"]["start_time"] == "2026-11-06T05:00:00Z"
+
+
+def test_request_agent_name_groups_calls_without_tags_or_registered_identity() -> None:
+    named: Final = tuple(
+        SpendLogRow(**{**row(f"r{i}", "00.000", "01.000"), "metadata": '{"agent_name": "Support agent"}'})
+        for i in range(2)
+    )
+    trace: Final = build_trace(named)
+    assert trace["summary"]["name"] == "Support agent"
+    assert trace["summary"]["agent_names"] == ("Support agent",)
+    assert trace["summary"]["llm_calls"] == 2
+    assert {span["agent"] for span in trace["spans"]} == {"Support agent"}
+
+
+def test_responses_api_input_is_read_from_the_stored_request() -> None:
+    trace: Final = build_trace(
+        (row("r1", "00.000", "01.000", tags=("agent:triage",), request={"model": "gpt-5", "input": "refund me"}),)
+    )
+
+    assert trace["summary"]["input_preview"] == "refund me"

ui/litellm-dashboard/package-lock.json

@@ -14,7 +14,7 @@
         "@headlessui/tailwindcss": "0.2.2",
         "@heroicons/react": "1.0.6",
         "@hookform/resolvers": "5.4.0",
-        "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz",
+        "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz",
         "@shadcn/react": "0.3.1",
         "@tanstack/react-pacer": "0.22.1",
         "@tanstack/react-query": "5.100.7",
@@ -2096,9 +2096,9 @@
       }
     },
     "node_modules/@litellm/lens-ui": {
-      "version": "0.1.0-dev.0",
-      "resolved": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz",
-      "integrity": "sha512-YG8ydmSqOzCumVWSPg6G39yzLGTqfTEy1rqIC4FpysCRW+3KtYnEdpj/SsO9xzgb9tYJIdZhYBHs/027Lh0qvQ==",
+      "version": "0.1.0-dev.1",
+      "resolved": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz",
+      "integrity": "sha512-Wzy+7aS+nHphlcnkd5vRTxvpQy4rJqWBQvh11nDQc/8TyW0qTpacQYHONw7gLt0VNL+7rFHeo8a7vTpgXBkKAw==",
       "license": "MIT",
       "dependencies": {
         "@base-ui/react": "1.6.0",

ui/litellm-dashboard/package.json

@@ -31,7 +31,7 @@
     "@headlessui/tailwindcss": "0.2.2",
     "@heroicons/react": "1.0.6",
     "@hookform/resolvers": "5.4.0",
-    "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz",
+    "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz",
     "@shadcn/react": "0.3.1",
     "@tanstack/react-pacer": "0.22.1",
     "@tanstack/react-query": "5.100.7",

ui/litellm-dashboard/src/components/lens/EmbeddedLens.integration.test.tsx

@@ -25,41 +25,22 @@ afterEach(() => {
   vi.unstubAllGlobals();
 });
 
-it("renders the published Lens settings and uses the gateway session, prefix, and public contract", async () => {
-  const user = userEvent.setup();
+it("opens spend logs from a legacy settings link using the gateway session and URL prefix", async () => {
   network.mockImplementation(async (input, init) => {
     const request = input instanceof Request ? input : new Request(String(input), init);
-    expect(request.url).toMatch(/^https:\/\/gateway\.test\/prefix\/lens/);
+    expect(request.url).toMatch(/^https:\/\/gateway\.test\/prefix\/v1\/traces/);
     expect(request.headers.get("X-Gateway-Key")).toBe("Bearer gateway-session");
-    expect(request.headers.get("X-Lens-Contract")).toBe("1");
     const path = new URL(request.url).pathname;
-    if (path === "/prefix/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: true });
-    if (path === "/prefix/lens/signals") return Response.json({ model: "", threshold: 0.5, signals: [] });
-    if (path === "/prefix/lens/model_group/info") return Response.json({ data: [] });
-    if (path === "/prefix/lens/service")
-      return Response.json({ configured: false, connected: false, status: { storage_ready: false } });
+    if (path === "/prefix/v1/traces") return Response.json({ data: [], next_cursor: null });
+    if (path === "/prefix/v1/traces/agents") return Response.json({ agents: [] });
     throw new Error(`Unexpected Lens request ${path}`);
   });
   renderWithProviders(<EmbeddedLens accessToken="gateway-session" userRole="Admin" readOnly={false} />, {
     searchParams: "?tab=settings",
   });
-  expect(await screen.findByRole("heading", { name: "Add an analysis provider" })).toBeVisible();
-  expect(screen.getByRole("link", { name: "Configure analysis models" })).toHaveAttribute(
-    "href",
-    "https://github.com/BerriAI/lens/blob/main/docs/analysis.md",
-  );
-  expect(screen.getByText("Tracing enabled")).toBeVisible();
-  const analysis = within(screen.getByRole("region", { name: "Analysis", exact: true }));
-  const copyPrompt = analysis.getByRole("button", { name: "Set it up for me" });
-  await user.click(copyPrompt);
-  const prompt = await navigator.clipboard.readText();
-  expect(prompt).toContain("I opened Lens from an existing LiteLLM admin dashboard.");
-  expect(prompt).toContain('LiteLLM API base: "https://gateway.test/prefix"');
-  expect(prompt).toContain("Configure Lens investigations");
-  expect(prompt).not.toContain("standalone Lens app");
-  expect(prompt).not.toContain("gateway-session");
-  expect(copyPrompt).toHaveTextContent("Prompt copied");
-  expect(analysis.getByRole("button", { name: "Check configuration" })).toBeVisible();
+  expect(await screen.findByRole("heading", { name: "Your agent runs will appear here" })).toBeVisible();
+  expect(screen.queryByRole("heading", { name: "Add an analysis provider" })).not.toBeInTheDocument();
+  expect(screen.getByRole("tab", { name: "Traces", selected: true })).toBeVisible();
   expect(network).toHaveBeenCalled();
 });
 
@@ -79,46 +60,20 @@ it("opens the packaged demo trace and preserves its shareable selection in the g
   expect(network).not.toHaveBeenCalled();
 });
 
-it("keeps manual installation on the unconfigured gateway landing and copies its observed setup context", async () => {
-  const user = userEvent.setup();
+it("opens spend-log traces without asking to install a Lens service", async () => {
   network.mockImplementation(async (input, init) => {
     const request = input instanceof Request ? input : new Request(String(input), init);
     const path = new URL(request.url).pathname;
-    if (path === "/prefix/lens/service") {
-      const connection = { url: "", configured: false, connected: false, status: { storage_ready: false } };
-      return Response.json(connection);
-    }
-    if (path === "/prefix/lens")
-      return Response.json(
-        { detail: "Configure LITELLM_LENS_URL and LENS_GATEWAY_SECRET for the Lens service" },
-        { status: 503 },
-      );
-    if (path === "/prefix/v1/traces" || path === "/prefix/v1/traces/agents")
-      return Response.json(
-        { detail: "Agent tracing is not enabled. Configure the Lens service and LITELLM_LENS_URL." },
-        { status: 501 },
-      );
+    if (path === "/prefix/lens/service")
+      return Response.json({ configured: false, connected: false, status: { storage_ready: false } });
+    if (path === "/prefix/v1/traces") return Response.json({ data: [], next_cursor: null });
+    if (path === "/prefix/v1/traces/agents") return Response.json({ agents: [] });
     throw new Error(`Unexpected Lens request ${path}`);
   });
   renderWithProviders(<EmbeddedLens accessToken="gateway-session" userRole="Admin" readOnly={false} />);
-
-  const setup = within(await screen.findByRole("region", { name: "Get Lens running" }));
-  expect(setup.getByRole("button", { name: /Install Lens/ })).toHaveAttribute("aria-expanded", "true");
-  expect(await setup.findByRole("link", { name: "Helm setup" })).toHaveAttribute(
-    "href",
-    "https://docs.litellm.ai/docs/proxy/lens/deployment/kubernetes#existing-deployment",
-  );
-  expect(setup.getByRole("link", { name: "Docker setup" })).toHaveAttribute(
-    "href",
-    "https://docs.litellm.ai/docs/proxy/lens/deployment/docker-compose",
-  );
-  expect(setup.getByRole("button", { name: "Check setup" })).toBeEnabled();
-
-  await user.click(setup.getByRole("button", { name: "Set it up for me" }));
-  const prompt = await navigator.clipboard.readText();
-  expect(prompt).toContain("existing LiteLLM admin dashboard");
-  expect(prompt).toContain('LiteLLM API base: "https://gateway.test/prefix"');
-  expect(prompt).toContain("Lens connection is not configured");
-  expect(prompt).toContain("Check for an existing Lens service before installing another");
-  expect(prompt).not.toContain("gateway-session");
+  expect(await screen.findByRole("heading", { name: "Your agent runs will appear here" })).toBeVisible();
+  expect(screen.getByText(/x-litellm-agent-name/)).toBeVisible();
+  expect(screen.queryByRole("region", { name: "Get Lens running" })).not.toBeInTheDocument();
+  const tabs = within(screen.getByRole("tablist", { name: "Lens" }));
+  expect(tabs.getAllByRole("tab").map((tab) => tab.textContent)).toEqual(["Agents", "Traces"]);
 });

ui/litellm-dashboard/src/components/lens/EmbeddedLens.tsx

@@ -10,6 +10,7 @@ import { serverRootPath } from "@/lib/serverRootPath";
 const host: LensHost = {
   surface: "embedded",
   analysis: "deployment",
+  traces: "spend-logs",
   spendLogs: { lookup: uiSpendLogsCall, Drawer: LogDetailsDrawer },
 };