diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e53b5d4ad52..9cc0287acfe 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3785,6 +3785,7 @@ class SpendLogsRouterMetadata(TypedDict): class SpendLogsMetadata(TypedDict): + agent_name: ReadOnly[NotRequired[str | None]] actor_agent_id: ReadOnly[NotRequired[str | None]] target_agent_id: ReadOnly[NotRequired[str | None]] billing_agent_id: ReadOnly[NotRequired[str | None]] @@ -4583,6 +4584,7 @@ class LitellmMetadataFromRequestHeaders(TypedDict, total=False): """ spend_logs_metadata: dict | None + agent_name: ReadOnly[str] agent_id: str | None trace_id: str | None session_id: str | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4444d6d25f9..7a5e28d3196 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1938,7 +1938,15 @@ class LiteLLMProxyRequestSetup: """ from litellm.proxy._types import LitellmMetadataFromRequestHeaders - metadata_from_headers: Final = LitellmMetadataFromRequestHeaders() + normalized_headers: Final = MappingProxyType( + {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)} + ) + agent_name_from_header: Final = normalized_headers.get("x-litellm-agent-name", "").strip() + metadata_from_headers: Final = ( + LitellmMetadataFromRequestHeaders(agent_name=agent_name_from_header) + if agent_name_from_header + else LitellmMetadataFromRequestHeaders() + ) request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object _drop_client_session_metadata(request_body) spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) @@ -1949,9 +1957,6 @@ class LiteLLMProxyRequestSetup: # Finally update the requests metadata with the `metadata_from_headers` ######################################################################################### - normalized_headers: Final = MappingProxyType( - {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)} - ) agent_id_from_header: Final = headers.get("x-litellm-agent-id") # Explicit litellm headers take precedence; fall back to any x-*-session-id header. chain_id: Final = get_chain_id_from_headers(dict(headers)) diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py index 94ae1703a05..e8886a9bff1 100644 --- a/litellm/proxy/tracing_endpoints.py +++ b/litellm/proxy/tracing_endpoints.py @@ -27,8 +27,9 @@ from litellm.constants import ( TRACE_READ_RETRY_AFTER_SECONDS, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_trace_read_scope +from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_spend_log_read_scope, resolve_trace_read_scope from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.http_parsing_utils import is_otlp_trace_request from litellm.proxy.tracing_runtime import provide_receiver, require_receiver @@ -86,6 +87,10 @@ async def provide_trace_access( tracing: Annotated[TraceReceiver | None, Depends(provide_receiver)], log_team_lookup: LogTeamLookupDependency, ) -> TraceAccessContext: + if tracing is not None and isinstance(tracing.storage, SpendLogTraceStorage): + RouteChecks.should_call_route(route="/spend/logs/ui", valid_token=auth) + spend_scope: Final = await resolve_spend_log_read_scope(auth, log_team_lookup) + return TraceAccessContext(tracing, spend_scope if auth.user_id or isinstance(spend_scope, AllRows) else None) read_scope: Final = await resolve_trace_read_scope(auth, partial(log_team_lookup, auth)) return TraceAccessContext(tracing, read_scope) diff --git a/litellm/tracing/AGENTS.md b/litellm/tracing/AGENTS.md index 397252be7df..788562660bc 100644 --- a/litellm/tracing/AGENTS.md +++ b/litellm/tracing/AGENTS.md @@ -1,7 +1,7 @@ # Lens gateway boundary -- Lens owns trace normalization, schemas, storage, graph assembly and querying in `BerriAI/lens` -- This package owns the HTTP client, gateway response validation and asynchronous gateway spend export. Keep the trace reader remote-only +- Lens owns the canonical trace schemas and its service's normalization, storage, graph assembly and querying in `BerriAI/lens` +- This package owns the HTTP client, gateway response validation and asynchronous gateway spend export. When no Lens service is configured, its read-only Postgres spend-log adapter provides embedded agent traces using existing spend-log permissions. It does not ingest OTLP or implement Lens analysis - Gateway endpoints preserve authenticated user/team scope, trace references, pagination and public error categories - `generated/` comes from `scripts/generate_trace_types.py` using the Lens schemas pinned in `scripts/lens_assets/source.json`. Update the canonical Lens Rust contracts and import their schema outputs before regenerating. Never edit generated Python - Keep the pinned schema and fixture checksums current. CI verifies assets and regenerated Python without a Rust trace implementation in this repository diff --git a/litellm/tracing/spend_logs.py b/litellm/tracing/spend_logs.py index ef88892ca0e..a13559d8a58 100644 --- a/litellm/tracing/spend_logs.py +++ b/litellm/tracing/spend_logs.py @@ -4,19 +4,25 @@ A trace is every spend log row sharing a session id. A row's agent comes from an `parent_agent:` tag), the row's agent_id, a Claude Agent SDK subagent whose first prompt matches a parent's Task/Agent tool call, or the calling framework read from the user agent.""" +import asyncio import base64 +import hashlib import json -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone -from functools import reduce -from itertools import chain -from typing import Final, Protocol +from itertools import chain, groupby, islice +from typing import Final, Protocol, TypeAlias from pydantic import TypeAdapter -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE +from litellm.litellm_core_utils.prompt_templates.factory import ( + get_tool_calls_from_response, + resolve_structured_messages, +) +from litellm.tracing.errors import TraceChanged from litellm.tracing.generated.models import TraceAgentRow, TraceAgentsParams from litellm.tracing.generated.types import ( AgentNode, @@ -39,8 +45,9 @@ from litellm.tracing.generated.types import ( AGENT_TAG_PREFIX: Final = "agent:" PARENT_AGENT_TAG_PREFIX: Final = "parent_agent:" SPEND_LOGS_SERVICE: Final = "litellm" -TRACE_ROW_LIMIT: Final = 500 -AGENT_LIST_TRACE_LIMIT: Final = 200 +TRACE_ROW_LIMIT: Final = 10_000 +TRACE_SPAN_PAGE_SIZE: Final = 500 +AGENT_LIST_TRACE_PAGE_SIZE: Final = 200 PREVIEW_CHARS: Final = 300 ERROR_PREVIEW_CHARS: Final = 500 TRACE_KEY_SQL: Final = "COALESCE(NULLIF(session_id, ''), request_id)" @@ -80,6 +87,7 @@ class SpendLogRow(TypedDict): start_time: ReadOnly[str] end_time: ReadOnly[str] agent_id: ReadOnly[str | None] + agent_name: ReadOnly[NotRequired[str | None]] request_tags: ReadOnly[str | None] metadata: ReadOnly[str | None] messages: ReadOnly[str | None] @@ -96,6 +104,7 @@ _OBJECT_SEQUENCE: Final = TypeAdapter(tuple[object, ...]) @dataclass(frozen=True, slots=True) class _ToolCall: + id: str | None name: str arguments: str subagent: str | None @@ -150,7 +159,8 @@ def _text(value: object) -> str: def _parse_ms(timestamp: str) -> float: - return datetime.fromisoformat(timestamp).replace(tzinfo=timezone.utc).timestamp() * 1000 + parsed: Final = datetime.fromisoformat(timestamp) + return (parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc)).timestamp() * 1000 def _iso(ms: float) -> str: @@ -208,12 +218,9 @@ def _role(value: object) -> ChatRole: return "user" -def _block_tool_calls(content: object) -> tuple[UIToolCall, ...]: - return tuple( - UIToolCall(name=_text(_mapping(block).get("name")), arguments=json.dumps(_mapping(block).get("input"))) - for block in _sequence(content) - if _mapping(block).get("type") == "tool_use" - ) +def _message_tool_calls(message: Mapping[str, object]) -> tuple[UIToolCall, ...]: + response: Final = {"content": message.get("content"), "choices": [{"message": message}]} + return tuple(UIToolCall(name=call.name, arguments=call.arguments) for call in _response_tool_calls(response)) def _ui_messages(messages: object) -> tuple[UIMessage, ...]: @@ -221,7 +228,7 @@ def _ui_messages(messages: object) -> tuple[UIMessage, ...]: UIMessage( role=_role(_mapping(message).get("role")), content=_visible_text(_mapping(message).get("content")), - tool_calls=_block_tool_calls(_mapping(message).get("content")), + tool_calls=_message_tool_calls(_mapping(message)), ) for message in _sequence(messages) ) @@ -237,38 +244,20 @@ def _first_user_prompts(messages: object) -> frozenset[str]: def _response_tool_calls(response: Mapping[str, object]) -> tuple[_ToolCall, ...]: - anthropic: Final = tuple( - (_text(block.get("name")), _mapping(block.get("input"))) - for block in map(_mapping, _sequence(response.get("content"))) - if block.get("type") == "tool_use" - ) - openai: Final = tuple( - ( - _text(_mapping(call.get("function")).get("name")), - _mapping(_loads(_text(_mapping(call.get("function")).get("arguments")))), - ) - for call in map( - _mapping, - chain.from_iterable( - _sequence(_mapping(choice.get("message")).get("tool_calls")) - for choice in map(_mapping, _sequence(response.get("choices"))) - ), - ) - ) - responses_api: Final = tuple( - (_text(item.get("name")), _mapping(_loads(_text(item.get("arguments"))))) - for item in map(_mapping, _sequence(response.get("output"))) - if item.get("type") == "function_call" - ) return tuple( _ToolCall( - name=name, - arguments=json.dumps(arguments), - subagent=_text(arguments.get("subagent_type")) or None if name in SUBAGENT_TOOL_NAMES else None, - subagent_prompt=_text(arguments.get("prompt")).strip() or None if name in SUBAGENT_TOOL_NAMES else None, + id=call["id"], + name=call["name"], + arguments=json.dumps(call["arguments"]), + subagent=_text(call["arguments"].get("subagent_type")) or None + if call["name"] in SUBAGENT_TOOL_NAMES + else None, + subagent_prompt=_text(call["arguments"].get("prompt")).strip() or None + if call["name"] in SUBAGENT_TOOL_NAMES + else None, ) - for name, arguments in (*anthropic, *openai, *responses_api) - if name + for call in get_tool_calls_from_response(response) + if call["name"] ) @@ -288,7 +277,12 @@ def _request_messages(row: SpendLogRow) -> object: stored: Final = _loads(row["messages"]) if _sequence(stored): return stored - return _mapping(_loads(row["proxy_server_request"])).get("messages") + request: Final = _mapping(_loads(row["proxy_server_request"])) + body: Final = _mapping(request.get("body")) if "body" in request else request + messages: Final = _sequence(body.get("messages")) + if messages: + return messages + return resolve_structured_messages(None, dict(body)) if body.get("input") else messages def _call(row: SpendLogRow) -> _Call: @@ -302,7 +296,11 @@ def _call(row: SpendLogRow) -> _Call: start_ms=_parse_ms(row["start_time"]), end_ms=_parse_ms(row["end_time"]), framework=_framework(_text(metadata.get("user_agent"))), - explicit_agent=_tag_value(tags, AGENT_TAG_PREFIX) or row["agent_id"] or None, + explicit_agent=_text(metadata.get("agent_name")).strip() + or _tag_value(tags, AGENT_TAG_PREFIX) + or row.get("agent_name") + or row["agent_id"] + or None, explicit_parent=_tag_value(tags, PARENT_AGENT_TAG_PREFIX), prompts=_first_user_prompts(messages), input_messages=_ui_messages(messages), @@ -316,33 +314,33 @@ def _root_agent(calls: Sequence[_Call]) -> str: return next((call.framework for call in calls if call.framework != DEFAULT_FRAMEWORK), "agent") -def _placed(call: _Call, earlier: tuple[_Placed, ...], root: str) -> _Placed: - spawner: Final = next( - ( - (tool.subagent, placed) - for placed, tool in chain.from_iterable( - ((placed, tool) for tool in placed.call.tool_calls) for placed in reversed(earlier) - ) - if tool.subagent and tool.subagent_prompt in call.prompts - ), - None, +_Spawner: TypeAlias = tuple[int, _ToolCall, _Placed] + + +def _place_call(call: _Call, spawners: Mapping[str, _Spawner], root: str) -> _Placed: + matches: Final = tuple(spawners[prompt] for prompt in call.prompts if prompt in spawners) + spawner: Final = max(matches, key=lambda match: match[0]) if matches else None + parent: Final = call.explicit_parent if call.explicit_parent != call.explicit_agent else None + inferred: Final = spawner is not None and call.explicit_agent in (None, spawner[2].call.explicit_agent) + return _Placed( + call=call, + agent=(spawner[1].subagent or root) if inferred and spawner else (call.explicit_agent or root), + parent_agent=spawner[2].agent if inferred and spawner else parent, ) - if spawner is not None and call.explicit_agent in (None, spawner[1].call.explicit_agent): - return _Placed(call=call, agent=spawner[0], parent_agent=spawner[1].agent) - if call.explicit_agent is not None: - parent: Final = call.explicit_parent if call.explicit_parent != call.explicit_agent else None - return _Placed(call=call, agent=call.explicit_agent, parent_agent=parent) - return _Placed(call=call, agent=root, parent_agent=None) + + +def _placements(calls: Sequence[_Call]) -> Iterator[_Placed]: + root: Final = _root_agent(calls) + spawners: Final[dict[str, _Spawner]] = {} # mutable-ok: rolling index avoids quadratic scans + for index, placed in ((index, _place_call(call, spawners, root)) for index, call in enumerate(calls)): + yield placed + for tool in reversed(placed.call.tool_calls): + if tool.subagent and tool.subagent_prompt: + spawners[tool.subagent_prompt] = (index, tool, placed) def _place(calls: Sequence[_Call]) -> tuple[_Placed, ...]: - root: Final = _root_agent(calls) - - def step(earlier: tuple[_Placed, ...], call: _Call) -> tuple[_Placed, ...]: - return (*earlier, _placed(call, earlier, root)) - - initial: Final[tuple[_Placed, ...]] = () - return reduce(step, calls, initial) + return tuple(_placements(calls)) def _agent_span_id(agent: str) -> str: @@ -379,14 +377,18 @@ def _agent_run(name: str, own: tuple[_Placed, ...]) -> _AgentRun: def _agent_runs(placed: Sequence[_Placed]) -> tuple[_AgentRun, ...]: + groups: Final = { + name: tuple(items) + for name, items in groupby(sorted(placed, key=lambda item: item.agent), key=lambda item: item.agent) + } names: Final = tuple(dict.fromkeys(item.agent for item in placed)) - return tuple(_agent_run(name, tuple(item for item in placed if item.agent == name)) for name in names) + return tuple(_agent_run(name, groups[name]) for name in names) def _next_start(placed: Sequence[_Placed], index: int) -> float | None: current: Final = placed[index] return next( - (item.call.start_ms for item in placed[index + 1 :] if item.agent == current.agent), + (item.call.start_ms for item in islice(placed, index + 1, None) if item.agent == current.agent), None, ) @@ -424,6 +426,8 @@ def _llm_span(item: _Placed, trace_start: float) -> Span: def _tool_spans(placed: Sequence[_Placed], index: int, trace_start: float) -> tuple[Span, ...]: item: Final = placed[index] call: Final = item.call + if not call.tool_calls: + return () resumed_at: Final = _next_start(placed, index) duration: Final = max((resumed_at if resumed_at is not None else call.end_ms) - call.end_ms, 0.0) return tuple( @@ -535,6 +539,25 @@ def build_trace(rows: Sequence[SpendLogRow]) -> Trace: ) +def _tool_result(messages: object, tool_id: str) -> str | None: + for message in map(_mapping, _sequence(messages)): + if message.get("role") == "tool" and message.get("tool_call_id") == tool_id: + return _visible_text(message.get("content")) + for block in map(_mapping, _sequence(message.get("content"))): + if block.get("type") == "tool_result" and block.get("tool_use_id") == tool_id: + return _visible_text(block.get("content")) + return None + + +def _tool_output(calls: Sequence[_Call], tool: _ToolCall) -> str: + if tool.id is None: + return "Tool result was not recorded in the spend logs." + return next( + (result for call in calls if (result := _tool_result(_request_messages(call.row), tool.id)) is not None), + "Tool result was not recorded in the spend logs.", + ) + + def _span_detail(trace_rows: Sequence[SpendLogRow], span_id: str) -> SpanDetail | None: calls: Final = tuple(sorted(map(_call, trace_rows), key=lambda call: (call.start_ms, call.row["request_id"]))) llm: Final = next((call for call in calls if call.row["request_id"] == span_id), None) @@ -569,8 +592,8 @@ def _span_detail(trace_rows: Sequence[SpendLogRow], span_id: str) -> SpanDetail attributes={"tool.name": tool.name}, input=tool.arguments, input_ui=UIText(kind="text", text=tool.arguments), - output="", - output_ui=UIText(kind="text", text=""), + output=_tool_output(calls, tool), + output_ui=UIText(kind="text", text=_tool_output(calls, tool)), ) trace: Final = build_trace(trace_rows) agent: Final = next((span for span in trace["spans"] if span["span_id"] == span_id), None) @@ -596,6 +619,26 @@ def _span_detail(trace_rows: Sequence[SpendLogRow], span_id: str) -> SpanDetail ) +def _trace_page(trace: Trace, cursor: str | None, page_size: int | None) -> Trace: + snapshot: Final = hashlib.sha256(json.dumps(trace, sort_keys=True).encode()).hexdigest() + decoded: Final = TypeAdapter(tuple[str, int]).validate_json(base64.urlsafe_b64decode(cursor)) if cursor else None + if decoded is not None and decoded[0] != snapshot: + raise TraceChanged("The trace changed. Reload it to continue.") + offset: Final = decoded[1] if decoded else 0 + if offset < 0 or offset > len(trace["spans"]): + raise ValueError("Invalid span cursor") + size: Final = page_size if page_size is not None else TRACE_SPAN_PAGE_SIZE + if size < 1 or size > TRACE_SPAN_PAGE_SIZE: + raise ValueError("Invalid span page size") + end: Final = offset + size + next_cursor: Final = ( + base64.urlsafe_b64encode(json.dumps((snapshot, end)).encode()).decode() if end < len(trace["spans"]) else None + ) + return Trace( + summary=trace["summary"], agents=trace["agents"], spans=trace["spans"][offset:end], next_cursor=next_cursor + ) + + def _encode_cursor(row: _TraceKeyRow) -> str: return base64.urlsafe_b64encode(json.dumps((row["last_activity"], row["trace_key"])).encode()).decode() @@ -644,6 +687,7 @@ class SpendLogTraceStorage: {scope_sql} GROUP BY 1 HAVING COUNT(*) > 1 + OR bool_or(COALESCE(metadata->>'agent_name', '') <> '') OR bool_or(COALESCE(agent_id, '') <> '') OR bool_or(COALESCE(request_tags::text, '') LIKE '%"{AGENT_TAG_PREFIX}%') OR bool_or(COALESCE(metadata->>'user_agent', '') ~* '{AGENT_USER_AGENT_SQL}') @@ -667,6 +711,8 @@ class SpendLogTraceStorage: f""" SELECT request_id, {TRACE_KEY_SQL} AS trace_key, call_type, model, status, spend, prompt_tokens, completion_tokens, "startTime"::text AS start_time, "endTime"::text AS end_time, agent_id, + (SELECT agent_name FROM "LiteLLM_AgentsTable" a + WHERE a.agent_id = "LiteLLM_SpendLogs".agent_id) AS agent_name, request_tags::text AS request_tags, metadata::text AS metadata, messages::text AS messages, proxy_server_request::text AS proxy_server_request, response::text AS response FROM "LiteLLM_SpendLogs" @@ -674,17 +720,18 @@ class SpendLogTraceStorage: AND {TRACE_KEY_SQL} = ANY($1::text[]) {scope_sql} ORDER BY "startTime", request_id - LIMIT {TRACE_ROW_LIMIT * len(trace_keys)} + LIMIT {TRACE_ROW_LIMIT + 1} """, list(trace_keys), *scope_args, ) + if len(rows) > TRACE_ROW_LIMIT: + raise OverflowError("Spend-log trace read exceeds its row budget") return _SPEND_LOG_ROWS.validate_python(rows) async def _traces(self, keys: Sequence[str], scope: TraceScope) -> tuple[Trace, ...]: rows: Final = await self._rows(keys, scope) - groups: Final = tuple(tuple(row for row in rows if row["trace_key"] == key) for key in keys) - return tuple(build_trace(own) for own in groups if own) + return await asyncio.to_thread(_build_traces, keys, rows) async def list_traces( self, @@ -711,11 +758,13 @@ class SpendLogTraceStorage: page_size: int | None = None, ) -> Trace | None: traces: Final = await self._traces((trace_id,), scope) - return traces[0] if traces else None + if not traces: + return None + return await asyncio.to_thread(_trace_page, traces[0], cursor, page_size) async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: rows: Final = await self._rows((trace_id,), scope) - return _span_detail(rows, span_id) if rows else None + return await asyncio.to_thread(_span_detail, rows, span_id) if rows else None async def get_span_error( self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "", cursor: str | None = None @@ -730,17 +779,49 @@ class SpendLogTraceStorage: scope: Final = TraceScope( all_teams=parameters.all_teams, user_id=parameters.user_id, team_ids=parameters.team_ids ) - keys: Final = await self._trace_keys( - scope, parameters.start_ms, parameters.end_ms, None, AGENT_LIST_TRACE_LIMIT - ) - traces: Final = await self._traces(tuple(row["trace_key"] for row in keys), scope) - names: Final = tuple( - dict.fromkeys(chain.from_iterable(trace["summary"].get("agent_names", ()) for trace in traces)) - ) - return tuple( - _agent_row(name, tuple(trace for trace in traces if name in trace["summary"].get("agent_names", ()))) - for name in names[: parameters.limit] - ) + agents: Final[dict[str, TraceAgentRow]] = {} # mutable-ok: aggregate pages without retaining graphs + cursor: tuple[str, str] | None = None # rebind-ok: advance through database pages + while True: + keys = await self._trace_keys( + scope, parameters.start_ms, parameters.end_ms, cursor, AGENT_LIST_TRACE_PAGE_SIZE + ) + traces = await self._traces(tuple(row["trace_key"] for row in keys), scope) + page_agents = await asyncio.to_thread(_agent_rows, traces) + for agent in page_agents: + previous = agents.get(agent.agent_name) + agents[agent.agent_name] = TraceAgentRow( + agent_name=agent.agent_name, + runs=agent.runs + (previous.runs if previous else 0), + failed_runs=agent.failed_runs + (previous.failed_runs if previous else 0), + last_seen_ms=max(agent.last_seen_ms, previous.last_seen_ms if previous else 0), + frameworks=tuple(dict.fromkeys((*agent.frameworks, *(previous.frameworks if previous else ())))), + ) + if len(keys) < AGENT_LIST_TRACE_PAGE_SIZE: + return tuple(sorted(agents.values(), key=lambda agent: (-agent.last_seen_ms, agent.agent_name)))[ + : parameters.limit + ] + cursor = ( + keys[-1]["last_activity"], + keys[-1]["trace_key"], + ) + + +def _build_traces(keys: Sequence[str], rows: Sequence[SpendLogRow]) -> tuple[Trace, ...]: + groups: Final = { + key: tuple(own) + for key, own in groupby(sorted(rows, key=lambda row: row["trace_key"]), key=lambda row: row["trace_key"]) + } + return tuple(build_trace(groups[key]) for key in keys if key in groups) + + +def _agent_rows(traces: tuple[Trace, ...]) -> tuple[TraceAgentRow, ...]: + names: Final = tuple( + dict.fromkeys(chain.from_iterable(trace["summary"].get("agent_names", ()) for trace in traces)) + ) + return tuple( + _agent_row(name, tuple(trace for trace in traces if name in trace["summary"].get("agent_names", ()))) + for name in names + ) def _agent_row(name: str, own: tuple[Trace, ...]) -> TraceAgentRow: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a14dc95e7c..cef06e7ae13 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3499,6 +3499,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): Specific metadata k,v pairs logged to integration for easier cost tracking and prompt management """ + agent_name: ReadOnly[NotRequired[str | None]] spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call requester_ip_address: str | None user_agent: str | None diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 97c21fcc777..42194592e42 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -11443,3 +11443,18 @@ def test_caller_tags_reach_litellm_metadata_whether_metadata_is_a_dict_or_a_json LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data) assert set(data["litellm_metadata"]["tags"]) == {"key-tag", "client-tag"} assert data["metadata"] == caller_metadata + + +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +def test_agent_name_header_survives_spend_logging_without_an_agent_registration(metadata_field: str) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_spend_logs_metadata + + data = {metadata_field: {"agent_name": "body-default"}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers={"X-LiteLLM-Agent-Name": " Support agent "}, + data=data, + _metadata_variable_name=metadata_field, + ) + stored = _get_spend_logs_metadata(data[metadata_field]) + assert stored["agent_name"] == "Support agent" + assert data[metadata_field].get("agent_id") is None diff --git a/tests/unit/proxy/test_tracing_endpoints.py b/tests/unit/proxy/test_tracing_endpoints.py index 8ff0b0cbf20..24f67f77fde 100644 --- a/tests/unit/proxy/test_tracing_endpoints.py +++ b/tests/unit/proxy/test_tracing_endpoints.py @@ -1074,3 +1074,28 @@ async def test_storage_rejects_remote_query_help_that_drifts_from_the_contract(d ) as client: with pytest.raises(RuntimeError, match="invalid response"): await LensTraceStorage(RemoteTraceStore(client)).query_help({"kind": "all"}, "secret") + + +@pytest.mark.parametrize( + "path", ["/v1/traces", "/v1/traces/agents", "/v1/traces/session", "/v1/traces/session/spans/call"] +) +def test_spend_log_traces_require_spend_log_route_access(client: TestClient, path: str) -> None: + client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver( + storage=SpendLogTraceStorage(lambda: None) + ) + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=["llm_api_routes"] + ) + response: Final = client.get(path) + assert response.status_code == 403 + + +def test_spend_log_traces_allow_a_dashboard_session(client: TestClient) -> None: + client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver( + storage=SpendLogTraceStorage(lambda: None) + ) + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=["/spend/logs/ui", "/v1/traces"] + ) + response: Final = client.get("/v1/traces") + assert response.status_code == 200 and response.json()["data"] == [] diff --git a/tests/unit/tracing/test_spend_logs.py b/tests/unit/tracing/test_spend_logs.py index 7cd451d46e9..8211630abc7 100644 --- a/tests/unit/tracing/test_spend_logs.py +++ b/tests/unit/tracing/test_spend_logs.py @@ -268,3 +268,137 @@ async def test_no_database_reads_as_no_traces() -> None: page: Final = await SpendLogTraceStorage(lambda: None).list_traces(ALL, 0, 99) assert page == {"data": (), "next_cursor": None} + + +@pytest.mark.asyncio +async def test_trace_pages_include_every_call_and_keep_complete_totals() -> None: + rows: Final = tuple(row(f"r{i:04d}", "00.000", "01.000", tags=("agent:triage",)) for i in range(601)) + storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows)) + first: Final = await storage.get_trace("s1", ALL, page_size=500) + assert first is not None and first["next_cursor"] + second: Final = await storage.get_trace("s1", ALL, cursor=first["next_cursor"], page_size=500) + assert second is not None and second["next_cursor"] is None + assert len(first["spans"]) == 500 + assert {span["span_id"] for span in (*first["spans"], *second["spans"])} == { + "agent:triage", + *(item["request_id"] for item in rows), + } + assert first["summary"]["llm_calls"] == second["summary"]["llm_calls"] == 601 + assert math.isclose(first["summary"]["spend"] or 0, 6.01) + + +@pytest.mark.asyncio +async def test_cursor_rejects_a_changed_run_and_invalid_offsets() -> None: + from litellm.tracing.errors import TraceChanged + + rows: Final = (row("r1", "00.000", "01.000"), row("r2", "02.000", "03.000")) + db: Final = FakeDatabase(rows, (*rows, row("r3", "04.000", "05.000"))) + storage: Final = SpendLogTraceStorage(lambda: db) + first: Final = await storage.get_trace("s1", ALL, page_size=1) + assert first is not None and first["next_cursor"] + with pytest.raises(TraceChanged): + await storage.get_trace("s1", ALL, cursor=first["next_cursor"], page_size=1) + + +@pytest.mark.asyncio +async def test_oversized_reads_fail_instead_of_returning_incomplete_runs() -> None: + from litellm.tracing.spend_logs import TRACE_ROW_LIMIT + + rows: Final = (row("r1", "00.000", "01.000"),) * (TRACE_ROW_LIMIT + 1) + storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows)) + with pytest.raises(OverflowError): + await storage.get_trace("s1", ALL) + + +@pytest.mark.parametrize( + "body", + [ + {"input": "Find the invoice"}, + {"input": [{"role": "user", "content": [{"type": "input_text", "text": "Find the invoice"}]}]}, + ], +) +@pytest.mark.asyncio +async def test_responses_inputs_are_visible_in_the_run_and_step(body: Mapping[str, object]) -> None: + item: Final = SpendLogRow(**{**row("r1", "00.000", "01.000"), "proxy_server_request": json.dumps(body)}) + storage: Final = SpendLogTraceStorage(lambda: FakeDatabase((item,))) + assert build_trace((item,))["summary"]["input_preview"] == "Find the invoice" + detail: Final = await storage.get_span("s1", "r1", ALL) + assert detail is not None + assert "Find the invoice" in detail["input"] + + +@pytest.mark.parametrize( + ("response", "body"), + [ + ( + { + "choices": [ + { + "message": { + "tool_calls": [ + {"id": "call-1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + ] + } + } + ] + }, + {"messages": [{"role": "tool", "tool_call_id": "call-1", "content": "Invoice paid"}]}, + ), + ( + {"content": [{"type": "tool_use", "id": "call-1", "name": "lookup", "input": {}}]}, + { + "messages": [ + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "call-1", "content": "Invoice paid"}], + } + ] + }, + ), + ( + {"output": [{"type": "function_call", "call_id": "call-1", "name": "lookup", "arguments": "{}"}]}, + {"input": [{"type": "function_call_output", "call_id": "call-1", "output": "Invoice paid"}]}, + ), + ], +) +@pytest.mark.asyncio +async def test_tool_details_use_the_result_recorded_in_the_next_request( + response: Mapping[str, object], body: Mapping[str, object] +) -> None: + rows: Final = ( + row("r1", "00.000", "01.000", response=response), + SpendLogRow(**{**row("r2", "02.000", "03.000"), "proxy_server_request": json.dumps(body)}), + ) + storage: Final = SpendLogTraceStorage(lambda: FakeDatabase(rows)) + detail: Final = await storage.get_span("s1", "r1:tool:0", ALL) + assert detail is not None and detail["output"] == "Invoice paid" + + +def test_registered_agent_names_are_readable_and_request_tags_override_them() -> None: + item: Final = SpendLogRow(**{**row("r1", "00.000", "01.000", agent_id="agent-uuid"), "agent_name": "Support"}) + assert build_trace((item,))["summary"]["name"] == "Support" + tagged: Final = SpendLogRow(**{**item, "request_tags": '["agent:Research"]'}) + assert build_trace((tagged,))["summary"]["name"] == "Research" + + +def test_offset_timestamps_preserve_the_actual_instant() -> None: + item: Final = SpendLogRow( + **{ + **row("r1", "00.000", "01.000"), + "start_time": "2026-11-06T13:00:00+08:00", + "end_time": "2026-11-06T13:00:01+08:00", + } + ) + assert build_trace((item,))["summary"]["start_time"] == "2026-11-06T05:00:00Z" + + +def test_request_agent_name_groups_calls_without_tags_or_registered_identity() -> None: + named: Final = tuple( + SpendLogRow(**{**row(f"r{i}", "00.000", "01.000"), "metadata": '{"agent_name": "Support agent"}'}) + for i in range(2) + ) + trace: Final = build_trace(named) + assert trace["summary"]["name"] == "Support agent" + assert trace["summary"]["agent_names"] == ("Support agent",) + assert trace["summary"]["llm_calls"] == 2 + assert {span["agent"] for span in trace["spans"]} == {"Support agent"} diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index d506d114d8c..1c7450dabdf 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -14,7 +14,7 @@ "@headlessui/tailwindcss": "0.2.2", "@heroicons/react": "1.0.6", "@hookform/resolvers": "5.4.0", - "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz", + "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz", "@shadcn/react": "0.3.1", "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", @@ -2096,9 +2096,9 @@ } }, "node_modules/@litellm/lens-ui": { - "version": "0.1.0-dev.0", - "resolved": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz", - "integrity": "sha512-YG8ydmSqOzCumVWSPg6G39yzLGTqfTEy1rqIC4FpysCRW+3KtYnEdpj/SsO9xzgb9tYJIdZhYBHs/027Lh0qvQ==", + "version": "0.1.0-dev.1", + "resolved": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz", + "integrity": "sha512-RqzrW3XBLuTphJcc5tBDIKf5n0ms51BZLn2Q4iH36Q/EBXWE76ZBLqWwC+5aqblcwDKNsodz2UxR1TaRvRQ21w==", "license": "MIT", "dependencies": { "@base-ui/react": "1.6.0", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 9fc0242f040..edef9c9ee7a 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -31,7 +31,7 @@ "@headlessui/tailwindcss": "0.2.2", "@heroicons/react": "1.0.6", "@hookform/resolvers": "5.4.0", - "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.0.tgz", + "@litellm/lens-ui": "file:vendor/litellm-lens-ui-0.1.0-dev.1.tgz", "@shadcn/react": "0.3.1", "@tanstack/react-pacer": "0.22.1", "@tanstack/react-query": "5.100.7", diff --git a/ui/litellm-dashboard/src/components/lens/EmbeddedLens.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/EmbeddedLens.integration.test.tsx index 11e02dde26b..68210851dae 100644 --- a/ui/litellm-dashboard/src/components/lens/EmbeddedLens.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/EmbeddedLens.integration.test.tsx @@ -37,7 +37,7 @@ it("renders the published Lens settings and uses the gateway session, prefix, an if (path === "/prefix/lens/signals") return Response.json({ model: "", threshold: 0.5, signals: [] }); if (path === "/prefix/lens/model_group/info") return Response.json({ data: [] }); if (path === "/prefix/lens/service") - return Response.json({ configured: false, connected: false, status: { storage_ready: false } }); + return Response.json({ configured: true, connected: true, status: { storage_ready: true } }); throw new Error(`Unexpected Lens request ${path}`); }); renderWithProviders(, { @@ -79,46 +79,20 @@ it("opens the packaged demo trace and preserves its shareable selection in the g expect(network).not.toHaveBeenCalled(); }); -it("keeps manual installation on the unconfigured gateway landing and copies its observed setup context", async () => { - const user = userEvent.setup(); +it("opens spend-log traces without asking to install a Lens service", async () => { network.mockImplementation(async (input, init) => { const request = input instanceof Request ? input : new Request(String(input), init); const path = new URL(request.url).pathname; - if (path === "/prefix/lens/service") { - const connection = { url: "", configured: false, connected: false, status: { storage_ready: false } }; - return Response.json(connection); - } - if (path === "/prefix/lens") - return Response.json( - { detail: "Configure LITELLM_LENS_URL and LENS_GATEWAY_SECRET for the Lens service" }, - { status: 503 }, - ); - if (path === "/prefix/v1/traces" || path === "/prefix/v1/traces/agents") - return Response.json( - { detail: "Agent tracing is not enabled. Configure the Lens service and LITELLM_LENS_URL." }, - { status: 501 }, - ); + if (path === "/prefix/lens/service") + return Response.json({ configured: false, connected: false, status: { storage_ready: false } }); + if (path === "/prefix/v1/traces") return Response.json({ data: [], next_cursor: null }); + if (path === "/prefix/v1/traces/agents") return Response.json({ agents: [] }); throw new Error(`Unexpected Lens request ${path}`); }); renderWithProviders(); - - const setup = within(await screen.findByRole("region", { name: "Get Lens running" })); - expect(setup.getByRole("button", { name: /Install Lens/ })).toHaveAttribute("aria-expanded", "true"); - expect(await setup.findByRole("link", { name: "Helm setup" })).toHaveAttribute( - "href", - "https://docs.litellm.ai/docs/proxy/lens/deployment/kubernetes#existing-deployment", - ); - expect(setup.getByRole("link", { name: "Docker setup" })).toHaveAttribute( - "href", - "https://docs.litellm.ai/docs/proxy/lens/deployment/docker-compose", - ); - expect(setup.getByRole("button", { name: "Check setup" })).toBeEnabled(); - - await user.click(setup.getByRole("button", { name: "Set it up for me" })); - const prompt = await navigator.clipboard.readText(); - expect(prompt).toContain("existing LiteLLM admin dashboard"); - expect(prompt).toContain('LiteLLM API base: "https://gateway.test/prefix"'); - expect(prompt).toContain("Lens connection is not configured"); - expect(prompt).toContain("Check for an existing Lens service before installing another"); - expect(prompt).not.toContain("gateway-session"); + expect(await screen.findByRole("heading", { name: "Your agent runs will appear here" })).toBeVisible(); + expect(screen.getByText(/x-litellm-agent-name/)).toBeVisible(); + expect(screen.queryByRole("region", { name: "Get Lens running" })).not.toBeInTheDocument(); + const tabs = within(screen.getByRole("tablist", { name: "Lens" })); + expect(tabs.getAllByRole("tab").map((tab) => tab.textContent)).toEqual(["Agents", "Traces"]); }); diff --git a/ui/litellm-dashboard/src/components/lens/EmbeddedLens.tsx b/ui/litellm-dashboard/src/components/lens/EmbeddedLens.tsx index d0e177da2bf..7236922ab95 100644 --- a/ui/litellm-dashboard/src/components/lens/EmbeddedLens.tsx +++ b/ui/litellm-dashboard/src/components/lens/EmbeddedLens.tsx @@ -10,6 +10,7 @@ import { serverRootPath } from "@/lib/serverRootPath"; const host: LensHost = { surface: "embedded", analysis: "deployment", + traces: "auto", spendLogs: { lookup: uiSpendLogsCall, Drawer: LogDetailsDrawer }, }; diff --git a/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.0.tgz b/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.0.tgz deleted file mode 100644 index f2d71612605..00000000000 Binary files a/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.0.tgz and /dev/null differ diff --git a/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.1.tgz b/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.1.tgz new file mode 100644 index 00000000000..ee65415e693 Binary files /dev/null and b/ui/litellm-dashboard/vendor/litellm-lens-ui-0.1.0-dev.1.tgz differ