mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
feat(logs): add span type filter to request logs (#42491)
* feat(logs): add span type filter to request logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logs): look up span type sql conditions from a mapping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Mubashir Osmani <mubashir@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
cb2f22533c
commit
1373322e0c
10 changed files with 220 additions and 1 deletions
|
|
@ -69,6 +69,18 @@ _SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)"
|
|||
_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key"
|
||||
_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')"
|
||||
_AGENT_CALL_TYPE_SQL: Final = "'asend_message'"
|
||||
_BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')"
|
||||
_SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}",
|
||||
"agent": f"call_type = {_AGENT_CALL_TYPE_SQL}",
|
||||
"batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}",
|
||||
"llm": (
|
||||
f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} "
|
||||
f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})"
|
||||
),
|
||||
}
|
||||
)
|
||||
_SPEND_LOG_LIST_COLUMNS: Final = """
|
||||
request_id, call_type, api_key, spend, total_tokens,
|
||||
prompt_tokens, completion_tokens, "startTime", "endTime",
|
||||
|
|
@ -2410,6 +2422,10 @@ async def ui_view_spend_logs(
|
|||
default=None,
|
||||
description="Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state",
|
||||
),
|
||||
span_type: str | None = fastapi.Query(
|
||||
default=None,
|
||||
description="Filter logs by span type: llm, agent, mcp, or batch",
|
||||
),
|
||||
model: str | None = fastapi.Query(default=None, description="Filter logs by model"),
|
||||
model_id: str | None = fastapi.Query(
|
||||
default=None,
|
||||
|
|
@ -2512,6 +2528,13 @@ async def ui_view_spend_logs(
|
|||
param="cache_hit_filter",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(span_type, str) and span_type not in _SPAN_TYPE_SQL_CONDITIONS:
|
||||
raise ProxyException(
|
||||
message=f"Invalid span_type: {span_type}. Must be one of: llm, agent, mcp, batch",
|
||||
type="bad_request",
|
||||
param="span_type",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
try:
|
||||
is_admin_view: Final = _is_admin_view_safe(user_api_key_dict=user_api_key_dict)
|
||||
|
|
@ -2776,6 +2799,10 @@ async def ui_view_spend_logs(
|
|||
elif cache_hit_filter == "miss":
|
||||
sql_conditions.append("(cache_hit IS NULL OR LOWER(cache_hit) != 'true')")
|
||||
|
||||
span_type_condition: Final = _span_type_sql_condition(span_type)
|
||||
if span_type_condition is not None:
|
||||
sql_conditions.append(span_type_condition)
|
||||
|
||||
if exclude_internal_health_checks:
|
||||
sql_conditions.append(f"api_key NOT IN (${p}, ${p + 1})")
|
||||
sql_params.extend(_INTERNAL_HEALTH_CHECK_API_KEYS)
|
||||
|
|
@ -4673,6 +4700,12 @@ def _build_status_filter_condition(status_filter: str | None) -> Mapping[str, ob
|
|||
return {"status": {"equals": status_filter}}
|
||||
|
||||
|
||||
def _span_type_sql_condition(span_type: str | None) -> str | None:
|
||||
if span_type is None:
|
||||
return None
|
||||
return _SPAN_TYPE_SQL_CONDITIONS.get(span_type)
|
||||
|
||||
|
||||
def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
"""
|
||||
Safely determine if the current user has admin view permissions.
|
||||
|
|
|
|||
|
|
@ -139,6 +139,15 @@ def _reconstruct_ui_where_from_sql(sql_query, params):
|
|||
where["cache_hit"] = "hit"
|
||||
elif cond == "(cache_hit IS NULL OR LOWER(cache_hit) != 'true')":
|
||||
where["cache_hit"] = "miss"
|
||||
elif "call_type" in cond:
|
||||
if "call_type NOT IN" in cond:
|
||||
where["span_type"] = "llm"
|
||||
elif "call_mcp_tool" in cond:
|
||||
where["span_type"] = "mcp"
|
||||
elif "call_type = 'asend_message'" in cond:
|
||||
where["span_type"] = "agent"
|
||||
elif "acreate_batch" in cond:
|
||||
where["span_type"] = "batch"
|
||||
elif sess:
|
||||
where["session_id"] = {"contains": str(params[int(sess.group(1)) - 1]).strip("%")}
|
||||
elif status:
|
||||
|
|
@ -3418,6 +3427,95 @@ async def test_ui_view_spend_logs_with_cache_hit_filter(client, monkeypatch):
|
|||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_with_span_type_filter(client, monkeypatch):
|
||||
base = {
|
||||
"api_key": "sk-test-key",
|
||||
"user": "test_user_1",
|
||||
"team_id": "team1",
|
||||
"spend": 0.05,
|
||||
"startTime": datetime.datetime.now(timezone.utc).isoformat(),
|
||||
"model": "gpt-4",
|
||||
"status": "success",
|
||||
}
|
||||
mock_spend_logs = [
|
||||
{**base, "id": "log1", "request_id": "req-llm", "call_type": "acompletion"},
|
||||
{**base, "id": "log2", "request_id": "req-agent", "call_type": "asend_message"},
|
||||
{**base, "id": "log3", "request_id": "req-mcp", "call_type": "call_mcp_tool"},
|
||||
{**base, "id": "log4", "request_id": "req-batch", "call_type": "aretrieve_batch"},
|
||||
]
|
||||
|
||||
call_types_by_span = {
|
||||
"llm": lambda ct: ct not in {"call_mcp_tool", "list_mcp_tools", "asend_message"}
|
||||
and ct not in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"},
|
||||
"agent": lambda ct: ct == "asend_message",
|
||||
"mcp": lambda ct: ct in {"call_mcp_tool", "list_mcp_tools"},
|
||||
"batch": lambda ct: ct
|
||||
in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"},
|
||||
}
|
||||
|
||||
def filter_by_span_type(where):
|
||||
span_type = where.get("span_type")
|
||||
if span_type is None:
|
||||
return mock_spend_logs
|
||||
return [log for log in mock_spend_logs if call_types_by_span[span_type](log["call_type"])]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.prisma_client",
|
||||
make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_by_span_type),
|
||||
)
|
||||
|
||||
start_date, end_date = _default_date_range()
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN
|
||||
)
|
||||
try:
|
||||
for span_type, expected_ids in [
|
||||
("batch", ["req-batch"]),
|
||||
("llm", ["req-llm"]),
|
||||
("mcp", ["req-mcp"]),
|
||||
("agent", ["req-agent"]),
|
||||
]:
|
||||
response = client.get(
|
||||
"/spend/logs/ui",
|
||||
params={
|
||||
"span_type": span_type,
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
},
|
||||
headers={"Authorization": "Bearer sk-test"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
data = response.json()
|
||||
assert data["total"] == len(expected_ids)
|
||||
assert [row["request_id"] for row in data["data"]] == expected_ids
|
||||
|
||||
response = client.get(
|
||||
"/spend/logs/ui",
|
||||
params={
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
},
|
||||
headers={"Authorization": "Bearer sk-test"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["total"] == 4
|
||||
|
||||
response = client.get(
|
||||
"/spend/logs/ui",
|
||||
params={
|
||||
"span_type": "invalid",
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
},
|
||||
headers={"Authorization": "Bearer sk-test"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ui_view_spend_logs_with_model(client, monkeypatch):
|
||||
mock_spend_logs = [
|
||||
|
|
|
|||
|
|
@ -2003,6 +2003,7 @@ interface UiSpendLogsParams {
|
|||
end_user?: string;
|
||||
status_filter?: string;
|
||||
cache_hit_filter?: string;
|
||||
span_type?: string;
|
||||
/** Filter by model name (e.g. "gpt-4") */
|
||||
model?: string;
|
||||
/** Filter by model ID (litellm model deployment id) */
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ describe("RequestLogsFilters", () => {
|
|||
|
||||
for (const label of [
|
||||
"Team ID",
|
||||
"Span Type",
|
||||
"Status",
|
||||
"Cache",
|
||||
"Key Alias",
|
||||
|
|
@ -286,6 +287,38 @@ describe("RequestLogsFilters", () => {
|
|||
expect(await screen.findByText(label)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["", "All Types"],
|
||||
["llm", "LLM"],
|
||||
["agent", "Agent"],
|
||||
["mcp", "MCP"],
|
||||
["batch", "Batch"],
|
||||
])("shows the human label on the Span Type trigger for %s", async (spanType, label) => {
|
||||
renderFilters(spanType === "" ? {} : { [LOG_FILTER_IDS.SPAN_TYPE]: spanType });
|
||||
|
||||
expect(await screen.findByText(label)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("selecting Batch sets the span_type filter", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { set } = renderFilters();
|
||||
|
||||
await user.click(await screen.findByText("All Types"));
|
||||
await user.click(await screen.findByRole("option", { name: "Batch" }));
|
||||
|
||||
expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.SPAN_TYPE, "batch");
|
||||
});
|
||||
|
||||
it("selecting All Types clears the span_type filter", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { set } = renderFilters({ [LOG_FILTER_IDS.SPAN_TYPE]: "batch" });
|
||||
|
||||
await user.click(await screen.findByText("Batch"));
|
||||
await user.click(await screen.findByRole("option", { name: "All Types" }));
|
||||
|
||||
expect(set).toHaveBeenCalledWith(LOG_FILTER_IDS.SPAN_TYPE, undefined);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Cache Hit", "hit"],
|
||||
["Cache Miss", "miss"],
|
||||
|
|
|
|||
|
|
@ -37,6 +37,14 @@ const CACHE_FILTER_ITEMS = [
|
|||
{ value: "hit", label: "Cache Hit" },
|
||||
{ value: "miss", label: "Cache Miss" },
|
||||
] as const;
|
||||
|
||||
const SPAN_TYPE_FILTER_ITEMS = [
|
||||
{ value: ALL_VALUE, label: "All Types" },
|
||||
{ value: "llm", label: "LLM" },
|
||||
{ value: "agent", label: "Agent" },
|
||||
{ value: "mcp", label: "MCP" },
|
||||
{ value: "batch", label: "Batch" },
|
||||
] as const;
|
||||
const PAGE_SIZE = 50;
|
||||
|
||||
const SEARCH_INPUT_REASONS: ReadonlySet<string> = new Set(["input-change", "input-clear", "clear-press"]);
|
||||
|
|
@ -328,6 +336,27 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF
|
|||
teams={teams}
|
||||
/>
|
||||
|
||||
<DataTableFilterField label="Span Type">
|
||||
<Select
|
||||
items={SPAN_TYPE_FILTER_ITEMS}
|
||||
value={valueOf(LOG_FILTER_IDS.SPAN_TYPE) === "" ? ALL_VALUE : valueOf(LOG_FILTER_IDS.SPAN_TYPE)}
|
||||
onValueChange={(next) =>
|
||||
set(LOG_FILTER_IDS.SPAN_TYPE, next === null || next === ALL_VALUE ? undefined : next)
|
||||
}
|
||||
>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="All Types" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{SPAN_TYPE_FILTER_ITEMS.map((item) => (
|
||||
<SelectItem key={item.value} value={item.value}>
|
||||
{item.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</DataTableFilterField>
|
||||
|
||||
<DataTableFilterField label="Status">
|
||||
<Select
|
||||
items={STATUS_FILTER_ITEMS}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ import { DataTable, DataTableFilterDrawer, DataTableToolbar } from "@/components
|
|||
|
||||
import type { Team } from "../key_team_helpers/key_list";
|
||||
import type { LogEntry } from "./columns";
|
||||
import { LOG_FILTER_LABELS, type LogsWindow } from "./log_filter_logic";
|
||||
import { SPAN_TYPE_LABELS } from "./constants";
|
||||
import { LOG_FILTER_IDS, LOG_FILTER_LABELS, type LogsWindow } from "./log_filter_logic";
|
||||
import { RequestLogsFilters } from "./RequestLogsFilters";
|
||||
import { getRequestLogsTableColumns } from "./RequestLogsTableColumns";
|
||||
|
||||
|
|
@ -35,6 +36,13 @@ interface RequestLogsTableProps {
|
|||
toolbarChildren?: ReactNode;
|
||||
}
|
||||
|
||||
const formatFilterValue = (columnId: string, value: unknown): string => {
|
||||
if (columnId === LOG_FILTER_IDS.SPAN_TYPE) {
|
||||
return SPAN_TYPE_LABELS[String(value)] ?? String(value);
|
||||
}
|
||||
return Array.isArray(value) ? value.join(", ") : String(value);
|
||||
};
|
||||
|
||||
function RequestLogsEmptyState({ filtered }: { filtered: boolean }) {
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-1 py-6">
|
||||
|
|
@ -116,6 +124,7 @@ export function RequestLogsTable({
|
|||
isRefreshing={isRefreshing}
|
||||
onOpenFilters={() => setFiltersOpen(true)}
|
||||
filterLabels={LOG_FILTER_LABELS}
|
||||
formatFilterValue={formatFilterValue}
|
||||
showViewOptions={false}
|
||||
>
|
||||
{toolbarChildren}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,13 @@ export const AGENT_CALL_TYPES = ["asend_message"];
|
|||
/** Call types that represent Batch API operations (creation and retrieval, sync and async). */
|
||||
export const BATCH_CALL_TYPES = ["acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"];
|
||||
|
||||
export const SPAN_TYPE_LABELS: Record<string, string> = {
|
||||
llm: "LLM",
|
||||
agent: "Agent",
|
||||
mcp: "MCP",
|
||||
batch: "Batch",
|
||||
};
|
||||
|
||||
export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [
|
||||
{ label: "Last Minute", value: 1, unit: "minutes" },
|
||||
{ label: "Last 15 Minutes", value: 15, unit: "minutes" },
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ describe("useLogFilterLogic", () => {
|
|||
{ id: LOG_FILTER_IDS.STATUS, value: "failure", param: "status_filter" },
|
||||
{ id: LOG_FILTER_IDS.CACHE_STATUS, value: "hit", param: "cache_hit_filter" },
|
||||
{ id: LOG_FILTER_IDS.CACHE_STATUS, value: "miss", param: "cache_hit_filter" },
|
||||
{ id: LOG_FILTER_IDS.SPAN_TYPE, value: "batch", param: "span_type" },
|
||||
{ id: LOG_FILTER_IDS.SPAN_TYPE, value: "mcp", param: "span_type" },
|
||||
{ id: LOG_FILTER_IDS.MODEL_ID, value: "model-uuid-1", param: "model_id" },
|
||||
{ id: LOG_FILTER_IDS.PUBLIC_MODEL_OR_SEARCH_TOOL, value: "gpt-4o", param: "model" },
|
||||
{ id: LOG_FILTER_IDS.KEY_ALIAS, value: "alias-1", param: "key_alias" },
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ export interface PaginatedResponse {
|
|||
|
||||
export const LOG_FILTER_IDS = {
|
||||
TEAM_ID: "team_id",
|
||||
SPAN_TYPE: "span_type",
|
||||
STATUS: "status",
|
||||
CACHE_STATUS: "cache_hit",
|
||||
KEY_ALIAS: "key_alias",
|
||||
|
|
@ -38,6 +39,7 @@ export const LOG_FILTER_IDS = {
|
|||
|
||||
export const LOG_FILTER_LABELS: Record<string, string> = {
|
||||
[LOG_FILTER_IDS.TEAM_ID]: "Team ID",
|
||||
[LOG_FILTER_IDS.SPAN_TYPE]: "Span Type",
|
||||
[LOG_FILTER_IDS.STATUS]: "Status",
|
||||
[LOG_FILTER_IDS.CACHE_STATUS]: "Cache",
|
||||
[LOG_FILTER_IDS.KEY_ALIAS]: "Key Alias",
|
||||
|
|
@ -183,6 +185,7 @@ export function useLogFilterLogic({
|
|||
end_user: getFilterValue(columnFilters, LOG_FILTER_IDS.END_USER),
|
||||
status_filter: getFilterValue(columnFilters, LOG_FILTER_IDS.STATUS),
|
||||
cache_hit_filter: getFilterValue(columnFilters, LOG_FILTER_IDS.CACHE_STATUS),
|
||||
span_type: getFilterValue(columnFilters, LOG_FILTER_IDS.SPAN_TYPE),
|
||||
model_id: getFilterValue(columnFilters, LOG_FILTER_IDS.MODEL_ID),
|
||||
model: getFilterValue(columnFilters, LOG_FILTER_IDS.PUBLIC_MODEL_OR_SEARCH_TOOL),
|
||||
key_alias: getFilterValue(columnFilters, LOG_FILTER_IDS.KEY_ALIAS),
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -61025,6 +61025,8 @@ export interface operations {
|
|||
status_filter?: string | null;
|
||||
/** @description Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state */
|
||||
cache_hit_filter?: string | null;
|
||||
/** @description Filter logs by span type: llm, agent, mcp, or batch */
|
||||
span_type?: string | null;
|
||||
/** @description Filter logs by model */
|
||||
model?: string | null;
|
||||
/** @description Filter logs by model ID (litellm model deployment id) */
|
||||
|
|
@ -61143,6 +61145,8 @@ export interface operations {
|
|||
status_filter?: string | null;
|
||||
/** @description Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state */
|
||||
cache_hit_filter?: string | null;
|
||||
/** @description Filter logs by span type: llm, agent, mcp, or batch */
|
||||
span_type?: string | null;
|
||||
/** @description Filter logs by model */
|
||||
model?: string | null;
|
||||
/** @description Filter logs by model ID (litellm model deployment id) */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue