From 2a2c49ad4cd711aded30da0384c1a200ddfca5d7 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 3 Sep 2026 16:09:44 -0400 Subject: [PATCH] feat(ui): surface batch results on the logs page --- .../LogDetailContent.test.tsx | 78 ++++++++++++++++ .../LogDetailsDrawer/LogDetailContent.tsx | 61 +++++++++++++ .../RequestLogsTableColumns.test.tsx | 53 +++++++++++ .../view_logs/RequestLogsTableColumns.tsx | 29 +++++- .../components/view_logs/TypeBadges.test.tsx | 14 ++- .../src/components/view_logs/TypeBadges.tsx | 26 ++++++ .../view_logs/batchLogUtils.test.ts | 90 +++++++++++++++++++ .../src/components/view_logs/batchLogUtils.ts | 67 ++++++++++++++ .../src/components/view_logs/constants.ts | 3 + 9 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx index a679dc49427..91778d2a98a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx @@ -113,6 +113,84 @@ describe("LogDetailContent", () => { expect(screen.getAllByText("$0.00200000").length).toBeGreaterThanOrEqual(1); }); + it("shows reasoning tokens in Metrics when the usage breakout carries them", () => { + render( + , + ); + + expect(screen.getByText("Reasoning Tokens")).toBeInTheDocument(); + expect(screen.getByText("224")).toBeInTheDocument(); + }); + + it("hides the reasoning metric when the breakout is absent or zero", () => { + render( + , + ); + + expect(screen.queryByText("Reasoning Tokens")).not.toBeInTheDocument(); + }); + + describe("Batch Results section", () => { + const batchCostEntry = (metadata: Record) => + createLogEntry({ + request_id: "batch_abc123_batch_cost", + call_type: "aretrieve_batch", + metadata: { status: "success", ...metadata }, + }); + + it("renders batch id, per-request outcome counts, and batch models for a batch cost row", () => { + render( + , + ); + + const section = screen.getByText("Batch Results").closest('[data-slot="card"]') as HTMLElement; + expect(within(section).getByText("batch_abc123")).toBeInTheDocument(); + expect(within(section).getByText("2")).toBeInTheDocument(); + expect(within(section).getByText("1")).toBeInTheDocument(); + expect(within(section).getByText("gemini-2.5-flash")).toBeInTheDocument(); + }); + + it("still renders the batch id when a legacy row carries no counts", () => { + render(); + + expect(screen.getByText("Batch Results")).toBeInTheDocument(); + expect(screen.getByText("batch_abc123")).toBeInTheDocument(); + expect(screen.queryByText("Successful Requests")).not.toBeInTheDocument(); + }); + + it("never renders for a non-batch call type", () => { + render( + , + ); + + expect(screen.queryByText("Batch Results")).not.toBeInTheDocument(); + }); + }); + it("should show Input Tokens and Output Tokens for anthropic_messages when uncached text_tokens exist", () => { render( + {/* Batch Results */} + {isBatchCallType(logEntry.call_type) && } + {/* Routing */} @@ -374,6 +384,53 @@ function MetricLabel({ label, tooltip, docsUrl }: { label: string; tooltip: stri ); } +/** + * Aggregate per-request outcomes for a batch cost row: batch id, success/failure counts + * from the parsed output and error files, and the models the batch actually ran on. + */ +function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { + const counts = getBatchRequestCounts(metadata); + const batchId = getBatchIdFromRequestId(logEntry.request_id); + const batchModels = getBatchModels(metadata); + if (!counts && !batchId && !batchModels) return null; + + return ( +
+ + + Batch Results + + + + {batchId && ( + + + + )} + {counts && ( + <> + + {formatNumberWithCommas(counts.successful)} + + + {counts.failed > 0 ? ( + + {formatNumberWithCommas(counts.failed)} + + ) : ( + formatNumberWithCommas(counts.failed) + )} + + + )} + {batchModels && {batchModels.join(", ")}} + + + +
+ ); +} + function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record }) { const completionStartTime = logEntry.completionStartTime; const ttftMs = @@ -391,6 +448,7 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: const uncachedInputTokens = getUncachedInputTextTokens(metadata); const showAnthropicMessagesInputOutput = logEntry.call_type === "anthropic_messages" && uncachedInputTokens !== undefined; + const reasoningTokens = getReasoningTokens(metadata); return (
@@ -416,6 +474,9 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: /> )} + {reasoningTokens !== undefined && reasoningTokens > 0 && ( + {formatNumberWithCommas(reasoningTokens)} + )} ${formatNumberWithCommas(logEntry.spend || 0, 8)} {logEntry.request_duration_ms != null ? (logEntry.request_duration_ms / 1000).toFixed(3) : "-"} s diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx index 3c9e6543c1c..3e8e522afe1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx @@ -134,6 +134,59 @@ describe("Type column", () => { expect(screen.getByText("MCP")).toBeInTheDocument(); }); + + it("marks a batch cost row with the Batch badge instead of LLM", () => { + renderRows([logEntry({ request_id: "batch_1_batch_cost", call_type: "aretrieve_batch" })]); + + expect(screen.getByText("Batch")).toBeInTheDocument(); + expect(screen.queryByText("LLM")).not.toBeInTheDocument(); + }); +}); + +describe("batch rows", () => { + const batchRow = (overrides: Partial): LogEntry => + logEntry({ + request_id: "batch_abc123_batch_cost", + call_type: "aretrieve_batch", + ...overrides, + }); + + it("rolls partial failures into the status badge instead of reporting blanket Success", async () => { + const user = userEvent.setup(); + renderRows([batchRow({ metadata: { batch_successful_requests: 2, batch_failed_requests: 1 } })]); + + expect(screen.queryByText("Success")).not.toBeInTheDocument(); + await user.hover(screen.getByText("2/3 succeeded")); + expect(await screen.findByText("1 of 3 batch requests failed")).toBeInTheDocument(); + }); + + it("keeps the Success badge when every batch request succeeded", () => { + renderRows([batchRow({ metadata: { batch_successful_requests: 3, batch_failed_requests: 0 } })]); + + expect(screen.getByText("Success")).toBeInTheDocument(); + }); + + it("keeps the Failure badge when the batch row itself failed, whatever the counts say", () => { + renderRows([batchRow({ metadata: { status: "failure", batch_successful_requests: 2, batch_failed_requests: 1 } })]); + + expect(screen.getByText("Failure")).toBeInTheDocument(); + expect(screen.queryByText("2/3 succeeded")).not.toBeInTheDocument(); + }); + + it("shows the provider batch id, not the synthetic _batch_cost request id", () => { + renderRows([batchRow({ metadata: { batch_successful_requests: 1, batch_failed_requests: 0 } })]); + + expect(screen.getByText("batch_abc123")).toBeInTheDocument(); + expect(screen.queryByText("batch_abc123_batch_cost")).not.toBeInTheDocument(); + expect(screen.getByText("batch cost")).toBeInTheDocument(); + }); + + it("leaves ordinary request ids untouched", () => { + renderRows([logEntry({ request_id: "chatcmpl-42" })]); + + expect(screen.getByText("chatcmpl-42")).toBeInTheDocument(); + expect(screen.queryByText("batch cost")).not.toBeInTheDocument(); + }); }); describe("Model column", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx index 9d4dc4f7898..b4296abe266 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx @@ -7,9 +7,10 @@ import { CellTooltip, DateCell, IdCell, MoneyCell, StatusBadge } from "@/compone import { getSpendString } from "@/utils/dataUtils"; import { getProviderLogoAndName } from "../provider_info_helpers"; +import { getBatchIdFromRequestId, getBatchRequestCounts, isBatchCallType } from "./batchLogUtils"; import type { LogEntry } from "./columns"; import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "./constants"; -import { AgentBadge, AgentIcon, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; +import { AgentBadge, AgentIcon, BatchBadge, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; export interface RequestLogsTableColumnsDeps { onKeyHashClick: (keyHash: string) => void; @@ -66,6 +67,7 @@ export const getRequestLogsTableColumns = ({ if (sessionCount <= 1) { if (isMcp) return ; if (isAgent) return ; + if (isBatchCallType(log.call_type)) return ; return ; } @@ -106,6 +108,17 @@ export const getRequestLogsTableColumns = ({ cell: ({ row }) => { const status = readMetaString(row.original.metadata, "status") ?? "Success"; const isSuccess = status.toLowerCase() !== "failure"; + const batchCounts = isSuccess ? getBatchRequestCounts(row.original.metadata) : undefined; + if (batchCounts && batchCounts.failed > 0) { + const total = batchCounts.successful + batchCounts.failed; + return ( + + ); + } return ; }, }, @@ -122,7 +135,19 @@ export const getRequestLogsTableColumns = ({ accessorKey: "request_id", header: "Request ID", enableSorting: false, - cell: ({ row }) => , + cell: ({ row }) => { + const log = row.original; + const batchId = isBatchCallType(log.call_type) ? getBatchIdFromRequestId(log.request_id) : undefined; + if (batchId) { + return ( +
+ + batch cost +
+ ); + } + return ; + }, }, { id: "spend", diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx index e3467310265..8a964ea2124 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx @@ -1,6 +1,6 @@ import { render, screen } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import { LlmBadge, McpBadge, AgentBadge } from "./TypeBadges"; +import { LlmBadge, McpBadge, AgentBadge, BatchBadge } from "./TypeBadges"; describe("TypeBadges", () => { describe("LlmBadge", () => { @@ -43,4 +43,16 @@ describe("TypeBadges", () => { expect(screen.getByText("12")).toBeInTheDocument(); }); }); + + describe("BatchBadge", () => { + it("should render with default 'Batch' text when no count is provided", () => { + render(); + expect(screen.getByText("Batch")).toBeInTheDocument(); + }); + + it("should render the count when provided", () => { + render(); + expect(screen.getByText("4")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx index 1ba4365f66e..84079848487 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx @@ -57,6 +57,25 @@ export const AgentIcon = ({ size = 12 }: { size?: number }) => ( ); +/** Stacked-layers icon for Batch API call types (Lucide Layers-style). */ +export const LayersIcon = ({ size = 12 }: { size?: number }) => ( + + + + + +); + export const LlmBadge = ({ count }: { count?: number }) => ( @@ -77,3 +96,10 @@ export const AgentBadge = ({ count }: { count?: number }) => ( {count != null ? count : "Agent"} ); + +export const BatchBadge = ({ count }: { count?: number }) => ( + + + {count != null ? count : "Batch"} + +); diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts new file mode 100644 index 00000000000..4da1e389646 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; + +import { + getBatchIdFromRequestId, + getBatchModels, + getBatchRequestCounts, + getReasoningTokens, + isBatchCallType, +} from "./batchLogUtils"; + +/** Metadata shape the batch cost poller writes on an aretrieve_batch spend row. */ +const batchCostMetadata = { + batch_models: ["gemini-2.5-flash"], + batch_successful_requests: 2, + batch_failed_requests: 1, + usage_object: { + total_tokens: 270, + prompt_tokens: 14, + completion_tokens: 256, + completion_tokens_details: { text_tokens: 32, reasoning_tokens: 224 }, + }, +}; + +describe("isBatchCallType", () => { + it("recognizes the poller's aretrieve_batch and the create call types", () => { + for (const callType of ["aretrieve_batch", "retrieve_batch", "acreate_batch", "create_batch"]) { + expect(isBatchCallType(callType)).toBe(true); + } + expect(isBatchCallType("acompletion")).toBe(false); + }); +}); + +describe("getBatchRequestCounts", () => { + it("reads both counts off a batch cost row", () => { + expect(getBatchRequestCounts(batchCostMetadata)).toEqual({ successful: 2, failed: 1 }); + }); + + it("returns undefined for a non-batch row and for null counts, so no rollup renders", () => { + expect(getBatchRequestCounts({ status: "success" })).toBeUndefined(); + expect(getBatchRequestCounts({ batch_successful_requests: null, batch_failed_requests: null })).toBeUndefined(); + expect(getBatchRequestCounts(undefined)).toBeUndefined(); + }); + + it("treats a lone present count as the other being 0, for rows logged mid-rollout", () => { + expect(getBatchRequestCounts({ batch_successful_requests: 3 })).toEqual({ successful: 3, failed: 0 }); + }); +}); + +describe("getBatchIdFromRequestId", () => { + it("strips the poller's synthetic _batch_cost suffix down to the provider batch id", () => { + expect(getBatchIdFromRequestId("batch_abc123_batch_cost")).toBe("batch_abc123"); + }); + + it("returns undefined for ordinary request ids and a bare suffix", () => { + expect(getBatchIdFromRequestId("chatcmpl-123")).toBeUndefined(); + expect(getBatchIdFromRequestId("_batch_cost")).toBeUndefined(); + }); +}); + +describe("getBatchModels", () => { + it("returns the model list from metadata.batch_models", () => { + expect(getBatchModels(batchCostMetadata)).toEqual(["gemini-2.5-flash"]); + }); + + it("returns undefined when absent, null, or empty", () => { + expect(getBatchModels({})).toBeUndefined(); + expect(getBatchModels({ batch_models: null })).toBeUndefined(); + expect(getBatchModels({ batch_models: [] })).toBeUndefined(); + }); +}); + +describe("getReasoningTokens", () => { + it("reads reasoning tokens from usage_object on a batch cost row", () => { + expect(getReasoningTokens(batchCostMetadata)).toBe(224); + }); + + it("prefers additional_usage_values, which per-request rows carry", () => { + const metadata = { + additional_usage_values: { completion_tokens_details: { reasoning_tokens: 40 } }, + usage_object: { completion_tokens_details: { reasoning_tokens: 999 } }, + }; + expect(getReasoningTokens(metadata)).toBe(40); + }); + + it("returns undefined when the breakout is null or missing", () => { + expect(getReasoningTokens({ usage_object: { completion_tokens_details: null } })).toBeUndefined(); + expect(getReasoningTokens({})).toBeUndefined(); + expect(getReasoningTokens(undefined)).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts new file mode 100644 index 00000000000..ce7793065d2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts @@ -0,0 +1,67 @@ +/** + * Helpers for reading batch-specific fields off a spend log row. + * + * The proxy's batch cost poller (CheckBatchCost) writes one spend log per completed batch + * with request_id "_batch_cost" and call_type "aretrieve_batch", carrying + * batch_models / batch_successful_requests / batch_failed_requests in metadata + * (see litellm/proxy/spend_tracking/spend_tracking_utils.py). + */ + +import { BATCH_CALL_TYPES } from "./constants"; + +export const BATCH_COST_REQUEST_ID_SUFFIX = "_batch_cost"; + +export interface BatchRequestCounts { + successful: number; + failed: number; +} + +export const isBatchCallType = (callType: string): boolean => BATCH_CALL_TYPES.includes(callType); + +const readMetaNumber = (metadata: Record | undefined, key: string): number | undefined => { + const value = metadata?.[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +}; + +/** + * Per-request outcome counts of a batch cost row. Undefined when the row carries neither + * count (a non-batch row, or a batch logged before counts were tracked). + */ +export const getBatchRequestCounts = ( + metadata: Record | undefined, +): BatchRequestCounts | undefined => { + const successful = readMetaNumber(metadata, "batch_successful_requests"); + const failed = readMetaNumber(metadata, "batch_failed_requests"); + if (successful === undefined && failed === undefined) return undefined; + return { successful: successful ?? 0, failed: failed ?? 0 }; +}; + +/** The provider batch id behind a poller-written "_batch_cost" spend row. */ +export const getBatchIdFromRequestId = (requestId: string): string | undefined => + requestId.endsWith(BATCH_COST_REQUEST_ID_SUFFIX) && requestId.length > BATCH_COST_REQUEST_ID_SUFFIX.length + ? requestId.slice(0, -BATCH_COST_REQUEST_ID_SUFFIX.length) + : undefined; + +/** The models the batch's requests actually ran on, from metadata.batch_models. */ +export const getBatchModels = (metadata: Record | undefined): string[] | undefined => { + const models = metadata?.["batch_models"]; + if (!Array.isArray(models)) return undefined; + const names = models.filter((model): model is string => typeof model === "string" && model !== ""); + return names.length > 0 ? names : undefined; +}; + +/** + * Reasoning tokens aggregated across the row's completion usage. Read from the same two + * metadata containers the drawer already uses for prompt-token details: per-request rows + * carry additional_usage_values, batch cost rows carry usage_object. + */ +export const getReasoningTokens = (metadata: Record | undefined): number | undefined => { + const readDetails = (container: unknown): number | undefined => { + if (typeof container !== "object" || container === null) return undefined; + const details = (container as Record)["completion_tokens_details"]; + if (typeof details !== "object" || details === null) return undefined; + const reasoning = (details as Record)["reasoning_tokens"]; + return typeof reasoning === "number" && Number.isFinite(reasoning) ? reasoning : undefined; + }; + return readDetails(metadata?.["additional_usage_values"]) ?? readDetails(metadata?.["usage_object"]); +}; diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts index 1c17f398e35..51889ae7d04 100644 --- a/ui/litellm-dashboard/src/components/view_logs/constants.ts +++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts @@ -21,6 +21,9 @@ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"]; /** Call types that represent agent/A2A requests (e.g. asend_message). */ export const AGENT_CALL_TYPES = ["asend_message"]; +/** Call types that represent Batch API operations (creation and retrieval, sync and async). */ +export const BATCH_CALL_TYPES = ["acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"]; + export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" },