From bde3f6ae4605d356ac298ce470396c1b89ec535d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 19:35:43 -0700 Subject: [PATCH 1/7] feat(ui): show guardrail usage units and cost on the Guardrails Monitor The overview table gains Usage Units and Cost columns plus a Guardrail Cost card, and the detail page gains a Usage & Cost section that breaks units and cost down by counter, team and key. Units the cost map could not price are called out next to the cost they are left out of. Both pages now read /guardrails/usage/* through $api.useQuery so the rows are typed from schema.d.ts; the hand-written PerformanceRow and the untyped fetch helpers are gone. fetchClient resolves fetch per request so integration tests that stub the global see typed-client calls too. Refs LIT-5652 --- .../_components/GuardrailDetail.test.tsx | 59 +++-- .../_components/GuardrailDetail.tsx | 12 +- .../GuardrailUsageBreakdown.test.tsx | 114 ++++++++++ .../_components/GuardrailUsageBreakdown.tsx | 159 ++++++++++++++ .../GuardrailsMonitorView.test.tsx | 20 +- .../_components/GuardrailsOverview.test.tsx | 204 +++++++++++++----- .../_components/GuardrailsOverview.tsx | 121 ++++++++--- .../page.integration.test.tsx | 27 ++- .../guardrails/useGuardrailsUsage.test.ts | 81 +++++++ .../hooks/guardrails/useGuardrailsUsage.ts | 38 ++++ .../GuardrailsMonitor/MetricCard.tsx | 2 +- .../components/GuardrailsMonitor/mockData.ts | 33 --- .../GuardrailsMonitor/usageUnits.test.ts | 55 +++++ .../GuardrailsMonitor/usageUnits.ts | 21 ++ .../src/components/networking.tsx | 57 ----- ui/litellm-dashboard/src/lib/http/api.ts | 12 +- 16 files changed, 789 insertions(+), 226 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx index c7567aa80bb..bbcb8138d52 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx @@ -2,12 +2,16 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; import { render, screen, waitFor } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailDetail } from "./GuardrailDetail"; -const mockGetGuardrailsUsageDetail = vi.fn(); +const mockUseGuardrailsUsageDetail = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageDetail: (...args: unknown[]) => mockUseGuardrailsUsageDetail(...args), +})); + const mockGetGuardrailsUsageLogs = vi.fn(); vi.mock("@/components/networking", () => ({ - getGuardrailsUsageDetail: (...args: unknown[]) => mockGetGuardrailsUsageDetail(...args), getGuardrailsUsageLogs: (...args: unknown[]) => mockGetGuardrailsUsageLogs(...args), })); @@ -19,7 +23,8 @@ vi.mock("./EvaluationSettingsModal", () => ({ EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ?
: null), })); -const detail = { +const detail: GuardrailUsageDetail = { + guardrail_id: "pii-detector", guardrail_name: "pii-detector", description: "Blocks personally identifiable information", status: "warning", @@ -29,12 +34,25 @@ const detail = { failRate: 20, avgScore: 0.4, avgLatency: 180, + trend: "stable", + time_series: [], + usage_units: { sensitiveInformationPolicyUnits: 4 }, + usage_units_daily: [], + usage_units_by_team: { "": { sensitiveInformationPolicyUnits: 4 } }, + usage_units_by_key: { "hash-1": { sensitiveInformationPolicyUnits: 4 } }, + cost: 0.0004, + cost_by_unit: { sensitiveInformationPolicyUnits: 0.0004 }, + cost_by_team: { "": 0.0004 }, + cost_by_key: { "hash-1": 0.0004 }, + untracked_usage_units: {}, }; +const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null }); + const defaultProps = { guardrailId: "pii-detector", onBack: vi.fn(), - accessToken: "test-token", + accessToken: "test-token" as string | null, startDate: "2026-07-01", endDate: "2026-07-24", }; @@ -49,19 +67,19 @@ function renderDetail(props: Partial = {}) { describe("GuardrailDetail", () => { beforeEach(() => { vi.clearAllMocks(); - mockGetGuardrailsUsageDetail.mockResolvedValue(detail); + mockUseGuardrailsUsageDetail.mockReturnValue(loaded(detail)); mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 }); }); it("should show a busy indicator while the detail request is in flight", () => { - mockGetGuardrailsUsageDetail.mockReturnValue(new Promise(() => {})); + mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: true, error: null }); renderDetail(); expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument(); expect(screen.queryByText("pii-detector")).not.toBeInTheDocument(); }); it("should show an error message and a way back when the detail request fails", async () => { - mockGetGuardrailsUsageDetail.mockRejectedValue(new Error("boom")); + mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: false, error: new Error("boom") }); renderDetail(); expect(await screen.findByText("Failed to load guardrail details.")).toBeInTheDocument(); expect(screen.getByRole("button", { name: /back to overview/i })).toBeInTheDocument(); @@ -69,14 +87,13 @@ describe("GuardrailDetail", () => { it("should request the detail and the logs for the guardrail and date range", async () => { renderDetail(); - await waitFor(() => - expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith( - "test-token", - "pii-detector", - "2026-07-01", - "2026-07-24", - ), - ); + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith({ + accessToken: "test-token", + guardrailId: "pii-detector", + startDate: "2026-07-01", + endDate: "2026-07-24", + }); + await waitFor(() => expect(mockGetGuardrailsUsageLogs).toHaveBeenCalled()); expect(mockGetGuardrailsUsageLogs).toHaveBeenCalledWith( "test-token", expect.objectContaining({ guardrailId: "pii-detector", startDate: "2026-07-01", endDate: "2026-07-24" }), @@ -100,11 +117,18 @@ describe("GuardrailDetail", () => { }); it("should show a placeholder when no latency has been recorded", async () => { - mockGetGuardrailsUsageDetail.mockResolvedValue({ ...detail, avgLatency: null }); + mockUseGuardrailsUsageDetail.mockReturnValue(loaded({ ...detail, avgLatency: null })); renderDetail(); expect(await screen.findByText("No data")).toBeInTheDocument(); }); + it("should show the usage and cost breakdown for the guardrail on the overview tab", async () => { + renderDetail(); + const section = await screen.findByRole("region", { name: "Usage and cost" }); + expect(section).toHaveTextContent("$0.0004"); + expect(section).toHaveTextContent("Sensitive Information Policy"); + }); + it("should call onBack when 'Back to Overview' is clicked", async () => { const user = userEvent.setup(); const onBack = vi.fn(); @@ -138,8 +162,9 @@ describe("GuardrailDetail", () => { }); it("should not request anything without an access token", () => { + mockUseGuardrailsUsageDetail.mockReturnValue(loaded(undefined)); renderDetail({ accessToken: null }); - expect(mockGetGuardrailsUsageDetail).not.toHaveBeenCalled(); + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith(expect.objectContaining({ accessToken: null })); expect(mockGetGuardrailsUsageLogs).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx index 253477ffeac..1e82f1fee85 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx @@ -1,13 +1,15 @@ import { useQuery } from "@tanstack/react-query"; import { ArrowLeft, Settings, Shield, TriangleAlert } from "lucide-react"; import React, { useMemo, useState } from "react"; -import { getGuardrailsUsageDetail, getGuardrailsUsageLogs } from "@/components/networking"; +import { getGuardrailsUsageLogs } from "@/components/networking"; +import { useGuardrailsUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { StatusBadge, type StatusTone } from "@/components/shared/table_cells/status_badge"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; +import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; import { LogViewer } from "@/components/GuardrailsMonitor/LogViewer"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; import type { LogEntry } from "@/components/GuardrailsMonitor/mockData"; @@ -36,11 +38,7 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start data: detailData, isLoading: detailLoading, error: detailError, - } = useQuery({ - queryKey: ["guardrails-usage-detail", guardrailId, startDate, endDate], - queryFn: () => getGuardrailsUsageDetail(accessToken!, guardrailId, startDate, endDate), - enabled: !!accessToken && !!guardrailId, - }); + } = useGuardrailsUsageDetail({ accessToken, guardrailId, startDate, endDate }); const { data: logsData, isLoading: logsLoading } = useQuery({ queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize], queryFn: () => @@ -194,6 +192,8 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start />
+ {detailData && } + {logViewer("all")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx new file mode 100644 index 00000000000..3b24185f0b6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -0,0 +1,114 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; + +const detail: GuardrailUsageDetail = { + guardrail_id: "bedrock-pii-mask", + guardrail_name: "bedrock-pii-mask", + type: "pii", + provider: "Bedrock", + requestsEvaluated: 5, + failRate: 0, + avgScore: null, + avgLatency: 120, + status: "healthy", + trend: "stable", + description: null, + time_series: [], + usage_units: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300, someFutureCounter: 7 }, + usage_units_daily: [], + usage_units_by_team: { + "team-a": { contentPolicyUnits: 900, sensitiveInformationPolicyUnits: 300 }, + "": { contentPolicyUnits: 100, someFutureCounter: 7 }, + }, + usage_units_by_key: { + "hash-1": { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300 }, + "hash-2": { someFutureCounter: 7 }, + }, + cost: 0.18, + cost_by_unit: { contentPolicyUnits: 0.15, sensitiveInformationPolicyUnits: 0.03, someFutureCounter: null }, + cost_by_team: { "team-a": 0.165, "": 0.015 }, + cost_by_key: { "hash-1": 0.18, "hash-2": null }, + untracked_usage_units: { someFutureCounter: 7 }, +}; + +const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); + +describe("GuardrailUsageBreakdown", () => { + it("totals the units and the cost, and says how many units the cost leaves out", () => { + render(); + + const cost = screen.getByRole("group", { name: "Cost" }); + expect(cost).toHaveTextContent("$0.1800"); + expect(cost).toHaveTextContent("7 units unpriced"); + + const units = screen.getByRole("group", { name: "Usage Units" }); + expect(units).toHaveTextContent("1,307"); + expect(units).toHaveTextContent("3 counters"); + }); + + it("lists each counter with its units, cost and unpriced share", () => { + render(); + + const content = rowNamed("Content Policy"); + expect(within(content).getByText("1,000")).toBeInTheDocument(); + expect(within(content).getByText("$0.1500")).toBeInTheDocument(); + expect(within(content).getByText("—")).toBeInTheDocument(); + + const future = rowNamed("Some Future Counter"); + expect(within(future).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); + expect(within(future).getByText("—")).toBeInTheDocument(); + }); + + it("breaks units and cost down by team and by key, naming the rows without one", () => { + render(); + + expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "By key" })).toBeInTheDocument(); + const teamA = rowNamed("team-a"); + expect(within(teamA).getByText("1,200")).toBeInTheDocument(); + expect(within(teamA).getByText("$0.1650")).toBeInTheDocument(); + + const noTeam = rowNamed("No team"); + expect(within(noTeam).getByText("107")).toBeInTheDocument(); + expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument(); + + const unpricedKey = rowNamed("hash-2"); + expect(within(unpricedKey).getByText("7")).toBeInTheDocument(); + expect(within(unpricedKey).getByText("—")).toBeInTheDocument(); + }); + + it("orders teams and keys by units, largest first", () => { + render(); + + const rows = screen.getAllByRole("row").map((row) => row.textContent ?? ""); + expect(rows.findIndex((text) => text.includes("team-a"))).toBeLessThan( + rows.findIndex((text) => text.includes("No team")), + ); + expect(rows.findIndex((text) => text.includes("hash-1"))).toBeLessThan( + rows.findIndex((text) => text.includes("hash-2")), + ); + }); + + it("says so when the window has no billable units instead of rendering empty tables", () => { + render( + , + ); + + expect(screen.getByText("No billable usage units were recorded in this period.")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx new file mode 100644 index 00000000000..1eaab86c506 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -0,0 +1,159 @@ +import type { ColumnDef } from "@tanstack/react-table"; +import { CircleDollarSign } from "lucide-react"; +import React from "react"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { DataTable } from "@/components/shared/DataTable"; +import { IdCell } from "@/components/shared/table_cells/id_cell"; +import { MoneyCell } from "@/components/shared/table_cells/money_cell"; + +interface CounterRow { + counter: string; + units: number; + cost: number | null; + unpriced: number; +} + +interface GroupRow { + id: string; + units: number; + cost: number | null; +} + +const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => + Object.entries(detail.usage_units).map(([counter, units]) => ({ + counter, + units, + cost: detail.cost_by_unit[counter] ?? null, + unpriced: detail.untracked_usage_units[counter] ?? 0, + })); + +const groupRows = ( + unitsByGroup: GuardrailUsageDetail["usage_units_by_team"], + costByGroup: GuardrailUsageDetail["cost_by_team"], +): GroupRow[] => + Object.entries(unitsByGroup) + .map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null })) + .sort((a, b) => b.units - a.units); + +const counterColumns: ColumnDef[] = [ + { header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) }, + { + header: "Units", + accessorKey: "units", + meta: { numeric: true }, + cell: ({ row }) => row.original.units.toLocaleString(), + }, + { + header: "Cost", + accessorKey: "cost", + meta: { numeric: true }, + cell: ({ row }) => , + }, + { + header: "Unpriced Units", + accessorKey: "unpriced", + meta: { numeric: true }, + cell: ({ row }) => + row.original.unpriced > 0 ? ( + {row.original.unpriced.toLocaleString()} + ) : ( + — + ), + }, +]; + +const groupColumns = (label: string, emptyLabel: string): ColumnDef[] => [ + { + header: label, + accessorKey: "id", + cell: ({ row }) => + row.original.id ? ( + + ) : ( + {emptyLabel} + ), + }, + { + header: "Units", + accessorKey: "units", + meta: { numeric: true }, + cell: ({ row }) => row.original.units.toLocaleString(), + }, + { + header: "Cost", + accessorKey: "cost", + meta: { numeric: true }, + cell: ({ row }) => , + }, +]; + +const teamColumns = groupColumns("Team", "No team"); +const keyColumns = groupColumns("Key", "No key"); + +const TableHeading = ({ title }: { title: string }) => ( +
{title}
+); + +export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDetail }) { + const counters = counterRows(detail); + const unpriced = unpricedSummary(detail.untracked_usage_units); + + return ( +
+
+
Usage & Cost
+

+ Billable units the provider reported for this guardrail and what LiteLLM priced them at +

+
+ + {counters.length === 0 ? ( +

No billable usage units were recorded in this period.

+ ) : ( + <> +
+ } + subtitle={unpriced ?? undefined} + /> + +
+ + row.counter} + size="compact" + toolbar={() => } + /> + +
+ row.id || "no-team"} + size="compact" + toolbar={() => } + /> + row.id || "no-key"} + size="compact" + toolbar={() => } + /> +
+ + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index 9f27daab6b3..83b3a8f5d58 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -2,14 +2,15 @@ import { render, screen, waitFor } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { describe, expect, it, vi } from "vitest"; import GuardrailsMonitorView from "./GuardrailsMonitorView"; -import * as networking from "@/components/networking"; vi.mock("@/components/networking", () => ({ - getGuardrailsUsageOverview: vi.fn(), formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const mockUseGuardrailsUsageOverview = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageOverview: (...args: unknown[]) => mockUseGuardrailsUsageOverview(...args), +})); function wrapper({ children }: { children: React.ReactNode }) { const queryClient = new QueryClient({ @@ -22,23 +23,20 @@ function wrapper({ children }: { children: React.ReactNode }) { describe("GuardrailsMonitorView", () => { it("should render overview and fetch guardrails usage when accessToken is provided", async () => { - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [], - chart: [], - totalRequests: 0, - totalBlocked: 0, - passRate: 100, - }); + mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: true, error: null }); render(, { wrapper }); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); await waitFor(() => { - expect(mockGetGuardrailsUsageOverview).toHaveBeenCalled(); + expect(mockUseGuardrailsUsageOverview).toHaveBeenCalledWith( + expect.objectContaining({ accessToken: "test-token", startDate: expect.any(String) }), + ); }); }); it("should render without crashing when accessToken is null", async () => { + mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: false, error: null }); render(, { wrapper }); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index c62505cc74f..3a56667b156 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -1,12 +1,15 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import * as networking from "@/components/networking"; +import type { + GuardrailUsageOverview, + GuardrailUsageOverviewRow, +} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailsOverview } from "./GuardrailsOverview"; -vi.mock("@/components/networking", () => ({ - getGuardrailsUsageOverview: vi.fn(), +const useGuardrailsUsageOverviewMock = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageOverview: (...args: unknown[]) => useGuardrailsUsageOverviewMock(...args), })); vi.mock("./ScoreChart", () => ({ @@ -17,16 +20,63 @@ vi.mock("./EvaluationSettingsModal", () => ({ EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ?
Evaluation settings modal
: null), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const row = (overrides: Partial): GuardrailUsageOverviewRow => ({ + id: "guardrail", + name: "Guardrail", + type: "content_filter", + provider: "LiteLLM", + requestsEvaluated: 0, + failRate: 0, + avgScore: null, + avgLatency: null, + status: "healthy", + trend: "stable", + usageUnits: {}, + cost: null, + untrackedUsageUnits: {}, + ...overrides, +}); -function wrapper({ children }: { children: React.ReactNode }) { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, - }, - }); - return {children}; -} +const overview: GuardrailUsageOverview = { + rows: [ + row({ + id: "guardrail-low", + name: "Low Failure Guardrail", + requestsEvaluated: 1200, + failRate: 2.5, + avgLatency: 45, + trend: "down", + }), + row({ + id: "guardrail-high", + name: "High Failure Guardrail", + provider: "Bedrock", + requestsEvaluated: 300, + failRate: 18, + status: "warning", + trend: "up", + usageUnits: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 250 }, + cost: 0.15, + untrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 }, + }), + row({ + id: "guardrail-free", + name: "Free Bedrock Guardrail", + provider: "Bedrock", + requestsEvaluated: 10, + failRate: 0, + usageUnits: { contentPolicyUnits: 40 }, + cost: 0, + }), + ], + chart: [], + totalRequests: 1510, + totalBlocked: 84, + passRate: 94.4, + totalUsageUnits: { contentPolicyUnits: 1040, sensitiveInformationPolicyUnits: 250 }, + totalCost: 0.15, + totalUntrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 }, +}; function renderOverview(onSelectGuardrail = vi.fn()) { return render( @@ -36,41 +86,24 @@ function renderOverview(onSelectGuardrail = vi.fn()) { endDate="2026-08-12" onSelectGuardrail={onSelectGuardrail} />, - { wrapper }, ); } +const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); + describe("GuardrailsOverview", () => { beforeEach(() => { vi.clearAllMocks(); - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [ - { - id: "guardrail-low", - name: "Low Failure Guardrail", - type: "content_filter", - provider: "LiteLLM", - requestsEvaluated: 1200, - failRate: 2.5, - avgLatency: 45, - status: "healthy", - trend: "down", - }, - { - id: "guardrail-high", - name: "High Failure Guardrail", - type: "content_filter", - provider: "Bedrock", - requestsEvaluated: 300, - failRate: 18, - status: "warning", - trend: "up", - }, - ], - chart: [], - totalRequests: 1500, - totalBlocked: 84, - passRate: 94.4, + useGuardrailsUsageOverviewMock.mockReturnValue({ data: overview, isLoading: false, error: null }); + }); + + it("asks for the usage overview of the selected window", () => { + renderOverview(); + + expect(useGuardrailsUsageOverviewMock).toHaveBeenCalledWith({ + accessToken: "test-token", + startDate: "2026-08-01", + endDate: "2026-08-12", }); }); @@ -78,15 +111,7 @@ describe("GuardrailsOverview", () => { const onSelectGuardrail = vi.fn(); const user = userEvent.setup(); - render( - , - { wrapper }, - ); + renderOverview(onSelectGuardrail); expect(await screen.findByRole("columnheader", { name: "Guardrail" })).toBeInTheDocument(); expect(screen.getByRole("columnheader", { name: /Requests/ })).toBeInTheDocument(); @@ -105,6 +130,46 @@ describe("GuardrailsOverview", () => { expect(onSelectGuardrail).toHaveBeenCalledWith("guardrail-low"); }); + it("shows each guardrail's usage units and cost, marking the units cost leaves out", async () => { + renderOverview(); + + expect(await screen.findByRole("columnheader", { name: "Usage Units" })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Cost/ })).toBeInTheDocument(); + + const priced = rowNamed("High Failure Guardrail"); + expect(within(priced).getByText("1,250")).toBeInTheDocument(); + expect(within(priced).getByText("$0.1500")).toBeInTheDocument(); + expect(within(priced).getByLabelText("250 units unpriced")).toBeInTheDocument(); + + const free = rowNamed("Free Bedrock Guardrail"); + expect(within(free).getByText("40")).toBeInTheDocument(); + expect(within(free).getByText("$0.0000")).toBeInTheDocument(); + expect(within(free).queryByLabelText(/unpriced/)).not.toBeInTheDocument(); + + const unmetered = rowNamed("Low Failure Guardrail"); + expect(within(unmetered).getAllByText("—")).toHaveLength(2); + }); + + it("breaks the usage units down per counter on hover", async () => { + const user = userEvent.setup(); + renderOverview(); + + await user.hover(within(rowNamed("High Failure Guardrail")).getByText("1,250")); + + expect(await screen.findByText("Content Policy: 1,000")).toBeInTheDocument(); + expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument(); + }); + + it("sorts by cost when its header is clicked", async () => { + const user = userEvent.setup(); + renderOverview(); + + await user.click(await screen.findByRole("button", { name: /Cost/ })); + + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail")); + expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail"); + }); + it("renders the page header and the export action", async () => { renderOverview(); @@ -117,15 +182,36 @@ describe("GuardrailsOverview", () => { it("renders every summary metric card", async () => { renderOverview(); - expect(await screen.findByText("1,500")).toBeInTheDocument(); + expect(await screen.findByText("1,510")).toBeInTheDocument(); expect(screen.getByText("Total Evaluations")).toBeInTheDocument(); expect(screen.getByText("Blocked Requests")).toBeInTheDocument(); expect(screen.getByText("84")).toBeInTheDocument(); expect(screen.getByText("Pass Rate")).toBeInTheDocument(); expect(screen.getByText("94.4%")).toBeInTheDocument(); - expect(screen.getByText("23ms")).toBeInTheDocument(); + expect(screen.getByText("15ms")).toBeInTheDocument(); expect(screen.getByText("Active Guardrails")).toBeInTheDocument(); - expect(screen.getByText("2")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + }); + + it("totals guardrail cost across the window and says how many units it leaves out", async () => { + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + expect(card).toHaveTextContent("$0.1500"); + expect(card).toHaveTextContent("250 units unpriced"); + }); + + it("shows a dash for guardrail cost when nothing in the window was priced", async () => { + useGuardrailsUsageOverviewMock.mockReturnValue({ + data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} }, + isLoading: false, + error: null, + }); + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + expect(card).toHaveTextContent("—"); + expect(card).not.toHaveTextContent("unpriced"); }); it("renders the table toolbar heading and its description", async () => { @@ -147,14 +233,18 @@ describe("GuardrailsOverview", () => { }); it("marks the overview busy while the usage request is in flight", async () => { - mockGetGuardrailsUsageOverview.mockReturnValue(new Promise(() => {})); + useGuardrailsUsageOverviewMock.mockReturnValue({ data: undefined, isLoading: true, error: null }); renderOverview(); await waitFor(() => expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument()); }); it("shows a failure message when the usage request rejects", async () => { - mockGetGuardrailsUsageOverview.mockRejectedValue(new Error("network down")); + useGuardrailsUsageOverviewMock.mockReturnValue({ + data: undefined, + isLoading: false, + error: new Error("network down"), + }); renderOverview(); expect(await screen.findByText("Failed to load data. Try again.")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 5bc9eb16cee..67630fef13a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -1,10 +1,14 @@ -import { useQuery } from "@tanstack/react-query"; import type { ColumnDef, OnChangeFn, SortingState } from "@tanstack/react-table"; -import { Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react"; +import { CircleDollarSign, Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react"; import React, { useMemo, useState } from "react"; import { DataTable, DataTableSortHeader } from "@/components/shared/DataTable"; -import { getGuardrailsUsageOverview } from "@/components/networking"; -import { type PerformanceRow } from "@/components/GuardrailsMonitor/mockData"; +import { MoneyCell } from "@/components/shared/table_cells/money_cell"; +import { CellTooltip } from "@/components/shared/table_cells/cell_tooltip"; +import { + type GuardrailUsageOverviewRow, + useGuardrailsUsageOverview, +} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; import { Button } from "@/components/ui/button"; import { PageHeader } from "@/components/shared/PageHeader"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -20,7 +24,7 @@ interface GuardrailsOverviewProps { dateRangeControl?: React.ReactNode; } -type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "falsePositiveRate" | "falseNegativeRate"; +type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "cost"; const providerColors: Record = { Bedrock: "bg-warning/15 text-warning border-warning/20", @@ -30,14 +34,48 @@ const providerColors: Record = { Custom: "bg-muted text-muted-foreground border-border", }; -function computeMetricsFromRows(data: PerformanceRow[]) { - const totalRequests = data.reduce((sum, r) => sum + r.requestsEvaluated, 0); - const totalBlocked = data.reduce((sum, r) => sum + Math.round((r.requestsEvaluated * r.failRate) / 100), 0); - const passRate = totalRequests > 0 ? ((1 - totalBlocked / totalRequests) * 100).toFixed(1) : "0"; - const withLat = data.filter((r) => r.avgLatency != null); - const avgLatency = - withLat.length > 0 ? Math.round(withLat.reduce((sum, r) => sum + (r.avgLatency ?? 0), 0) / withLat.length) : 0; - return { totalRequests, totalBlocked, passRate, avgLatency, count: data.length }; +const EMPTY_METRICS = { + totalRequests: 0, + totalBlocked: 0, + passRate: "0", + avgLatency: 0, + count: 0, + totalCost: null as number | null, + unpriced: null as string | null, +}; + +function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) { + const counters = Object.entries(units); + if (counters.length === 0) return —; + return ( + + {counters.map(([counter, n]) => ( +
  • + {counterLabel(counter)}: {n.toLocaleString()} +
    • + ))} + + } + trigger={{totalUnits(units).toLocaleString()}} + /> + ); +} + +function CostCell({ row }: { row: GuardrailUsageOverviewRow }) { + const unpriced = unpricedSummary(row.untrackedUsageUnits); + return ( + + {unpriced && ( + } + /> + )} + + + ); } export function GuardrailsOverview({ @@ -55,26 +93,22 @@ export function GuardrailsOverview({ data: guardrailsData, isLoading: guardrailsLoading, error: guardrailsError, - } = useQuery({ - queryKey: ["guardrails-usage-overview", startDate, endDate], - queryFn: () => getGuardrailsUsageOverview(accessToken!, startDate, endDate), - enabled: !!accessToken, - }); + } = useGuardrailsUsageOverview({ accessToken, startDate, endDate }); - const activeData: PerformanceRow[] = guardrailsData?.rows ?? []; + const activeData: GuardrailUsageOverviewRow[] = useMemo(() => guardrailsData?.rows ?? [], [guardrailsData]); const metrics = useMemo(() => { - if (guardrailsData) { - return { - totalRequests: guardrailsData.totalRequests ?? 0, - totalBlocked: guardrailsData.totalBlocked ?? 0, - passRate: String(guardrailsData.passRate ?? 0), - avgLatency: activeData.length - ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) - : 0, - count: activeData.length, - }; - } - return computeMetricsFromRows(activeData); + if (!guardrailsData) return EMPTY_METRICS; + return { + totalRequests: guardrailsData.totalRequests, + totalBlocked: guardrailsData.totalBlocked, + passRate: String(guardrailsData.passRate), + avgLatency: activeData.length + ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) + : 0, + count: activeData.length, + totalCost: guardrailsData.totalCost, + unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits), + }; }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; const sorted = useMemo(() => { @@ -88,7 +122,7 @@ export function GuardrailsOverview({ const isLoading = guardrailsLoading; const error = guardrailsError; - const columns: ColumnDef[] = [ + const columns: ColumnDef[] = [ { header: "Guardrail", accessorKey: "name", @@ -166,6 +200,20 @@ export function GuardrailsOverview({ ), }, + { + header: "Usage Units", + accessorKey: "usageUnits", + enableSorting: false, + meta: { numeric: true }, + cell: ({ row }) => , + }, + { + header: ({ column }) => , + accessorKey: "cost", + meta: { numeric: true }, + sortDescFirst: false, + cell: ({ row }) => , + }, { header: "Status", accessorKey: "status", @@ -187,7 +235,7 @@ export function GuardrailsOverview({ }, ]; - const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency"]; + const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"]; const sorting = useMemo(() => [{ id: sortBy, desc: sortDir === "desc" }], [sortBy, sortDir]); const handleSortingChange: OnChangeFn = (updater) => { const nextSorting = typeof updater === "function" ? updater(sorting) : updater; @@ -236,6 +284,13 @@ export function GuardrailsOverview({ metrics.avgLatency > 150 ? "text-destructive" : metrics.avgLatency > 50 ? "text-warning" : "text-success" } /> + } + subtitle={metrics.unpriced ?? undefined} + /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx index fb521c0b8a3..ce8fda0ea69 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx @@ -11,7 +11,23 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ const fetchMock = vi.fn(); -const requestedUrls = () => fetchMock.mock.calls.map(([url]) => String(url)); +const requestUrl = (input: RequestInfo | URL) => (input instanceof Request ? input.url : String(input)); + +const requestedUrls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); + +const emptyOverview = { + rows: [], + chart: [], + totalRequests: 0, + totalBlocked: 0, + passRate: 100, + totalUsageUnits: {}, + totalCost: null, + totalUntrackedUsageUnits: {}, +}; + +const jsonResponse = (body: unknown) => + new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); const renderAs = (userRole: string) => { useAuthorizedMock.mockReturnValue({ accessToken: "sk-test", userId: "u1", userRole }); @@ -25,12 +41,9 @@ describe("Guardrails Monitor page access by role", () => { beforeEach(() => { testQueryClient.clear(); vi.clearAllMocks(); - fetchMock.mockResolvedValue({ - ok: true, - status: 200, - statusText: "OK", - json: async () => ({ rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 }), - }); + fetchMock.mockImplementation(async (input: RequestInfo | URL) => + jsonResponse(requestUrl(input).includes("/guardrails/usage/overview") ? emptyOverview : []), + ); vi.stubGlobal("fetch", fetchMock); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts new file mode 100644 index 00000000000..f0b2709484d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts @@ -0,0 +1,81 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useGuardrailsUsageDetail, useGuardrailsUsageOverview } from "./useGuardrailsUsage"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const lastCall = () => { + const calls = useQueryMock.mock.calls; + return calls[calls.length - 1] as [string, string, unknown, { enabled: boolean }]; +}; + +describe("useGuardrailsUsageOverview", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + }); + + it("queries GET /guardrails/usage/overview with the window as query params", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "2026-09-01", endDate: "2026-09-04" })); + + expect(lastCall().slice(0, 3)).toEqual([ + "get", + "/guardrails/usage/overview", + { params: { query: { start_date: "2026-09-01", end_date: "2026-09-04" } } }, + ]); + expect(lastCall()[3].enabled).toBe(true); + }); + + it("omits blank dates so the proxy applies its default window", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "", endDate: "" })); + + expect(lastCall()[2]).toEqual({ params: { query: { start_date: undefined, end_date: undefined } } }); + }); + + it("stays disabled without an access token", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: null, startDate: "2026-09-01", endDate: "2026-09-04" })); + + expect(lastCall()[3].enabled).toBe(false); + }); +}); + +describe("useGuardrailsUsageDetail", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + }); + + it("queries GET /guardrails/usage/detail/{guardrail_id} with the id as a path param", () => { + renderHook(() => + useGuardrailsUsageDetail({ + accessToken: "sk", + guardrailId: "bedrock-pii-mask", + startDate: "2026-09-01", + endDate: "2026-09-04", + }), + ); + + expect(lastCall().slice(0, 3)).toEqual([ + "get", + "/guardrails/usage/detail/{guardrail_id}", + { + params: { + path: { guardrail_id: "bedrock-pii-mask" }, + query: { start_date: "2026-09-01", end_date: "2026-09-04" }, + }, + }, + ]); + expect(lastCall()[3].enabled).toBe(true); + }); + + it("stays disabled without a guardrail id", () => { + renderHook(() => + useGuardrailsUsageDetail({ accessToken: "sk", guardrailId: "", startDate: "2026-09-01", endDate: "2026-09-04" }), + ); + + expect(lastCall()[3].enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts new file mode 100644 index 00000000000..dc7f58fbc8f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts @@ -0,0 +1,38 @@ +import { $api } from "@/lib/http/api"; +import type { components } from "@/lib/http/schema"; + +export type GuardrailUsageOverview = components["schemas"]["UsageOverviewResponse"]; +export type GuardrailUsageOverviewRow = components["schemas"]["UsageOverviewRow"]; +export type GuardrailUsageDetail = components["schemas"]["UsageDetailResponse"]; + +export interface GuardrailsUsageWindow { + accessToken: string | null; + startDate: string; + endDate: string; +} + +const dateQuery = (startDate: string, endDate: string) => ({ + start_date: startDate || undefined, + end_date: endDate || undefined, +}); + +export const useGuardrailsUsageOverview = ({ accessToken, startDate, endDate }: GuardrailsUsageWindow) => + $api.useQuery( + "get", + "/guardrails/usage/overview", + { params: { query: dateQuery(startDate, endDate) } }, + { enabled: Boolean(accessToken) }, + ); + +export const useGuardrailsUsageDetail = ({ + accessToken, + guardrailId, + startDate, + endDate, +}: GuardrailsUsageWindow & { guardrailId: string }) => + $api.useQuery( + "get", + "/guardrails/usage/detail/{guardrail_id}", + { params: { path: { guardrail_id: guardrailId }, query: dateQuery(startDate, endDate) } }, + { enabled: Boolean(accessToken && guardrailId) }, + ); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index d5d249e4799..c0b5e0a50d1 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -10,7 +10,7 @@ interface MetricCardProps { export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) { return ( -
    +
    {label} {icon && {icon}} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 7d99ebe7c44..2b42f7907f1 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -2,39 +2,6 @@ * Types for Guardrails Monitor dashboard (data from usage API). */ -export interface PerformanceRow { - id: string; - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falseNegativeRate?: number; - status: "healthy" | "warning" | "critical"; - trend: "up" | "down" | "stable"; -} - -export interface GuardrailDetailRecord { - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falsePositiveCount?: number; - falseNegativeRate?: number; - falseNegativeCount?: number; - status: string; - description: string; -} - export interface LogEntry { id: string; timestamp: string; diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts new file mode 100644 index 00000000000..29362cc3701 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits"; + +describe("formatCost", () => { + it("renders a dash when nothing was priced", () => { + expect(formatCost(null)).toBe("—"); + expect(formatCost(undefined)).toBe("—"); + }); + + it("keeps an explicit zero as a real price rather than a dash", () => { + expect(formatCost(0)).toBe("$0.0000"); + }); + + it("shows four decimals for the sub-cent amounts guardrail units cost", () => { + expect(formatCost(0.0003)).toBe("$0.0003"); + expect(formatCost(12.5)).toBe("$12.5000"); + }); + + it("flags amounts below the displayed precision instead of rounding them to zero", () => { + expect(formatCost(0.00001)).toBe("< $0.0001"); + }); +}); + +describe("totalUnits", () => { + it("sums every counter", () => { + expect(totalUnits({ contentPolicyUnits: 3, sensitiveInformationPolicyUnits: 4 })).toBe(7); + }); + + it("is zero for no counters", () => { + expect(totalUnits({})).toBe(0); + }); +}); + +describe("counterLabel", () => { + it("turns a Bedrock counter name into words without the Units suffix", () => { + expect(counterLabel("sensitiveInformationPolicyUnits")).toBe("Sensitive Information Policy"); + expect(counterLabel("contentPolicyUnits")).toBe("Content Policy"); + }); + + it("leaves a name it cannot split alone apart from capitalising it", () => { + expect(counterLabel("units")).toBe("Units"); + }); +}); + +describe("unpricedSummary", () => { + it("is null when every unit was priced", () => { + expect(unpricedSummary({})).toBeNull(); + expect(unpricedSummary({ contentPolicyUnits: 0 })).toBeNull(); + }); + + it("counts unpriced units across counters with a pluralised label", () => { + expect(unpricedSummary({ contentPolicyUnits: 1200, someFutureCounter: 34 })).toBe("1,234 units unpriced"); + expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts new file mode 100644 index 00000000000..f3a5e9d7140 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -0,0 +1,21 @@ +import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils"; + +export type UsageUnits = Readonly>; + +export const formatCost = (cost: number | null | undefined): string => { + if (cost == null) return "—"; + return cost === 0 ? `$${formatNumberWithCommas(0, 4)}` : getSpendString(cost, 4); +}; + +export const totalUnits = (units: UsageUnits): number => Object.values(units).reduce((sum, n) => sum + n, 0); + +export const counterLabel = (counter: string): string => + counter + .replace(/Units$/, "") + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/^./, (c) => c.toUpperCase()); + +export const unpricedSummary = (untracked: UsageUnits): string | null => { + const total = totalUnits(untracked); + return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null; +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index e06d457cfa9..ccf4bd748e5 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -3956,63 +3956,6 @@ export const rejectGuardrailSubmission = async ( }; // Guardrails / Policies usage (dashboard) -export const getGuardrailsUsageOverview = async (accessToken: string, startDate?: string, endDate?: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/guardrails/usage/overview` : `/guardrails/usage/overview`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage overview:", error); - throw error; - } -}; - -export const getGuardrailsUsageDetail = async ( - accessToken: string, - guardrailId: string, - startDate?: string, - endDate?: string, -) => { - try { - let url = proxyBaseUrl - ? `${proxyBaseUrl}/guardrails/usage/detail/${encodeURIComponent(guardrailId)}` - : `/guardrails/usage/detail/${encodeURIComponent(guardrailId)}`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage detail:", error); - throw error; - } -}; - export const getGuardrailsUsageLogs = async ( accessToken: string, options: { diff --git a/ui/litellm-dashboard/src/lib/http/api.ts b/ui/litellm-dashboard/src/lib/http/api.ts index 508a27db78d..904e4e4f3ab 100644 --- a/ui/litellm-dashboard/src/lib/http/api.ts +++ b/ui/litellm-dashboard/src/lib/http/api.ts @@ -43,11 +43,15 @@ const middleware: Middleware = { * * The base URL is injected, not fixed at import: every request is built against * whatever registerBaseUrlGetter supplies at call time (a split-origin proxy or - * worker URL), falling back to the current origin. The middleware injects the - * auth header and maps non-2xx responses to ApiError so query functions can just - * read `.data`. + * worker URL), falling back to the current origin. `fetch` is looked up per + * request for the same reason, so a test that stubs the global sees these calls + * too. The middleware injects the auth header and maps non-2xx responses to + * ApiError so query functions can just read `.data`. */ -export const fetchClient = createFetchClient({ Request: BaseAwareRequest }); +export const fetchClient = createFetchClient({ + Request: BaseAwareRequest, + fetch: (request) => globalThis.fetch(request), +}); fetchClient.use(middleware); /** From 1d375d8ada91eb6f6aeceb8af8bc649a031a5012 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 19:46:14 -0700 Subject: [PATCH 2/7] fix(guardrails): flag unpriced units per team and key, sort unknown cost last The detail endpoint now returns untracked_usage_units_by_team and untracked_usage_units_by_key next to the cost breakdowns, and the By team and By key tables show them in an Unpriced Units column, so a row that pairs its total units with a partial cost says how many units that cost leaves out. The overview comparator no longer treats a missing cost as zero: guardrails with no known cost sort last in both directions instead of mixing in with genuinely free ones. Refs LIT-5652 --- litellm/proxy/_lazy_openapi_snapshot.json | 24 ++++++++++- litellm/proxy/guardrails/usage_endpoints.py | 20 ++++++++-- .../proxy/guardrails/test_usage_endpoints.py | 8 ++++ .../_components/GuardrailDetail.test.tsx | 2 + .../GuardrailUsageBreakdown.test.tsx | 11 ++++- .../_components/GuardrailUsageBreakdown.tsx | 40 ++++++++++++------- .../_components/GuardrailsOverview.test.tsx | 16 ++++++-- .../_components/GuardrailsOverview.tsx | 9 +++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 ++++++ 9 files changed, 114 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c24eea968f8..ddf6a59bea7 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13218,6 +13218,26 @@ "title": "Untracked Usage Units", "type": "object" }, + "untracked_usage_units_by_key": { + "additionalProperties": { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + "title": "Untracked Usage Units By Key", + "type": "object" + }, + "untracked_usage_units_by_team": { + "additionalProperties": { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + "title": "Untracked Usage Units By Team", + "type": "object" + }, "usage_units": { "additionalProperties": { "type": "integer" @@ -13274,7 +13294,9 @@ "cost_by_unit", "cost_by_team", "cost_by_key", - "untracked_usage_units" + "untracked_usage_units", + "untracked_usage_units_by_team", + "untracked_usage_units_by_key" ], "title": "UsageDetailResponse", "type": "object" diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 0390a2b5013..d3bd09f7d27 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -156,6 +156,14 @@ def _counter_name(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: return row.usage_unit +def _team_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: + return row.team_id + + +def _key_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: + return row.api_key + + def _row_untracked_units(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> int: """A row written before the cost column carries NULL cost and is untracked in full.""" return int(row.units) if row.cost is None else int(row.untracked_units) @@ -308,6 +316,8 @@ class UsageDetailResponse(BaseModel): cost_by_team: Mapping[str, float | None] cost_by_key: Mapping[str, float | None] untracked_usage_units: Mapping[str, int] + untracked_usage_units_by_team: Mapping[str, Mapping[str, int]] + untracked_usage_units_by_key: Mapping[str, Mapping[str, int]] class UsageLogEntry(BaseModel): @@ -705,13 +715,15 @@ async def guardrails_usage_detail( time_series=time_series, usage_units=_sum_counter_units(units_rows), usage_units_daily=units_daily, - usage_units_by_team=_by(units_rows, lambda r: r.team_id, _sum_counter_units), - usage_units_by_key=_by(units_rows, lambda r: r.api_key, _sum_counter_units), + usage_units_by_team=_by(units_rows, _team_of, _sum_counter_units), + usage_units_by_key=_by(units_rows, _key_of, _sum_counter_units), cost=_sum_tracked_cost(units_rows), cost_by_unit=_by(units_rows, _counter_name, _sum_tracked_cost), - cost_by_team=_by(units_rows, lambda r: r.team_id, _sum_tracked_cost), - cost_by_key=_by(units_rows, lambda r: r.api_key, _sum_tracked_cost), + cost_by_team=_by(units_rows, _team_of, _sum_tracked_cost), + cost_by_key=_by(units_rows, _key_of, _sum_tracked_cost), untracked_usage_units=_sum_untracked_units(units_rows), + untracked_usage_units_by_team=_by(units_rows, _team_of, _sum_untracked_units), + untracked_usage_units_by_key=_by(units_rows, _key_of, _sum_untracked_units), ) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index ebb2be6edc2..1ff33c76035 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -430,6 +430,13 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys() assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys() assert resp.untracked_usage_units == {"contentPolicyUnits": 50, "topicPolicyUnits": 10} + assert resp.untracked_usage_units_by_team == {"team-a": {"topicPolicyUnits": 10}, "": {"contentPolicyUnits": 50}} + assert resp.untracked_usage_units_by_key == { + "hash-1": {"topicPolicyUnits": 10}, + "hash-2": {"contentPolicyUnits": 50}, + } + assert resp.untracked_usage_units_by_team.keys() == resp.usage_units_by_team.keys() + assert resp.untracked_usage_units_by_key.keys() == resp.usage_units_by_key.keys() @pytest.mark.asyncio @@ -451,6 +458,7 @@ async def test_detail_degrades_units_to_empty_when_units_table_is_missing(): ) assert (resp.cost, resp.cost_by_unit, resp.cost_by_team, resp.cost_by_key) == (None, {}, {}, {}) assert resp.untracked_usage_units == {} + assert (resp.untracked_usage_units_by_team, resp.untracked_usage_units_by_key) == ({}, {}) # ---- logs ------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx index bbcb8138d52..3d00e29245d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx @@ -45,6 +45,8 @@ const detail: GuardrailUsageDetail = { cost_by_team: { "": 0.0004 }, cost_by_key: { "hash-1": 0.0004 }, untracked_usage_units: {}, + untracked_usage_units_by_team: {}, + untracked_usage_units_by_key: {}, }; const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 3b24185f0b6..7929b97b000 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -31,6 +31,8 @@ const detail: GuardrailUsageDetail = { cost_by_team: { "team-a": 0.165, "": 0.015 }, cost_by_key: { "hash-1": 0.18, "hash-2": null }, untracked_usage_units: { someFutureCounter: 7 }, + untracked_usage_units_by_team: { "team-a": {}, "": { someFutureCounter: 7 } }, + untracked_usage_units_by_key: { "hash-1": {}, "hash-2": { someFutureCounter: 7 } }, }; const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); @@ -61,7 +63,7 @@ describe("GuardrailUsageBreakdown", () => { expect(within(future).getByText("—")).toBeInTheDocument(); }); - it("breaks units and cost down by team and by key, naming the rows without one", () => { + it("breaks units and cost down by team and by key, flagging the unpriced share of each row", () => { render(); expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument(); @@ -69,14 +71,17 @@ describe("GuardrailUsageBreakdown", () => { const teamA = rowNamed("team-a"); expect(within(teamA).getByText("1,200")).toBeInTheDocument(); expect(within(teamA).getByText("$0.1650")).toBeInTheDocument(); + expect(within(teamA).getByText("—")).toBeInTheDocument(); + expect(within(teamA).queryByText("7")).not.toBeInTheDocument(); const noTeam = rowNamed("No team"); expect(within(noTeam).getByText("107")).toBeInTheDocument(); expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument(); + expect(within(noTeam).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); const unpricedKey = rowNamed("hash-2"); - expect(within(unpricedKey).getByText("7")).toBeInTheDocument(); expect(within(unpricedKey).getByText("—")).toBeInTheDocument(); + expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); it("orders teams and keys by units, largest first", () => { @@ -104,6 +109,8 @@ describe("GuardrailUsageBreakdown", () => { cost_by_team: {}, cost_by_key: {}, untracked_usage_units: {}, + untracked_usage_units_by_team: {}, + untracked_usage_units_by_key: {}, }} />, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 1eaab86c506..6e8b725aa2e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -19,6 +19,7 @@ interface GroupRow { id: string; units: number; cost: number | null; + unpriced: number; } const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => @@ -32,11 +33,31 @@ const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => const groupRows = ( unitsByGroup: GuardrailUsageDetail["usage_units_by_team"], costByGroup: GuardrailUsageDetail["cost_by_team"], + untrackedByGroup: GuardrailUsageDetail["untracked_usage_units_by_team"], ): GroupRow[] => Object.entries(unitsByGroup) - .map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null })) + .map(([id, units]) => ({ + id, + units: totalUnits(units), + cost: costByGroup[id] ?? null, + unpriced: totalUnits(untrackedByGroup[id] ?? {}), + })) .sort((a, b) => b.units - a.units); +const UnpricedUnitsCell = ({ unpriced }: { unpriced: number }) => + unpriced > 0 ? ( + {unpriced.toLocaleString()} + ) : ( + — + ); + +const unpricedColumn = (): ColumnDef => ({ + header: "Unpriced Units", + accessorKey: "unpriced", + meta: { numeric: true }, + cell: ({ row }) => , +}); + const counterColumns: ColumnDef[] = [ { header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) }, { @@ -51,17 +72,7 @@ const counterColumns: ColumnDef[] = [ meta: { numeric: true }, cell: ({ row }) => , }, - { - header: "Unpriced Units", - accessorKey: "unpriced", - meta: { numeric: true }, - cell: ({ row }) => - row.original.unpriced > 0 ? ( - {row.original.unpriced.toLocaleString()} - ) : ( - — - ), - }, + unpricedColumn(), ]; const groupColumns = (label: string, emptyLabel: string): ColumnDef[] => [ @@ -87,6 +98,7 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] meta: { numeric: true }, cell: ({ row }) => , }, + unpricedColumn(), ]; const teamColumns = groupColumns("Team", "No team"); @@ -139,14 +151,14 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta
    row.id || "no-team"} size="compact" toolbar={() => } /> row.id || "no-key"} size="compact" toolbar={() => } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index 3a56667b156..c52645def70 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -160,14 +160,24 @@ describe("GuardrailsOverview", () => { expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument(); }); - it("sorts by cost when its header is clicked", async () => { + it("sorts by cost when its header is clicked, keeping guardrails with no known cost last either way", async () => { const user = userEvent.setup(); renderOverview(); + const rowNames = () => + screen + .getAllByRole("row") + .slice(1) + .map((r) => r.textContent ?? ""); await user.click(await screen.findByRole("button", { name: /Cost/ })); + await waitFor(() => expect(rowNames()[0]).toContain("Free Bedrock Guardrail")); + expect(rowNames()[1]).toContain("High Failure Guardrail"); + expect(rowNames()[2]).toContain("Low Failure Guardrail"); - await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail")); - expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail"); + await user.click(screen.getByRole("button", { name: /Cost/ })); + await waitFor(() => expect(rowNames()[0]).toContain("High Failure Guardrail")); + expect(rowNames()[1]).toContain("Free Bedrock Guardrail"); + expect(rowNames()[2]).toContain("Low Failure Guardrail"); }); it("renders the page header and the export action", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 67630fef13a..0bbda6015b4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -112,11 +112,12 @@ export function GuardrailsOverview({ }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; const sorted = useMemo(() => { + const mult = sortDir === "desc" ? -1 : 1; return [...activeData].sort((a, b) => { - const mult = sortDir === "desc" ? -1 : 1; - const aVal = a[sortBy] ?? 0; - const bVal = b[sortBy] ?? 0; - return (Number(aVal) - Number(bVal)) * mult; + const aVal = a[sortBy]; + const bVal = b[sortBy]; + if (aVal == null || bVal == null) return Number(aVal == null) - Number(bVal == null); + return (aVal - bVal) * mult; }); }, [activeData, sortBy, sortDir]); const isLoading = guardrailsLoading; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 427e5deb555..c1f65299c52 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38461,6 +38461,18 @@ export interface components { untracked_usage_units: { [key: string]: number; }; + /** Untracked Usage Units By Key */ + untracked_usage_units_by_key: { + [key: string]: { + [key: string]: number; + }; + }; + /** Untracked Usage Units By Team */ + untracked_usage_units_by_team: { + [key: string]: { + [key: string]: number; + }; + }; /** Usage Units */ usage_units: { [key: string]: number; From f73e6838000d7bca1af751fcbcf83fc59d663a9b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 20:26:13 -0700 Subject: [PATCH 3/7] chore(ui): drop the fetch lookup note from the fetchClient docblock --- ui/litellm-dashboard/src/lib/http/api.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/api.ts b/ui/litellm-dashboard/src/lib/http/api.ts index 904e4e4f3ab..9aa6bddf704 100644 --- a/ui/litellm-dashboard/src/lib/http/api.ts +++ b/ui/litellm-dashboard/src/lib/http/api.ts @@ -43,10 +43,9 @@ const middleware: Middleware = { * * The base URL is injected, not fixed at import: every request is built against * whatever registerBaseUrlGetter supplies at call time (a split-origin proxy or - * worker URL), falling back to the current origin. `fetch` is looked up per - * request for the same reason, so a test that stubs the global sees these calls - * too. The middleware injects the auth header and maps non-2xx responses to - * ApiError so query functions can just read `.data`. + * worker URL), falling back to the current origin. The middleware injects the + * auth header and maps non-2xx responses to ApiError so query functions can just + * read `.data`. */ export const fetchClient = createFetchClient({ Request: BaseAwareRequest, From 5a22edb6c3e223f1fecd08eeb966b4ce65b7b3ce Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 12:07:48 -0700 Subject: [PATCH 4/7] feat(ui): explain how guardrail usage and cost are calculated Adds a "How is this calculated?" hover to the Guardrail Cost card on the overview and to the Cost and Usage Units cards on the detail page. The overview hint lists each guardrail's cost and the total; the detail cost hint shows units x per-unit price per counter with unpriced units called out, and the units hint shows the per-counter sum. Also moves the Status column to the front of the overview table. Refs LIT-5652 --- .../GuardrailUsageBreakdown.test.tsx | 32 +++++++++ .../_components/GuardrailUsageBreakdown.tsx | 31 +++++++- .../_components/GuardrailsOverview.test.tsx | 14 ++++ .../_components/GuardrailsOverview.tsx | 67 ++++++++++++----- .../GuardrailsMonitor/MetricCard.tsx | 25 ++++++- .../GuardrailsMonitor/usageUnits.test.ts | 71 ++++++++++++++++++- .../GuardrailsMonitor/usageUnits.ts | 32 +++++++++ 7 files changed, 250 insertions(+), 22 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 7929b97b000..ba90ca8e6ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -1,4 +1,5 @@ import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it } from "vitest"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; @@ -84,6 +85,37 @@ describe("GuardrailUsageBreakdown", () => { expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); + it("explains the cost math per counter on hover", async () => { + const user = userEvent.setup(); + render(); + + await user.hover( + within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), + ); + + expect(await screen.findByText("Content Policy: 1,000 × $0.00015 = $0.1500")).toBeInTheDocument(); + expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); + expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); + expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); + }); + + it("explains the units sum on hover", async () => { + const user = userEvent.setup(); + render(); + + await user.hover( + within(screen.getByRole("group", { name: "Usage Units" })).getByRole("button", { + name: /How is this calculated/, + }), + ); + + expect( + await screen.findByText( + "Content Policy 1,000 + Sensitive Information Policy 300 + Some Future Counter 7 = 1,307", + ), + ).toBeInTheDocument(); + }); + it("orders teams and keys by units, largest first", () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 6e8b725aa2e..01dd8f79ce4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -3,7 +3,14 @@ import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { + counterLabel, + counterMathLine, + formatCost, + totalUnits, + unitsSumLine, + unpricedSummary, +} from "@/components/GuardrailsMonitor/usageUnits"; import { DataTable } from "@/components/shared/DataTable"; import { IdCell } from "@/components/shared/table_cells/id_cell"; import { MoneyCell } from "@/components/shared/table_cells/money_cell"; @@ -104,6 +111,26 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); +const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => ( +
    + {counters.map((row) => ( +
    {counterMathLine(row)}
    + ))} +
    Total: {formatCost(total)}
    +
    Per-unit prices come from the bedrock/guardrails entry in the cost map.
    +
    +); + +const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => ( +
    +
    {unitsSumLine(units)}
    +
    + Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call, + blocked or not. +
    +
    +); + const TableHeading = ({ title }: { title: string }) => (
    {title}
    ); @@ -132,11 +159,13 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={unpriced ?? undefined} + hint={} /> } />
    diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index c52645def70..14e070afc0d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -211,6 +211,20 @@ describe("GuardrailsOverview", () => { expect(card).toHaveTextContent("250 units unpriced"); }); + it("explains the guardrail cost total on hover", async () => { + const user = userEvent.setup(); + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + await user.hover(within(card).getByRole("button", { name: /How is this calculated/ })); + + expect(await screen.findByText("High Failure Guardrail: $0.1500")).toBeInTheDocument(); + expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); + expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); + expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); + expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument(); + }); + it("shows a dash for guardrail cost when nothing in the window was priced", async () => { useGuardrailsUsageOverviewMock.mockReturnValue({ data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 0bbda6015b4..3df7058baba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -63,6 +63,34 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit ); } +function TotalCostMath({ + rows, + total, + unpriced, +}: { + rows: GuardrailUsageOverviewRow[]; + total: number | null; + unpriced: string | null; +}) { + return ( +
    + {rows + .filter((row) => row.cost != null) + .map((row) => ( +
    + {row.name}: {formatCost(row.cost)} +
    + ))} +
    Total: {formatCost(total)}
    +
    + {`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${ + unpriced ? `; ${unpriced} had no known price and are left out` : "" + }. Open a guardrail for its per-policy math.`} +
    +
    + ); +} + function CostCell({ row }: { row: GuardrailUsageOverviewRow }) { const unpriced = unpricedSummary(row.untrackedUsageUnits); return ( @@ -124,6 +152,25 @@ export function GuardrailsOverview({ const error = guardrailsError; const columns: ColumnDef[] = [ + { + header: "Status", + accessorKey: "status", + enableSorting: false, + cell: ({ row }) => ( + + + {row.original.status} + + ), + }, { header: "Guardrail", accessorKey: "name", @@ -215,25 +262,6 @@ export function GuardrailsOverview({ sortDescFirst: false, cell: ({ row }) => , }, - { - header: "Status", - accessorKey: "status", - enableSorting: false, - cell: ({ row }) => ( - - - {row.original.status} - - ), - }, ]; const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"]; @@ -291,6 +319,7 @@ export function GuardrailsOverview({ valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={metrics.unpriced ?? undefined} + hint={} />
    diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index c0b5e0a50d1..1805dc797e4 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -1,4 +1,6 @@ +import { CircleHelp } from "lucide-react"; import React, { type ReactNode } from "react"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; interface MetricCardProps { label: string; @@ -6,9 +8,10 @@ interface MetricCardProps { valueColor?: string; icon?: ReactNode; subtitle?: string; + hint?: ReactNode; } -export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) { +export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle, hint }: MetricCardProps) { return (
    @@ -17,6 +20,26 @@ export function MetricCard({ label, value, valueColor = "text-foreground", icon,
    {value}
    {subtitle &&

    {subtitle}

    } + {hint && ( + + + + + How is this calculated? + + } + /> + + {hint} + + + + )}
    ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 29362cc3701..560010bd852 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from "vitest"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits"; +import { + counterLabel, + counterMathLine, + formatCost, + formatUnitPrice, + totalUnits, + unitPrice, + unitsSumLine, + unpricedSummary, +} from "./usageUnits"; describe("formatCost", () => { it("renders a dash when nothing was priced", () => { @@ -53,3 +62,63 @@ describe("unpricedSummary", () => { expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced"); }); }); + +describe("unitPrice", () => { + it("backs the per-unit price out of the priced share only", () => { + expect(unitPrice({ counter: "contentPolicyUnits", units: 1200, unpriced: 200, cost: 0.15 })).toBeCloseTo( + 0.00015, + 10, + ); + }); + + it("is null when nothing was priced", () => { + expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBeNull(); + expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: 0 })).toBeNull(); + }); +}); + +describe("formatUnitPrice", () => { + it("keeps the significant decimals and drops trailing zeros", () => { + expect(formatUnitPrice(0.0001)).toBe("$0.0001"); + expect(formatUnitPrice(0.00015)).toBe("$0.00015"); + expect(formatUnitPrice(0)).toBe("$0"); + expect(formatUnitPrice(1)).toBe("$1"); + }); +}); + +describe("counterMathLine", () => { + it("shows units × price = cost for a fully priced counter", () => { + expect(counterMathLine({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toBe( + "Content Policy: 1,000 × $0.00015 = $0.1500", + ); + }); + + it("prices only the priced share and calls out the rest", () => { + expect(counterMathLine({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toBe( + "Sensitive Information Policy: 6 × $0.0001 = $0.0006 (2 unpriced left out)", + ); + }); + + it("says so when a counter has no known price at all", () => { + expect(counterMathLine({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBe( + "Some Future Counter: 7 units with no known price, left out", + ); + expect(counterMathLine({ counter: "someFutureCounter", units: 1, unpriced: 1, cost: null })).toBe( + "Some Future Counter: 1 unit with no known price, left out", + ); + }); + + it("shows a free counter as × $0", () => { + expect(counterMathLine({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 })).toBe( + "Word Policy: 2 × $0 = $0.0000", + ); + }); +}); + +describe("unitsSumLine", () => { + it("adds the counters up in order", () => { + expect(unitsSumLine({ contentPolicyUnits: 2, topicPolicyUnits: 2, wordPolicyUnits: 1200 })).toBe( + "Content Policy 2 + Topic Policy 2 + Word Policy 1,200 = 1,204", + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index f3a5e9d7140..05e046aaace 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -19,3 +19,35 @@ export const unpricedSummary = (untracked: UsageUnits): string | null => { const total = totalUnits(untracked); return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null; }; + +export interface CounterMath { + readonly counter: string; + readonly units: number; + readonly unpriced: number; + readonly cost: number | null; +} + +export const pricedUnits = ({ units, unpriced }: Pick): number => + Math.max(units - unpriced, 0); + +export const unitPrice = (row: CounterMath): number | null => { + const priced = pricedUnits(row); + return row.cost != null && priced > 0 ? row.cost / priced : null; +}; + +export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`; + +export const counterMathLine = (row: CounterMath): string => { + const label = counterLabel(row.counter); + const price = unitPrice(row); + if (price == null) { + return `${label}: ${row.units.toLocaleString()} ${row.units === 1 ? "unit" : "units"} with no known price, left out`; + } + const line = `${label}: ${pricedUnits(row).toLocaleString()} × ${formatUnitPrice(price)} = ${formatCost(row.cost)}`; + return row.unpriced > 0 ? `${line} (${row.unpriced.toLocaleString()} unpriced left out)` : line; +}; + +export const unitsSumLine = (units: UsageUnits): string => + `${Object.entries(units) + .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) + .join(" + ")} = ${totalUnits(units).toLocaleString()}`; From e66ba0533fe43a626b8a157fae59f507894aa13b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 12:23:07 -0700 Subject: [PATCH 5/7] fix(ui): keep guardrail cost hints provider neutral and link to a pricing request The hint copy described Bedrock's unit semantics and cost map entry even though any provider's units reach this view, so it now explains the math in provider-neutral terms. When units have no known price, the hint says so and links to a prefilled GitHub feature request (provider and counter names filled in) so the reader can ask for pricing. Per-unit prices below $0.000001 now read "< $0.000001" instead of "$0". Refs LIT-5652 --- .../GuardrailUsageBreakdown.test.tsx | 28 +++++++++++++++++++ .../_components/GuardrailUsageBreakdown.tsx | 15 +++++----- .../_components/GuardrailsOverview.test.tsx | 6 +++- .../_components/GuardrailsOverview.tsx | 26 ++++++++++------- .../GuardrailsMonitor/UnpricedNote.tsx | 21 ++++++++++++++ .../GuardrailsMonitor/usageUnits.test.ts | 22 +++++++++++++++ .../GuardrailsMonitor/usageUnits.ts | 15 +++++++++- 7 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index ba90ca8e6ff..3a0a4c38ecb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -97,6 +97,34 @@ describe("GuardrailUsageBreakdown", () => { expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); + expect(screen.getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + expect(issueLink).toHaveAttribute("target", "_blank"); + const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); + expect(issueUrl.searchParams.get("title")).toBe("[Feature]: add Bedrock guardrail pricing to the cost map"); + expect(issueUrl.searchParams.get("the-feature")).toContain("someFutureCounter"); + }); + + it("does not ask for pricing when every unit was priced", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.hover( + within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), + ); + + expect(await screen.findByText("Total: $0.1500")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); }); it("explains the units sum on hover", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 01dd8f79ce4..e67425adb10 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -3,6 +3,7 @@ import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, counterMathLine, @@ -111,23 +112,21 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); -const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => ( +const CostMath = ({ counters, detail }: { counters: CounterRow[]; detail: GuardrailUsageDetail }) => (
    {counters.map((row) => (
    {counterMathLine(row)}
    ))} -
    Total: {formatCost(total)}
    -
    Per-unit prices come from the bedrock/guardrails entry in the cost map.
    +
    Total: {formatCost(detail.cost)}
    +
    Each counter is its priced units × the per-unit price LiteLLM has for it in the cost map.
    +
    ); const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => (
    {unitsSumLine(units)}
    -
    - Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call, - blocked or not. -
    +
    Units are the billable counters the provider reported for this guardrail, added up over every call.
    ); @@ -159,7 +158,7 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={unpriced ?? undefined} - hint={} + hint={} /> { expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument(); + expect(screen.getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); + expect(issueUrl.searchParams.get("template")).toBe("feature_request.yml"); + expect(issueUrl.searchParams.get("the-feature")).toContain("sensitiveInformationPolicyUnits"); }); it("shows a dash for guardrail cost when nothing in the window was priced", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 3df7058baba..33be85f3c81 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -8,7 +8,14 @@ import { type GuardrailUsageOverviewRow, useGuardrailsUsageOverview, } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; +import { + counterLabel, + formatCost, + totalUnits, + unpricedSummary, + type UsageUnits, +} from "@/components/GuardrailsMonitor/usageUnits"; import { Button } from "@/components/ui/button"; import { PageHeader } from "@/components/shared/PageHeader"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -41,7 +48,7 @@ const EMPTY_METRICS = { avgLatency: 0, count: 0, totalCost: null as number | null, - unpriced: null as string | null, + untracked: {} as UsageUnits, }; function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) { @@ -66,11 +73,11 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit function TotalCostMath({ rows, total, - unpriced, + untracked, }: { rows: GuardrailUsageOverviewRow[]; total: number | null; - unpriced: string | null; + untracked: UsageUnits; }) { return (
    @@ -83,10 +90,9 @@ function TotalCostMath({ ))}
    Total: {formatCost(total)}
    - {`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${ - unpriced ? `; ${unpriced} had no known price and are left out` : "" - }. Open a guardrail for its per-policy math.`} + {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map, added up. Open a guardrail for its per-counter math.`}
    +
    ); } @@ -135,7 +141,7 @@ export function GuardrailsOverview({ : 0, count: activeData.length, totalCost: guardrailsData.totalCost, - unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits), + untracked: guardrailsData.totalUntrackedUsageUnits, }; }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; @@ -318,8 +324,8 @@ export function GuardrailsOverview({ value={formatCost(metrics.totalCost)} valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"} icon={} - subtitle={metrics.unpriced ?? undefined} - hint={} + subtitle={unpricedSummary(metrics.untracked) ?? undefined} + hint={} />
    diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx new file mode 100644 index 00000000000..174124d18aa --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx @@ -0,0 +1,21 @@ +import React from "react"; +import { pricingIssueUrl, totalUnits, type UsageUnits } from "./usageUnits"; + +export function UnpricedNote({ unpriced, provider }: { unpriced: UsageUnits; provider?: string }) { + const total = totalUnits(unpriced); + if (total === 0) return null; + const [noun, verb] = total === 1 ? ["unit", "is"] : ["units", "are"]; + return ( +
    + {`${total.toLocaleString()} ${noun} with no known price ${verb} left out of the cost. `} + + Request pricing on GitHub + +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 560010bd852..9b45eefaf54 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -4,6 +4,7 @@ import { counterMathLine, formatCost, formatUnitPrice, + pricingIssueUrl, totalUnits, unitPrice, unitsSumLine, @@ -84,6 +85,10 @@ describe("formatUnitPrice", () => { expect(formatUnitPrice(0)).toBe("$0"); expect(formatUnitPrice(1)).toBe("$1"); }); + + it("never shows a positive price as free", () => { + expect(formatUnitPrice(0.0000002)).toBe("< $0.000001"); + }); }); describe("counterMathLine", () => { @@ -122,3 +127,20 @@ describe("unitsSumLine", () => { ); }); }); + +describe("pricingIssueUrl", () => { + it("prefills the feature request with the provider and the unpriced counters", () => { + const url = new URL(pricingIssueUrl({ text_records: 5, someFutureCounter: 7 }, "azure/prompt_shield")); + + expect(url.origin + url.pathname).toBe("https://github.com/BerriAI/litellm/issues/new"); + expect(url.searchParams.get("template")).toBe("feature_request.yml"); + expect(url.searchParams.get("title")).toBe("[Feature]: add azure/prompt_shield guardrail pricing to the cost map"); + expect(url.searchParams.get("the-feature")).toContain("text_records, someFutureCounter"); + }); + + it("stays generic when no provider is known", () => { + const url = new URL(pricingIssueUrl({ text_records: 5 })); + + expect(url.searchParams.get("title")).toBe("[Feature]: add guardrail pricing to the cost map"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index 05e046aaace..f914a3e9698 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -35,7 +35,10 @@ export const unitPrice = (row: CounterMath): number | null => { return row.cost != null && priced > 0 ? row.cost / priced : null; }; -export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`; +export const formatUnitPrice = (price: number): string => { + const fixed = price.toFixed(6).replace(/\.?0+$/, ""); + return price > 0 && Number(fixed) === 0 ? "< $0.000001" : `$${fixed}`; +}; export const counterMathLine = (row: CounterMath): string => { const label = counterLabel(row.counter); @@ -51,3 +54,13 @@ export const unitsSumLine = (units: UsageUnits): string => `${Object.entries(units) .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) .join(" + ")} = ${totalUnits(units).toLocaleString()}`; + +export const pricingIssueUrl = (unpriced: UsageUnits, provider?: string): string => { + const subject = provider ? `${provider} guardrail` : "guardrail"; + const params = new URLSearchParams({ + template: "feature_request.yml", + title: `[Feature]: add ${subject} pricing to the cost map`, + "the-feature": `LiteLLM has no price for these ${subject} usage units, so the Guardrails Monitor leaves them out of the cost: ${Object.keys(unpriced).join(", ")}`, + }); + return `https://github.com/BerriAI/litellm/issues/new?${params.toString()}`; +}; From 29b93b57aaa0cef7660da3636df377422a2cd704 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 12:58:18 -0700 Subject: [PATCH 6/7] feat(ui): show the guardrail cost math in a popover table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "How is this calculated?" hover was a plain-text tooltip. It is now a popover (opens on hover or click) with a title, the formula, a table of one row per counter or guardrail (units, × price, = cost, with unpriced units called out under the row) and a total row, so the math reads as a worked sum instead of a sentence. Refs LIT-5652 --- .../GuardrailUsageBreakdown.test.tsx | 55 ++++++++++----- .../_components/GuardrailUsageBreakdown.tsx | 26 +++---- .../_components/GuardrailsOverview.test.tsx | 25 ++++--- .../_components/GuardrailsOverview.tsx | 25 ++++--- .../GuardrailsMonitor/CalcPopover.tsx | 67 +++++++++++++++++++ .../GuardrailsMonitor/MetricCard.tsx | 23 +------ .../GuardrailsMonitor/UnpricedNote.tsx | 4 +- .../GuardrailsMonitor/usageUnits.test.ts | 55 +++++++++------ .../GuardrailsMonitor/usageUnits.ts | 30 ++++++--- 9 files changed, 204 insertions(+), 106 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 3a0a4c38ecb..db7855ab0d5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -85,20 +85,33 @@ describe("GuardrailUsageBreakdown", () => { expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); - it("explains the cost math per counter on hover", async () => { + const cellsOf = (dialog: HTMLElement): string[][] => + within(dialog) + .getAllByRole("row") + .map((row) => + within(row) + .getAllByRole("cell") + .map((cell) => cell.textContent ?? ""), + ); + + it("lays the cost math out per counter as units × price = cost", async () => { const user = userEvent.setup(); render(); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), ); - expect(await screen.findByText("Content Policy: 1,000 × $0.00015 = $0.1500")).toBeInTheDocument(); - expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); - expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); - expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); - expect(screen.getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); - const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000", "× $0.00015", "= $0.1500"], + ["Sensitive Information Policy", "300", "× $0.0001", "= $0.0300"], + ["Some Future Counter", "7", "× —", "= —"], + ["no known price, left out"], + ["Total", "$0.1800"], + ]); + expect(within(dialog).getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = within(dialog).getByRole("link", { name: "Request pricing on GitHub" }); expect(issueLink).toHaveAttribute("target", "_blank"); const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); expect(issueUrl.searchParams.get("title")).toBe("[Feature]: add Bedrock guardrail pricing to the cost map"); @@ -119,29 +132,35 @@ describe("GuardrailUsageBreakdown", () => { />, ); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), ); - expect(await screen.findByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000", "× $0.00015", "= $0.1500"], + ["Total", "$0.1500"], + ]); + expect(within(dialog).queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); }); - it("explains the units sum on hover", async () => { + it("lays the units sum out per counter", async () => { const user = userEvent.setup(); render(); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Usage Units" })).getByRole("button", { name: /How is this calculated/, }), ); - expect( - await screen.findByText( - "Content Policy 1,000 + Sensitive Information Policy 300 + Some Future Counter 7 = 1,307", - ), - ).toBeInTheDocument(); + const dialog = await screen.findByRole("dialog", { name: "How usage units add up" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000"], + ["Sensitive Information Policy", "300"], + ["Some Future Counter", "7"], + ["Total", "1,307"], + ]); }); it("orders teams and keys by units, largest first", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index e67425adb10..27d9ba5162f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -2,14 +2,15 @@ import type { ColumnDef } from "@tanstack/react-table"; import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { CalcPopover, MathTable } from "@/components/GuardrailsMonitor/CalcPopover"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, - counterMathLine, + counterMathRow, formatCost, totalUnits, - unitsSumLine, + unitsMathRows, unpricedSummary, } from "@/components/GuardrailsMonitor/usageUnits"; import { DataTable } from "@/components/shared/DataTable"; @@ -113,21 +114,20 @@ const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); const CostMath = ({ counters, detail }: { counters: CounterRow[]; detail: GuardrailUsageDetail }) => ( -
    - {counters.map((row) => ( -
    {counterMathLine(row)}
    - ))} -
    Total: {formatCost(detail.cost)}
    -
    Each counter is its priced units × the per-unit price LiteLLM has for it in the cost map.
    + + +

    Per-unit prices come from the cost map LiteLLM ships with.

    -
    + ); const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => ( -
    -
    {unitsSumLine(units)}
    -
    Units are the billable counters the provider reported for this guardrail, added up over every call.
    -
    + + +

    + Units are the billable counters the provider reported for this guardrail, added up over every call. +

    +     ); const TableHeading = ({ title }: { title: string }) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index 16361bdec27..959ed8b172e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -211,19 +211,28 @@ describe("GuardrailsOverview", () => { expect(card).toHaveTextContent("250 units unpriced"); }); - it("explains the guardrail cost total on hover", async () => { + it("lays the guardrail cost total out per guardrail", async () => { const user = userEvent.setup(); renderOverview(); const card = await screen.findByRole("group", { name: "Guardrail Cost" }); - await user.hover(within(card).getByRole("button", { name: /How is this calculated/ })); + await user.click(within(card).getByRole("button", { name: /How is this calculated/ })); - expect(await screen.findByText("High Failure Guardrail: $0.1500")).toBeInTheDocument(); - expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); - expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); - expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); - const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + const cells = within(dialog) + .getAllByRole("row") + .map((row) => + within(row) + .getAllByRole("cell") + .map((cell) => cell.textContent ?? ""), + ); + expect(cells).toEqual([ + ["High Failure Guardrail", "$0.1500"], + ["Free Bedrock Guardrail", "$0.0000"], + ["Total", "$0.1500"], + ]); + expect(within(dialog).getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = within(dialog).getByRole("link", { name: "Request pricing on GitHub" }); const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); expect(issueUrl.searchParams.get("template")).toBe("feature_request.yml"); expect(issueUrl.searchParams.get("the-feature")).toContain("sensitiveInformationPolicyUnits"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 33be85f3c81..468e6967d81 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -8,6 +8,7 @@ import { type GuardrailUsageOverviewRow, useGuardrailsUsageOverview, } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { CalcPopover, MathTable } from "@/components/GuardrailsMonitor/CalcPopover"; import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, @@ -80,20 +81,18 @@ function TotalCostMath({ untracked: UsageUnits; }) { return ( -
    - {rows - .filter((row) => row.cost != null) - .map((row) => ( -
    - {row.name}: {formatCost(row.cost)} -
    - ))} -
    Total: {formatCost(total)}
    -
    - {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map, added up. Open a guardrail for its per-counter math.`} -
    + + row.cost != null) + .map((row) => ({ label: row.name, parts: [formatCost(row.cost)], note: null }))} + total={formatCost(total)} + /> +

    + {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map. Open a guardrail for its per-counter math.`} +

    -
    + ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx new file mode 100644 index 00000000000..686992a6fb4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx @@ -0,0 +1,67 @@ +import { CircleHelp } from "lucide-react"; +import React, { type ReactNode } from "react"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import type { MathRow } from "./usageUnits"; + +export function CalcPopover({ title, formula, children }: { title: string; formula: string; children: ReactNode }) { + return ( + + + } + > + + How is this calculated? + + + {title} + {formula} + {children} + + + ); +} + +export function MathTable({ rows, total }: { rows: readonly MathRow[]; total: string }) { + const width = 1 + Math.max(...rows.map((row) => row.parts.length), 1); + return ( + + + {rows.map((row) => ( + + + + {row.parts.map((part, i) => ( + + ))} + + {row.note && ( + + + + )} + + ))} + + + + + + + +
    {row.label} + {part} +
    + {row.note} +
    + Total + {total}
    + ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index 1805dc797e4..008dc279f13 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -1,6 +1,4 @@ -import { CircleHelp } from "lucide-react"; import React, { type ReactNode } from "react"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; interface MetricCardProps { label: string; @@ -20,26 +18,7 @@ export function MetricCard({ label, value, valueColor = "text-foreground", icon,
    {value}
    {subtitle &&

    {subtitle}

    } - {hint && ( - - - - - How is this calculated? - - } - /> - - {hint} - - - - )} + {hint} ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx index 174124d18aa..b43d9d18841 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx @@ -6,7 +6,7 @@ export function UnpricedNote({ unpriced, provider }: { unpriced: UsageUnits; pro if (total === 0) return null; const [noun, verb] = total === 1 ? ["unit", "is"] : ["units", "are"]; return ( -
    +

    {`${total.toLocaleString()} ${noun} with no known price ${verb} left out of the cost. `} Request pricing on GitHub -

    +

    ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 9b45eefaf54..8e2baaaa3c5 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it } from "vitest"; import { counterLabel, - counterMathLine, + counterMathRow, formatCost, formatUnitPrice, pricingIssueUrl, totalUnits, unitPrice, - unitsSumLine, + unitsMathRows, unpricedSummary, } from "./usageUnits"; @@ -91,40 +91,51 @@ describe("formatUnitPrice", () => { }); }); -describe("counterMathLine", () => { +describe("counterMathRow", () => { it("shows units × price = cost for a fully priced counter", () => { - expect(counterMathLine({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toBe( - "Content Policy: 1,000 × $0.00015 = $0.1500", - ); + expect(counterMathRow({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toEqual({ + label: "Content Policy", + parts: ["1,000", "× $0.00015", "= $0.1500"], + note: null, + }); }); it("prices only the priced share and calls out the rest", () => { - expect(counterMathLine({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toBe( - "Sensitive Information Policy: 6 × $0.0001 = $0.0006 (2 unpriced left out)", + expect(counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toEqual( + { + label: "Sensitive Information Policy", + parts: ["6", "× $0.0001", "= $0.0006"], + note: "2 unpriced units left out", + }, ); + expect( + counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 1, cost: 0.0007 }).note, + ).toBe("1 unpriced unit left out"); }); it("says so when a counter has no known price at all", () => { - expect(counterMathLine({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBe( - "Some Future Counter: 7 units with no known price, left out", - ); - expect(counterMathLine({ counter: "someFutureCounter", units: 1, unpriced: 1, cost: null })).toBe( - "Some Future Counter: 1 unit with no known price, left out", - ); + expect(counterMathRow({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toEqual({ + label: "Some Future Counter", + parts: ["7", "× —", "= —"], + note: "no known price, left out", + }); }); it("shows a free counter as × $0", () => { - expect(counterMathLine({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 })).toBe( - "Word Policy: 2 × $0 = $0.0000", - ); + expect(counterMathRow({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 }).parts).toEqual([ + "2", + "× $0", + "= $0.0000", + ]); }); }); -describe("unitsSumLine", () => { - it("adds the counters up in order", () => { - expect(unitsSumLine({ contentPolicyUnits: 2, topicPolicyUnits: 2, wordPolicyUnits: 1200 })).toBe( - "Content Policy 2 + Topic Policy 2 + Word Policy 1,200 = 1,204", - ); +describe("unitsMathRows", () => { + it("lists the counters in order with their counts", () => { + expect(unitsMathRows({ contentPolicyUnits: 2, wordPolicyUnits: 1200 })).toEqual([ + { label: "Content Policy", parts: ["2"], note: null }, + { label: "Word Policy", parts: ["1,200"], note: null }, + ]); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index f914a3e9698..c47442de200 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -40,20 +40,34 @@ export const formatUnitPrice = (price: number): string => { return price > 0 && Number(fixed) === 0 ? "< $0.000001" : `$${fixed}`; }; -export const counterMathLine = (row: CounterMath): string => { +export interface MathRow { + readonly label: string; + readonly parts: readonly string[]; + readonly note: string | null; +} + +export const counterMathRow = (row: CounterMath): MathRow => { const label = counterLabel(row.counter); const price = unitPrice(row); if (price == null) { - return `${label}: ${row.units.toLocaleString()} ${row.units === 1 ? "unit" : "units"} with no known price, left out`; + return { label, parts: [row.units.toLocaleString(), "× —", "= —"], note: "no known price, left out" }; } - const line = `${label}: ${pricedUnits(row).toLocaleString()} × ${formatUnitPrice(price)} = ${formatCost(row.cost)}`; - return row.unpriced > 0 ? `${line} (${row.unpriced.toLocaleString()} unpriced left out)` : line; + return { + label, + parts: [pricedUnits(row).toLocaleString(), `× ${formatUnitPrice(price)}`, `= ${formatCost(row.cost)}`], + note: + row.unpriced > 0 + ? `${row.unpriced.toLocaleString()} unpriced ${row.unpriced === 1 ? "unit" : "units"} left out` + : null, + }; }; -export const unitsSumLine = (units: UsageUnits): string => - `${Object.entries(units) - .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) - .join(" + ")} = ${totalUnits(units).toLocaleString()}`; +export const unitsMathRows = (units: UsageUnits): readonly MathRow[] => + Object.entries(units).map(([counter, n]) => ({ + label: counterLabel(counter), + parts: [n.toLocaleString()], + note: null, + })); export const pricingIssueUrl = (unpriced: UsageUnits, provider?: string): string => { const subject = provider ? `${provider} guardrail` : "guardrail"; From b99d8ac38ee021b4a58b1fcfd4f4fbc1cf0c5b62 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 14:27:08 -0700 Subject: [PATCH 7/7] refactor(ui): keep guardrail usage code under the inline-object-arg lint budget The staging merge pushed local/no-large-inline-object-arg to 567 against a 554 ceiling, and 15 of those hits came from this branch. useGuardrailsUsageDetail now takes the guardrail id positionally with the date window as its second argument, the usageUnits tests build CounterMath rows through a positional helper, and the overview fixture spreads a base row inside the array instead of calling a factory Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- .../_components/GuardrailDetail.test.tsx | 8 ++-- .../_components/GuardrailDetail.tsx | 2 +- .../GuardrailsMonitorView.test.tsx | 3 +- .../_components/GuardrailsOverview.test.tsx | 20 +++++---- .../guardrails/useGuardrailsUsage.test.ts | 5 +-- .../hooks/guardrails/useGuardrailsUsage.ts | 10 ++--- .../GuardrailsMonitor/usageUnits.test.ts | 43 +++++++++---------- 7 files changed, 46 insertions(+), 45 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx index 3d00e29245d..9f2a4c42228 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx @@ -89,9 +89,8 @@ describe("GuardrailDetail", () => { it("should request the detail and the logs for the guardrail and date range", async () => { renderDetail(); - expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith({ + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith("pii-detector", { accessToken: "test-token", - guardrailId: "pii-detector", startDate: "2026-07-01", endDate: "2026-07-24", }); @@ -166,7 +165,10 @@ describe("GuardrailDetail", () => { it("should not request anything without an access token", () => { mockUseGuardrailsUsageDetail.mockReturnValue(loaded(undefined)); renderDetail({ accessToken: null }); - expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith(expect.objectContaining({ accessToken: null })); + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith( + "pii-detector", + expect.objectContaining({ accessToken: null }), + ); expect(mockGetGuardrailsUsageLogs).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx index 1e82f1fee85..81c39258f67 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx @@ -38,7 +38,7 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start data: detailData, isLoading: detailLoading, error: detailError, - } = useGuardrailsUsageDetail({ accessToken, guardrailId, startDate, endDate }); + } = useGuardrailsUsageDetail(guardrailId, { accessToken, startDate, endDate }); const { data: logsData, isLoading: logsLoading } = useQuery({ queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize], queryFn: () => diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index e86b6fa53b6..df106fc38c3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -107,7 +107,8 @@ describe("GuardrailsMonitorView", () => { expect(await screen.findByRole("heading", { name: "PII Guard" })).toBeInTheDocument(); expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith( - expect.objectContaining({ accessToken: "test-token", guardrailId: "gr-pii", startDate: expect.any(String) }), + "gr-pii", + expect.objectContaining({ accessToken: "test-token", startDate: expect.any(String) }), ); expect(screen.queryByRole("heading", { name: /Guardrails Monitor/i })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index 959ed8b172e..ead5fc9f845 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -20,7 +20,7 @@ vi.mock("./EvaluationSettingsModal", () => ({ EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ?
    Evaluation settings modal
    : null), })); -const row = (overrides: Partial): GuardrailUsageOverviewRow => ({ +const baseRow: GuardrailUsageOverviewRow = { id: "guardrail", name: "Guardrail", type: "content_filter", @@ -34,20 +34,21 @@ const row = (overrides: Partial): GuardrailUsageOverv usageUnits: {}, cost: null, untrackedUsageUnits: {}, - ...overrides, -}); +}; const overview: GuardrailUsageOverview = { rows: [ - row({ + { + ...baseRow, id: "guardrail-low", name: "Low Failure Guardrail", requestsEvaluated: 1200, failRate: 2.5, avgLatency: 45, trend: "down", - }), - row({ + }, + { + ...baseRow, id: "guardrail-high", name: "High Failure Guardrail", provider: "Bedrock", @@ -58,8 +59,9 @@ const overview: GuardrailUsageOverview = { usageUnits: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 250 }, cost: 0.15, untrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 }, - }), - row({ + }, + { + ...baseRow, id: "guardrail-free", name: "Free Bedrock Guardrail", provider: "Bedrock", @@ -67,7 +69,7 @@ const overview: GuardrailUsageOverview = { failRate: 0, usageUnits: { contentPolicyUnits: 40 }, cost: 0, - }), + }, ], chart: [], totalRequests: 1510, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts index f0b2709484d..70fc874fb50 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts @@ -50,9 +50,8 @@ describe("useGuardrailsUsageDetail", () => { it("queries GET /guardrails/usage/detail/{guardrail_id} with the id as a path param", () => { renderHook(() => - useGuardrailsUsageDetail({ + useGuardrailsUsageDetail("bedrock-pii-mask", { accessToken: "sk", - guardrailId: "bedrock-pii-mask", startDate: "2026-09-01", endDate: "2026-09-04", }), @@ -73,7 +72,7 @@ describe("useGuardrailsUsageDetail", () => { it("stays disabled without a guardrail id", () => { renderHook(() => - useGuardrailsUsageDetail({ accessToken: "sk", guardrailId: "", startDate: "2026-09-01", endDate: "2026-09-04" }), + useGuardrailsUsageDetail("", { accessToken: "sk", startDate: "2026-09-01", endDate: "2026-09-04" }), ); expect(lastCall()[3].enabled).toBe(false); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts index dc7f58fbc8f..5569bdc8beb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts @@ -24,12 +24,10 @@ export const useGuardrailsUsageOverview = ({ accessToken, startDate, endDate }: { enabled: Boolean(accessToken) }, ); -export const useGuardrailsUsageDetail = ({ - accessToken, - guardrailId, - startDate, - endDate, -}: GuardrailsUsageWindow & { guardrailId: string }) => +export const useGuardrailsUsageDetail = ( + guardrailId: string, + { accessToken, startDate, endDate }: GuardrailsUsageWindow, +) => $api.useQuery( "get", "/guardrails/usage/detail/{guardrail_id}", diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 8e2baaaa3c5..dd5b564b49a 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -9,8 +9,16 @@ import { unitPrice, unitsMathRows, unpricedSummary, + type CounterMath, } from "./usageUnits"; +const counterOf = (counter: string, units: number, unpriced: number, cost: number | null): CounterMath => ({ + counter, + units, + unpriced, + cost, +}); + describe("formatCost", () => { it("renders a dash when nothing was priced", () => { expect(formatCost(null)).toBe("—"); @@ -66,15 +74,12 @@ describe("unpricedSummary", () => { describe("unitPrice", () => { it("backs the per-unit price out of the priced share only", () => { - expect(unitPrice({ counter: "contentPolicyUnits", units: 1200, unpriced: 200, cost: 0.15 })).toBeCloseTo( - 0.00015, - 10, - ); + expect(unitPrice(counterOf("contentPolicyUnits", 1200, 200, 0.15))).toBeCloseTo(0.00015, 10); }); it("is null when nothing was priced", () => { - expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBeNull(); - expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: 0 })).toBeNull(); + expect(unitPrice(counterOf("someFutureCounter", 7, 7, null))).toBeNull(); + expect(unitPrice(counterOf("someFutureCounter", 7, 7, 0))).toBeNull(); }); }); @@ -93,7 +98,7 @@ describe("formatUnitPrice", () => { describe("counterMathRow", () => { it("shows units × price = cost for a fully priced counter", () => { - expect(counterMathRow({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toEqual({ + expect(counterMathRow(counterOf("contentPolicyUnits", 1000, 0, 0.15))).toEqual({ label: "Content Policy", parts: ["1,000", "× $0.00015", "= $0.1500"], note: null, @@ -101,20 +106,18 @@ describe("counterMathRow", () => { }); it("prices only the priced share and calls out the rest", () => { - expect(counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toEqual( - { - label: "Sensitive Information Policy", - parts: ["6", "× $0.0001", "= $0.0006"], - note: "2 unpriced units left out", - }, + expect(counterMathRow(counterOf("sensitiveInformationPolicyUnits", 8, 2, 0.0006))).toEqual({ + label: "Sensitive Information Policy", + parts: ["6", "× $0.0001", "= $0.0006"], + note: "2 unpriced units left out", + }); + expect(counterMathRow(counterOf("sensitiveInformationPolicyUnits", 8, 1, 0.0007)).note).toBe( + "1 unpriced unit left out", ); - expect( - counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 1, cost: 0.0007 }).note, - ).toBe("1 unpriced unit left out"); }); it("says so when a counter has no known price at all", () => { - expect(counterMathRow({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toEqual({ + expect(counterMathRow(counterOf("someFutureCounter", 7, 7, null))).toEqual({ label: "Some Future Counter", parts: ["7", "× —", "= —"], note: "no known price, left out", @@ -122,11 +125,7 @@ describe("counterMathRow", () => { }); it("shows a free counter as × $0", () => { - expect(counterMathRow({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 }).parts).toEqual([ - "2", - "× $0", - "= $0.0000", - ]); + expect(counterMathRow(counterOf("wordPolicyUnits", 2, 0, 0)).parts).toEqual(["2", "× $0", "= $0.0000"]); }); });