From bde3f6ae4605d356ac298ce470396c1b89ec535d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 19:35:43 -0700 Subject: [PATCH] feat(ui): show guardrail usage units and cost on the Guardrails Monitor The overview table gains Usage Units and Cost columns plus a Guardrail Cost card, and the detail page gains a Usage & Cost section that breaks units and cost down by counter, team and key. Units the cost map could not price are called out next to the cost they are left out of. Both pages now read /guardrails/usage/* through $api.useQuery so the rows are typed from schema.d.ts; the hand-written PerformanceRow and the untyped fetch helpers are gone. fetchClient resolves fetch per request so integration tests that stub the global see typed-client calls too. Refs LIT-5652 --- .../_components/GuardrailDetail.test.tsx | 59 +++-- .../_components/GuardrailDetail.tsx | 12 +- .../GuardrailUsageBreakdown.test.tsx | 114 ++++++++++ .../_components/GuardrailUsageBreakdown.tsx | 159 ++++++++++++++ .../GuardrailsMonitorView.test.tsx | 20 +- .../_components/GuardrailsOverview.test.tsx | 204 +++++++++++++----- .../_components/GuardrailsOverview.tsx | 121 ++++++++--- .../page.integration.test.tsx | 27 ++- .../guardrails/useGuardrailsUsage.test.ts | 81 +++++++ .../hooks/guardrails/useGuardrailsUsage.ts | 38 ++++ .../GuardrailsMonitor/MetricCard.tsx | 2 +- .../components/GuardrailsMonitor/mockData.ts | 33 --- .../GuardrailsMonitor/usageUnits.test.ts | 55 +++++ .../GuardrailsMonitor/usageUnits.ts | 21 ++ .../src/components/networking.tsx | 57 ----- ui/litellm-dashboard/src/lib/http/api.ts | 12 +- 16 files changed, 789 insertions(+), 226 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx index c7567aa80bb..bbcb8138d52 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx @@ -2,12 +2,16 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; import { render, screen, waitFor } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailDetail } from "./GuardrailDetail"; -const mockGetGuardrailsUsageDetail = vi.fn(); +const mockUseGuardrailsUsageDetail = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageDetail: (...args: unknown[]) => mockUseGuardrailsUsageDetail(...args), +})); + const mockGetGuardrailsUsageLogs = vi.fn(); vi.mock("@/components/networking", () => ({ - getGuardrailsUsageDetail: (...args: unknown[]) => mockGetGuardrailsUsageDetail(...args), getGuardrailsUsageLogs: (...args: unknown[]) => mockGetGuardrailsUsageLogs(...args), })); @@ -19,7 +23,8 @@ vi.mock("./EvaluationSettingsModal", () => ({ EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ?
: null), })); -const detail = { +const detail: GuardrailUsageDetail = { + guardrail_id: "pii-detector", guardrail_name: "pii-detector", description: "Blocks personally identifiable information", status: "warning", @@ -29,12 +34,25 @@ const detail = { failRate: 20, avgScore: 0.4, avgLatency: 180, + trend: "stable", + time_series: [], + usage_units: { sensitiveInformationPolicyUnits: 4 }, + usage_units_daily: [], + usage_units_by_team: { "": { sensitiveInformationPolicyUnits: 4 } }, + usage_units_by_key: { "hash-1": { sensitiveInformationPolicyUnits: 4 } }, + cost: 0.0004, + cost_by_unit: { sensitiveInformationPolicyUnits: 0.0004 }, + cost_by_team: { "": 0.0004 }, + cost_by_key: { "hash-1": 0.0004 }, + untracked_usage_units: {}, }; +const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null }); + const defaultProps = { guardrailId: "pii-detector", onBack: vi.fn(), - accessToken: "test-token", + accessToken: "test-token" as string | null, startDate: "2026-07-01", endDate: "2026-07-24", }; @@ -49,19 +67,19 @@ function renderDetail(props: Partial = {}) { describe("GuardrailDetail", () => { beforeEach(() => { vi.clearAllMocks(); - mockGetGuardrailsUsageDetail.mockResolvedValue(detail); + mockUseGuardrailsUsageDetail.mockReturnValue(loaded(detail)); mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 }); }); it("should show a busy indicator while the detail request is in flight", () => { - mockGetGuardrailsUsageDetail.mockReturnValue(new Promise(() => {})); + mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: true, error: null }); renderDetail(); expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument(); expect(screen.queryByText("pii-detector")).not.toBeInTheDocument(); }); it("should show an error message and a way back when the detail request fails", async () => { - mockGetGuardrailsUsageDetail.mockRejectedValue(new Error("boom")); + mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: false, error: new Error("boom") }); renderDetail(); expect(await screen.findByText("Failed to load guardrail details.")).toBeInTheDocument(); expect(screen.getByRole("button", { name: /back to overview/i })).toBeInTheDocument(); @@ -69,14 +87,13 @@ describe("GuardrailDetail", () => { it("should request the detail and the logs for the guardrail and date range", async () => { renderDetail(); - await waitFor(() => - expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith( - "test-token", - "pii-detector", - "2026-07-01", - "2026-07-24", - ), - ); + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith({ + accessToken: "test-token", + guardrailId: "pii-detector", + startDate: "2026-07-01", + endDate: "2026-07-24", + }); + await waitFor(() => expect(mockGetGuardrailsUsageLogs).toHaveBeenCalled()); expect(mockGetGuardrailsUsageLogs).toHaveBeenCalledWith( "test-token", expect.objectContaining({ guardrailId: "pii-detector", startDate: "2026-07-01", endDate: "2026-07-24" }), @@ -100,11 +117,18 @@ describe("GuardrailDetail", () => { }); it("should show a placeholder when no latency has been recorded", async () => { - mockGetGuardrailsUsageDetail.mockResolvedValue({ ...detail, avgLatency: null }); + mockUseGuardrailsUsageDetail.mockReturnValue(loaded({ ...detail, avgLatency: null })); renderDetail(); expect(await screen.findByText("No data")).toBeInTheDocument(); }); + it("should show the usage and cost breakdown for the guardrail on the overview tab", async () => { + renderDetail(); + const section = await screen.findByRole("region", { name: "Usage and cost" }); + expect(section).toHaveTextContent("$0.0004"); + expect(section).toHaveTextContent("Sensitive Information Policy"); + }); + it("should call onBack when 'Back to Overview' is clicked", async () => { const user = userEvent.setup(); const onBack = vi.fn(); @@ -138,8 +162,9 @@ describe("GuardrailDetail", () => { }); it("should not request anything without an access token", () => { + mockUseGuardrailsUsageDetail.mockReturnValue(loaded(undefined)); renderDetail({ accessToken: null }); - expect(mockGetGuardrailsUsageDetail).not.toHaveBeenCalled(); + expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith(expect.objectContaining({ accessToken: null })); expect(mockGetGuardrailsUsageLogs).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx index 253477ffeac..1e82f1fee85 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx @@ -1,13 +1,15 @@ import { useQuery } from "@tanstack/react-query"; import { ArrowLeft, Settings, Shield, TriangleAlert } from "lucide-react"; import React, { useMemo, useState } from "react"; -import { getGuardrailsUsageDetail, getGuardrailsUsageLogs } from "@/components/networking"; +import { getGuardrailsUsageLogs } from "@/components/networking"; +import { useGuardrailsUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { StatusBadge, type StatusTone } from "@/components/shared/table_cells/status_badge"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; +import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; import { LogViewer } from "@/components/GuardrailsMonitor/LogViewer"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; import type { LogEntry } from "@/components/GuardrailsMonitor/mockData"; @@ -36,11 +38,7 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start data: detailData, isLoading: detailLoading, error: detailError, - } = useQuery({ - queryKey: ["guardrails-usage-detail", guardrailId, startDate, endDate], - queryFn: () => getGuardrailsUsageDetail(accessToken!, guardrailId, startDate, endDate), - enabled: !!accessToken && !!guardrailId, - }); + } = useGuardrailsUsageDetail({ accessToken, guardrailId, startDate, endDate }); const { data: logsData, isLoading: logsLoading } = useQuery({ queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize], queryFn: () => @@ -194,6 +192,8 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start />
+ {detailData && } + {logViewer("all")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx new file mode 100644 index 00000000000..3b24185f0b6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -0,0 +1,114 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; + +const detail: GuardrailUsageDetail = { + guardrail_id: "bedrock-pii-mask", + guardrail_name: "bedrock-pii-mask", + type: "pii", + provider: "Bedrock", + requestsEvaluated: 5, + failRate: 0, + avgScore: null, + avgLatency: 120, + status: "healthy", + trend: "stable", + description: null, + time_series: [], + usage_units: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300, someFutureCounter: 7 }, + usage_units_daily: [], + usage_units_by_team: { + "team-a": { contentPolicyUnits: 900, sensitiveInformationPolicyUnits: 300 }, + "": { contentPolicyUnits: 100, someFutureCounter: 7 }, + }, + usage_units_by_key: { + "hash-1": { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300 }, + "hash-2": { someFutureCounter: 7 }, + }, + cost: 0.18, + cost_by_unit: { contentPolicyUnits: 0.15, sensitiveInformationPolicyUnits: 0.03, someFutureCounter: null }, + cost_by_team: { "team-a": 0.165, "": 0.015 }, + cost_by_key: { "hash-1": 0.18, "hash-2": null }, + untracked_usage_units: { someFutureCounter: 7 }, +}; + +const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); + +describe("GuardrailUsageBreakdown", () => { + it("totals the units and the cost, and says how many units the cost leaves out", () => { + render(); + + const cost = screen.getByRole("group", { name: "Cost" }); + expect(cost).toHaveTextContent("$0.1800"); + expect(cost).toHaveTextContent("7 units unpriced"); + + const units = screen.getByRole("group", { name: "Usage Units" }); + expect(units).toHaveTextContent("1,307"); + expect(units).toHaveTextContent("3 counters"); + }); + + it("lists each counter with its units, cost and unpriced share", () => { + render(); + + const content = rowNamed("Content Policy"); + expect(within(content).getByText("1,000")).toBeInTheDocument(); + expect(within(content).getByText("$0.1500")).toBeInTheDocument(); + expect(within(content).getByText("—")).toBeInTheDocument(); + + const future = rowNamed("Some Future Counter"); + expect(within(future).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); + expect(within(future).getByText("—")).toBeInTheDocument(); + }); + + it("breaks units and cost down by team and by key, naming the rows without one", () => { + render(); + + expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "By key" })).toBeInTheDocument(); + const teamA = rowNamed("team-a"); + expect(within(teamA).getByText("1,200")).toBeInTheDocument(); + expect(within(teamA).getByText("$0.1650")).toBeInTheDocument(); + + const noTeam = rowNamed("No team"); + expect(within(noTeam).getByText("107")).toBeInTheDocument(); + expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument(); + + const unpricedKey = rowNamed("hash-2"); + expect(within(unpricedKey).getByText("7")).toBeInTheDocument(); + expect(within(unpricedKey).getByText("—")).toBeInTheDocument(); + }); + + it("orders teams and keys by units, largest first", () => { + render(); + + const rows = screen.getAllByRole("row").map((row) => row.textContent ?? ""); + expect(rows.findIndex((text) => text.includes("team-a"))).toBeLessThan( + rows.findIndex((text) => text.includes("No team")), + ); + expect(rows.findIndex((text) => text.includes("hash-1"))).toBeLessThan( + rows.findIndex((text) => text.includes("hash-2")), + ); + }); + + it("says so when the window has no billable units instead of rendering empty tables", () => { + render( + , + ); + + expect(screen.getByText("No billable usage units were recorded in this period.")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx new file mode 100644 index 00000000000..1eaab86c506 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -0,0 +1,159 @@ +import type { ColumnDef } from "@tanstack/react-table"; +import { CircleDollarSign } from "lucide-react"; +import React from "react"; +import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { DataTable } from "@/components/shared/DataTable"; +import { IdCell } from "@/components/shared/table_cells/id_cell"; +import { MoneyCell } from "@/components/shared/table_cells/money_cell"; + +interface CounterRow { + counter: string; + units: number; + cost: number | null; + unpriced: number; +} + +interface GroupRow { + id: string; + units: number; + cost: number | null; +} + +const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => + Object.entries(detail.usage_units).map(([counter, units]) => ({ + counter, + units, + cost: detail.cost_by_unit[counter] ?? null, + unpriced: detail.untracked_usage_units[counter] ?? 0, + })); + +const groupRows = ( + unitsByGroup: GuardrailUsageDetail["usage_units_by_team"], + costByGroup: GuardrailUsageDetail["cost_by_team"], +): GroupRow[] => + Object.entries(unitsByGroup) + .map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null })) + .sort((a, b) => b.units - a.units); + +const counterColumns: ColumnDef[] = [ + { header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) }, + { + header: "Units", + accessorKey: "units", + meta: { numeric: true }, + cell: ({ row }) => row.original.units.toLocaleString(), + }, + { + header: "Cost", + accessorKey: "cost", + meta: { numeric: true }, + cell: ({ row }) => , + }, + { + header: "Unpriced Units", + accessorKey: "unpriced", + meta: { numeric: true }, + cell: ({ row }) => + row.original.unpriced > 0 ? ( + {row.original.unpriced.toLocaleString()} + ) : ( + — + ), + }, +]; + +const groupColumns = (label: string, emptyLabel: string): ColumnDef[] => [ + { + header: label, + accessorKey: "id", + cell: ({ row }) => + row.original.id ? ( + + ) : ( + {emptyLabel} + ), + }, + { + header: "Units", + accessorKey: "units", + meta: { numeric: true }, + cell: ({ row }) => row.original.units.toLocaleString(), + }, + { + header: "Cost", + accessorKey: "cost", + meta: { numeric: true }, + cell: ({ row }) => , + }, +]; + +const teamColumns = groupColumns("Team", "No team"); +const keyColumns = groupColumns("Key", "No key"); + +const TableHeading = ({ title }: { title: string }) => ( +
{title}
+); + +export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDetail }) { + const counters = counterRows(detail); + const unpriced = unpricedSummary(detail.untracked_usage_units); + + return ( +
+
+
Usage & Cost
+

+ Billable units the provider reported for this guardrail and what LiteLLM priced them at +

+
+ + {counters.length === 0 ? ( +

No billable usage units were recorded in this period.

+ ) : ( + <> +
+ } + subtitle={unpriced ?? undefined} + /> + +
+ + row.counter} + size="compact" + toolbar={() => } + /> + +
+ row.id || "no-team"} + size="compact" + toolbar={() => } + /> + row.id || "no-key"} + size="compact" + toolbar={() => } + /> +
+ + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index 9f27daab6b3..83b3a8f5d58 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -2,14 +2,15 @@ import { render, screen, waitFor } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { describe, expect, it, vi } from "vitest"; import GuardrailsMonitorView from "./GuardrailsMonitorView"; -import * as networking from "@/components/networking"; vi.mock("@/components/networking", () => ({ - getGuardrailsUsageOverview: vi.fn(), formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const mockUseGuardrailsUsageOverview = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageOverview: (...args: unknown[]) => mockUseGuardrailsUsageOverview(...args), +})); function wrapper({ children }: { children: React.ReactNode }) { const queryClient = new QueryClient({ @@ -22,23 +23,20 @@ function wrapper({ children }: { children: React.ReactNode }) { describe("GuardrailsMonitorView", () => { it("should render overview and fetch guardrails usage when accessToken is provided", async () => { - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [], - chart: [], - totalRequests: 0, - totalBlocked: 0, - passRate: 100, - }); + mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: true, error: null }); render(, { wrapper }); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); await waitFor(() => { - expect(mockGetGuardrailsUsageOverview).toHaveBeenCalled(); + expect(mockUseGuardrailsUsageOverview).toHaveBeenCalledWith( + expect.objectContaining({ accessToken: "test-token", startDate: expect.any(String) }), + ); }); }); it("should render without crashing when accessToken is null", async () => { + mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: false, error: null }); render(, { wrapper }); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index c62505cc74f..3a56667b156 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -1,12 +1,15 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import * as networking from "@/components/networking"; +import type { + GuardrailUsageOverview, + GuardrailUsageOverviewRow, +} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailsOverview } from "./GuardrailsOverview"; -vi.mock("@/components/networking", () => ({ - getGuardrailsUsageOverview: vi.fn(), +const useGuardrailsUsageOverviewMock = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({ + useGuardrailsUsageOverview: (...args: unknown[]) => useGuardrailsUsageOverviewMock(...args), })); vi.mock("./ScoreChart", () => ({ @@ -17,16 +20,63 @@ vi.mock("./EvaluationSettingsModal", () => ({ EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ?
Evaluation settings modal
: null), })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const row = (overrides: Partial): GuardrailUsageOverviewRow => ({ + id: "guardrail", + name: "Guardrail", + type: "content_filter", + provider: "LiteLLM", + requestsEvaluated: 0, + failRate: 0, + avgScore: null, + avgLatency: null, + status: "healthy", + trend: "stable", + usageUnits: {}, + cost: null, + untrackedUsageUnits: {}, + ...overrides, +}); -function wrapper({ children }: { children: React.ReactNode }) { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, - }, - }); - return {children}; -} +const overview: GuardrailUsageOverview = { + rows: [ + row({ + id: "guardrail-low", + name: "Low Failure Guardrail", + requestsEvaluated: 1200, + failRate: 2.5, + avgLatency: 45, + trend: "down", + }), + row({ + id: "guardrail-high", + name: "High Failure Guardrail", + provider: "Bedrock", + requestsEvaluated: 300, + failRate: 18, + status: "warning", + trend: "up", + usageUnits: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 250 }, + cost: 0.15, + untrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 }, + }), + row({ + id: "guardrail-free", + name: "Free Bedrock Guardrail", + provider: "Bedrock", + requestsEvaluated: 10, + failRate: 0, + usageUnits: { contentPolicyUnits: 40 }, + cost: 0, + }), + ], + chart: [], + totalRequests: 1510, + totalBlocked: 84, + passRate: 94.4, + totalUsageUnits: { contentPolicyUnits: 1040, sensitiveInformationPolicyUnits: 250 }, + totalCost: 0.15, + totalUntrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 }, +}; function renderOverview(onSelectGuardrail = vi.fn()) { return render( @@ -36,41 +86,24 @@ function renderOverview(onSelectGuardrail = vi.fn()) { endDate="2026-08-12" onSelectGuardrail={onSelectGuardrail} />, - { wrapper }, ); } +const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); + describe("GuardrailsOverview", () => { beforeEach(() => { vi.clearAllMocks(); - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [ - { - id: "guardrail-low", - name: "Low Failure Guardrail", - type: "content_filter", - provider: "LiteLLM", - requestsEvaluated: 1200, - failRate: 2.5, - avgLatency: 45, - status: "healthy", - trend: "down", - }, - { - id: "guardrail-high", - name: "High Failure Guardrail", - type: "content_filter", - provider: "Bedrock", - requestsEvaluated: 300, - failRate: 18, - status: "warning", - trend: "up", - }, - ], - chart: [], - totalRequests: 1500, - totalBlocked: 84, - passRate: 94.4, + useGuardrailsUsageOverviewMock.mockReturnValue({ data: overview, isLoading: false, error: null }); + }); + + it("asks for the usage overview of the selected window", () => { + renderOverview(); + + expect(useGuardrailsUsageOverviewMock).toHaveBeenCalledWith({ + accessToken: "test-token", + startDate: "2026-08-01", + endDate: "2026-08-12", }); }); @@ -78,15 +111,7 @@ describe("GuardrailsOverview", () => { const onSelectGuardrail = vi.fn(); const user = userEvent.setup(); - render( - , - { wrapper }, - ); + renderOverview(onSelectGuardrail); expect(await screen.findByRole("columnheader", { name: "Guardrail" })).toBeInTheDocument(); expect(screen.getByRole("columnheader", { name: /Requests/ })).toBeInTheDocument(); @@ -105,6 +130,46 @@ describe("GuardrailsOverview", () => { expect(onSelectGuardrail).toHaveBeenCalledWith("guardrail-low"); }); + it("shows each guardrail's usage units and cost, marking the units cost leaves out", async () => { + renderOverview(); + + expect(await screen.findByRole("columnheader", { name: "Usage Units" })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Cost/ })).toBeInTheDocument(); + + const priced = rowNamed("High Failure Guardrail"); + expect(within(priced).getByText("1,250")).toBeInTheDocument(); + expect(within(priced).getByText("$0.1500")).toBeInTheDocument(); + expect(within(priced).getByLabelText("250 units unpriced")).toBeInTheDocument(); + + const free = rowNamed("Free Bedrock Guardrail"); + expect(within(free).getByText("40")).toBeInTheDocument(); + expect(within(free).getByText("$0.0000")).toBeInTheDocument(); + expect(within(free).queryByLabelText(/unpriced/)).not.toBeInTheDocument(); + + const unmetered = rowNamed("Low Failure Guardrail"); + expect(within(unmetered).getAllByText("—")).toHaveLength(2); + }); + + it("breaks the usage units down per counter on hover", async () => { + const user = userEvent.setup(); + renderOverview(); + + await user.hover(within(rowNamed("High Failure Guardrail")).getByText("1,250")); + + expect(await screen.findByText("Content Policy: 1,000")).toBeInTheDocument(); + expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument(); + }); + + it("sorts by cost when its header is clicked", async () => { + const user = userEvent.setup(); + renderOverview(); + + await user.click(await screen.findByRole("button", { name: /Cost/ })); + + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail")); + expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail"); + }); + it("renders the page header and the export action", async () => { renderOverview(); @@ -117,15 +182,36 @@ describe("GuardrailsOverview", () => { it("renders every summary metric card", async () => { renderOverview(); - expect(await screen.findByText("1,500")).toBeInTheDocument(); + expect(await screen.findByText("1,510")).toBeInTheDocument(); expect(screen.getByText("Total Evaluations")).toBeInTheDocument(); expect(screen.getByText("Blocked Requests")).toBeInTheDocument(); expect(screen.getByText("84")).toBeInTheDocument(); expect(screen.getByText("Pass Rate")).toBeInTheDocument(); expect(screen.getByText("94.4%")).toBeInTheDocument(); - expect(screen.getByText("23ms")).toBeInTheDocument(); + expect(screen.getByText("15ms")).toBeInTheDocument(); expect(screen.getByText("Active Guardrails")).toBeInTheDocument(); - expect(screen.getByText("2")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + }); + + it("totals guardrail cost across the window and says how many units it leaves out", async () => { + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + expect(card).toHaveTextContent("$0.1500"); + expect(card).toHaveTextContent("250 units unpriced"); + }); + + it("shows a dash for guardrail cost when nothing in the window was priced", async () => { + useGuardrailsUsageOverviewMock.mockReturnValue({ + data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} }, + isLoading: false, + error: null, + }); + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + expect(card).toHaveTextContent("—"); + expect(card).not.toHaveTextContent("unpriced"); }); it("renders the table toolbar heading and its description", async () => { @@ -147,14 +233,18 @@ describe("GuardrailsOverview", () => { }); it("marks the overview busy while the usage request is in flight", async () => { - mockGetGuardrailsUsageOverview.mockReturnValue(new Promise(() => {})); + useGuardrailsUsageOverviewMock.mockReturnValue({ data: undefined, isLoading: true, error: null }); renderOverview(); await waitFor(() => expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument()); }); it("shows a failure message when the usage request rejects", async () => { - mockGetGuardrailsUsageOverview.mockRejectedValue(new Error("network down")); + useGuardrailsUsageOverviewMock.mockReturnValue({ + data: undefined, + isLoading: false, + error: new Error("network down"), + }); renderOverview(); expect(await screen.findByText("Failed to load data. Try again.")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 5bc9eb16cee..67630fef13a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -1,10 +1,14 @@ -import { useQuery } from "@tanstack/react-query"; import type { ColumnDef, OnChangeFn, SortingState } from "@tanstack/react-table"; -import { Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react"; +import { CircleDollarSign, Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react"; import React, { useMemo, useState } from "react"; import { DataTable, DataTableSortHeader } from "@/components/shared/DataTable"; -import { getGuardrailsUsageOverview } from "@/components/networking"; -import { type PerformanceRow } from "@/components/GuardrailsMonitor/mockData"; +import { MoneyCell } from "@/components/shared/table_cells/money_cell"; +import { CellTooltip } from "@/components/shared/table_cells/cell_tooltip"; +import { + type GuardrailUsageOverviewRow, + useGuardrailsUsageOverview, +} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; import { Button } from "@/components/ui/button"; import { PageHeader } from "@/components/shared/PageHeader"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -20,7 +24,7 @@ interface GuardrailsOverviewProps { dateRangeControl?: React.ReactNode; } -type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "falsePositiveRate" | "falseNegativeRate"; +type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "cost"; const providerColors: Record = { Bedrock: "bg-warning/15 text-warning border-warning/20", @@ -30,14 +34,48 @@ const providerColors: Record = { Custom: "bg-muted text-muted-foreground border-border", }; -function computeMetricsFromRows(data: PerformanceRow[]) { - const totalRequests = data.reduce((sum, r) => sum + r.requestsEvaluated, 0); - const totalBlocked = data.reduce((sum, r) => sum + Math.round((r.requestsEvaluated * r.failRate) / 100), 0); - const passRate = totalRequests > 0 ? ((1 - totalBlocked / totalRequests) * 100).toFixed(1) : "0"; - const withLat = data.filter((r) => r.avgLatency != null); - const avgLatency = - withLat.length > 0 ? Math.round(withLat.reduce((sum, r) => sum + (r.avgLatency ?? 0), 0) / withLat.length) : 0; - return { totalRequests, totalBlocked, passRate, avgLatency, count: data.length }; +const EMPTY_METRICS = { + totalRequests: 0, + totalBlocked: 0, + passRate: "0", + avgLatency: 0, + count: 0, + totalCost: null as number | null, + unpriced: null as string | null, +}; + +function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) { + const counters = Object.entries(units); + if (counters.length === 0) return —; + return ( + + {counters.map(([counter, n]) => ( +
  • + {counterLabel(counter)}: {n.toLocaleString()} +
    • + ))} + + } + trigger={{totalUnits(units).toLocaleString()}} + /> + ); +} + +function CostCell({ row }: { row: GuardrailUsageOverviewRow }) { + const unpriced = unpricedSummary(row.untrackedUsageUnits); + return ( + + {unpriced && ( + } + /> + )} + + + ); } export function GuardrailsOverview({ @@ -55,26 +93,22 @@ export function GuardrailsOverview({ data: guardrailsData, isLoading: guardrailsLoading, error: guardrailsError, - } = useQuery({ - queryKey: ["guardrails-usage-overview", startDate, endDate], - queryFn: () => getGuardrailsUsageOverview(accessToken!, startDate, endDate), - enabled: !!accessToken, - }); + } = useGuardrailsUsageOverview({ accessToken, startDate, endDate }); - const activeData: PerformanceRow[] = guardrailsData?.rows ?? []; + const activeData: GuardrailUsageOverviewRow[] = useMemo(() => guardrailsData?.rows ?? [], [guardrailsData]); const metrics = useMemo(() => { - if (guardrailsData) { - return { - totalRequests: guardrailsData.totalRequests ?? 0, - totalBlocked: guardrailsData.totalBlocked ?? 0, - passRate: String(guardrailsData.passRate ?? 0), - avgLatency: activeData.length - ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) - : 0, - count: activeData.length, - }; - } - return computeMetricsFromRows(activeData); + if (!guardrailsData) return EMPTY_METRICS; + return { + totalRequests: guardrailsData.totalRequests, + totalBlocked: guardrailsData.totalBlocked, + passRate: String(guardrailsData.passRate), + avgLatency: activeData.length + ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) + : 0, + count: activeData.length, + totalCost: guardrailsData.totalCost, + unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits), + }; }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; const sorted = useMemo(() => { @@ -88,7 +122,7 @@ export function GuardrailsOverview({ const isLoading = guardrailsLoading; const error = guardrailsError; - const columns: ColumnDef[] = [ + const columns: ColumnDef[] = [ { header: "Guardrail", accessorKey: "name", @@ -166,6 +200,20 @@ export function GuardrailsOverview({ ), }, + { + header: "Usage Units", + accessorKey: "usageUnits", + enableSorting: false, + meta: { numeric: true }, + cell: ({ row }) => , + }, + { + header: ({ column }) => , + accessorKey: "cost", + meta: { numeric: true }, + sortDescFirst: false, + cell: ({ row }) => , + }, { header: "Status", accessorKey: "status", @@ -187,7 +235,7 @@ export function GuardrailsOverview({ }, ]; - const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency"]; + const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"]; const sorting = useMemo(() => [{ id: sortBy, desc: sortDir === "desc" }], [sortBy, sortDir]); const handleSortingChange: OnChangeFn = (updater) => { const nextSorting = typeof updater === "function" ? updater(sorting) : updater; @@ -236,6 +284,13 @@ export function GuardrailsOverview({ metrics.avgLatency > 150 ? "text-destructive" : metrics.avgLatency > 50 ? "text-warning" : "text-success" } /> + } + subtitle={metrics.unpriced ?? undefined} + /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx index fb521c0b8a3..ce8fda0ea69 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.integration.test.tsx @@ -11,7 +11,23 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ const fetchMock = vi.fn(); -const requestedUrls = () => fetchMock.mock.calls.map(([url]) => String(url)); +const requestUrl = (input: RequestInfo | URL) => (input instanceof Request ? input.url : String(input)); + +const requestedUrls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input)); + +const emptyOverview = { + rows: [], + chart: [], + totalRequests: 0, + totalBlocked: 0, + passRate: 100, + totalUsageUnits: {}, + totalCost: null, + totalUntrackedUsageUnits: {}, +}; + +const jsonResponse = (body: unknown) => + new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } }); const renderAs = (userRole: string) => { useAuthorizedMock.mockReturnValue({ accessToken: "sk-test", userId: "u1", userRole }); @@ -25,12 +41,9 @@ describe("Guardrails Monitor page access by role", () => { beforeEach(() => { testQueryClient.clear(); vi.clearAllMocks(); - fetchMock.mockResolvedValue({ - ok: true, - status: 200, - statusText: "OK", - json: async () => ({ rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 }), - }); + fetchMock.mockImplementation(async (input: RequestInfo | URL) => + jsonResponse(requestUrl(input).includes("/guardrails/usage/overview") ? emptyOverview : []), + ); vi.stubGlobal("fetch", fetchMock); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts new file mode 100644 index 00000000000..f0b2709484d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.test.ts @@ -0,0 +1,81 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useGuardrailsUsageDetail, useGuardrailsUsageOverview } from "./useGuardrailsUsage"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const lastCall = () => { + const calls = useQueryMock.mock.calls; + return calls[calls.length - 1] as [string, string, unknown, { enabled: boolean }]; +}; + +describe("useGuardrailsUsageOverview", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + }); + + it("queries GET /guardrails/usage/overview with the window as query params", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "2026-09-01", endDate: "2026-09-04" })); + + expect(lastCall().slice(0, 3)).toEqual([ + "get", + "/guardrails/usage/overview", + { params: { query: { start_date: "2026-09-01", end_date: "2026-09-04" } } }, + ]); + expect(lastCall()[3].enabled).toBe(true); + }); + + it("omits blank dates so the proxy applies its default window", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "", endDate: "" })); + + expect(lastCall()[2]).toEqual({ params: { query: { start_date: undefined, end_date: undefined } } }); + }); + + it("stays disabled without an access token", () => { + renderHook(() => useGuardrailsUsageOverview({ accessToken: null, startDate: "2026-09-01", endDate: "2026-09-04" })); + + expect(lastCall()[3].enabled).toBe(false); + }); +}); + +describe("useGuardrailsUsageDetail", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + }); + + it("queries GET /guardrails/usage/detail/{guardrail_id} with the id as a path param", () => { + renderHook(() => + useGuardrailsUsageDetail({ + accessToken: "sk", + guardrailId: "bedrock-pii-mask", + startDate: "2026-09-01", + endDate: "2026-09-04", + }), + ); + + expect(lastCall().slice(0, 3)).toEqual([ + "get", + "/guardrails/usage/detail/{guardrail_id}", + { + params: { + path: { guardrail_id: "bedrock-pii-mask" }, + query: { start_date: "2026-09-01", end_date: "2026-09-04" }, + }, + }, + ]); + expect(lastCall()[3].enabled).toBe(true); + }); + + it("stays disabled without a guardrail id", () => { + renderHook(() => + useGuardrailsUsageDetail({ accessToken: "sk", guardrailId: "", startDate: "2026-09-01", endDate: "2026-09-04" }), + ); + + expect(lastCall()[3].enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts new file mode 100644 index 00000000000..dc7f58fbc8f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrails/useGuardrailsUsage.ts @@ -0,0 +1,38 @@ +import { $api } from "@/lib/http/api"; +import type { components } from "@/lib/http/schema"; + +export type GuardrailUsageOverview = components["schemas"]["UsageOverviewResponse"]; +export type GuardrailUsageOverviewRow = components["schemas"]["UsageOverviewRow"]; +export type GuardrailUsageDetail = components["schemas"]["UsageDetailResponse"]; + +export interface GuardrailsUsageWindow { + accessToken: string | null; + startDate: string; + endDate: string; +} + +const dateQuery = (startDate: string, endDate: string) => ({ + start_date: startDate || undefined, + end_date: endDate || undefined, +}); + +export const useGuardrailsUsageOverview = ({ accessToken, startDate, endDate }: GuardrailsUsageWindow) => + $api.useQuery( + "get", + "/guardrails/usage/overview", + { params: { query: dateQuery(startDate, endDate) } }, + { enabled: Boolean(accessToken) }, + ); + +export const useGuardrailsUsageDetail = ({ + accessToken, + guardrailId, + startDate, + endDate, +}: GuardrailsUsageWindow & { guardrailId: string }) => + $api.useQuery( + "get", + "/guardrails/usage/detail/{guardrail_id}", + { params: { path: { guardrail_id: guardrailId }, query: dateQuery(startDate, endDate) } }, + { enabled: Boolean(accessToken && guardrailId) }, + ); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index d5d249e4799..c0b5e0a50d1 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -10,7 +10,7 @@ interface MetricCardProps { export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) { return ( -
    +
    {label} {icon && {icon}} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 7d99ebe7c44..2b42f7907f1 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -2,39 +2,6 @@ * Types for Guardrails Monitor dashboard (data from usage API). */ -export interface PerformanceRow { - id: string; - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falseNegativeRate?: number; - status: "healthy" | "warning" | "critical"; - trend: "up" | "down" | "stable"; -} - -export interface GuardrailDetailRecord { - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falsePositiveCount?: number; - falseNegativeRate?: number; - falseNegativeCount?: number; - status: string; - description: string; -} - export interface LogEntry { id: string; timestamp: string; diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts new file mode 100644 index 00000000000..29362cc3701 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits"; + +describe("formatCost", () => { + it("renders a dash when nothing was priced", () => { + expect(formatCost(null)).toBe("—"); + expect(formatCost(undefined)).toBe("—"); + }); + + it("keeps an explicit zero as a real price rather than a dash", () => { + expect(formatCost(0)).toBe("$0.0000"); + }); + + it("shows four decimals for the sub-cent amounts guardrail units cost", () => { + expect(formatCost(0.0003)).toBe("$0.0003"); + expect(formatCost(12.5)).toBe("$12.5000"); + }); + + it("flags amounts below the displayed precision instead of rounding them to zero", () => { + expect(formatCost(0.00001)).toBe("< $0.0001"); + }); +}); + +describe("totalUnits", () => { + it("sums every counter", () => { + expect(totalUnits({ contentPolicyUnits: 3, sensitiveInformationPolicyUnits: 4 })).toBe(7); + }); + + it("is zero for no counters", () => { + expect(totalUnits({})).toBe(0); + }); +}); + +describe("counterLabel", () => { + it("turns a Bedrock counter name into words without the Units suffix", () => { + expect(counterLabel("sensitiveInformationPolicyUnits")).toBe("Sensitive Information Policy"); + expect(counterLabel("contentPolicyUnits")).toBe("Content Policy"); + }); + + it("leaves a name it cannot split alone apart from capitalising it", () => { + expect(counterLabel("units")).toBe("Units"); + }); +}); + +describe("unpricedSummary", () => { + it("is null when every unit was priced", () => { + expect(unpricedSummary({})).toBeNull(); + expect(unpricedSummary({ contentPolicyUnits: 0 })).toBeNull(); + }); + + it("counts unpriced units across counters with a pluralised label", () => { + expect(unpricedSummary({ contentPolicyUnits: 1200, someFutureCounter: 34 })).toBe("1,234 units unpriced"); + expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts new file mode 100644 index 00000000000..f3a5e9d7140 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -0,0 +1,21 @@ +import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils"; + +export type UsageUnits = Readonly>; + +export const formatCost = (cost: number | null | undefined): string => { + if (cost == null) return "—"; + return cost === 0 ? `$${formatNumberWithCommas(0, 4)}` : getSpendString(cost, 4); +}; + +export const totalUnits = (units: UsageUnits): number => Object.values(units).reduce((sum, n) => sum + n, 0); + +export const counterLabel = (counter: string): string => + counter + .replace(/Units$/, "") + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/^./, (c) => c.toUpperCase()); + +export const unpricedSummary = (untracked: UsageUnits): string | null => { + const total = totalUnits(untracked); + return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null; +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index e06d457cfa9..ccf4bd748e5 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -3956,63 +3956,6 @@ export const rejectGuardrailSubmission = async ( }; // Guardrails / Policies usage (dashboard) -export const getGuardrailsUsageOverview = async (accessToken: string, startDate?: string, endDate?: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/guardrails/usage/overview` : `/guardrails/usage/overview`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage overview:", error); - throw error; - } -}; - -export const getGuardrailsUsageDetail = async ( - accessToken: string, - guardrailId: string, - startDate?: string, - endDate?: string, -) => { - try { - let url = proxyBaseUrl - ? `${proxyBaseUrl}/guardrails/usage/detail/${encodeURIComponent(guardrailId)}` - : `/guardrails/usage/detail/${encodeURIComponent(guardrailId)}`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage detail:", error); - throw error; - } -}; - export const getGuardrailsUsageLogs = async ( accessToken: string, options: { diff --git a/ui/litellm-dashboard/src/lib/http/api.ts b/ui/litellm-dashboard/src/lib/http/api.ts index 508a27db78d..904e4e4f3ab 100644 --- a/ui/litellm-dashboard/src/lib/http/api.ts +++ b/ui/litellm-dashboard/src/lib/http/api.ts @@ -43,11 +43,15 @@ const middleware: Middleware = { * * The base URL is injected, not fixed at import: every request is built against * whatever registerBaseUrlGetter supplies at call time (a split-origin proxy or - * worker URL), falling back to the current origin. The middleware injects the - * auth header and maps non-2xx responses to ApiError so query functions can just - * read `.data`. + * worker URL), falling back to the current origin. `fetch` is looked up per + * request for the same reason, so a test that stubs the global sees these calls + * too. The middleware injects the auth header and maps non-2xx responses to + * ApiError so query functions can just read `.data`. */ -export const fetchClient = createFetchClient({ Request: BaseAwareRequest }); +export const fetchClient = createFetchClient({ + Request: BaseAwareRequest, + fetch: (request) => globalThis.fetch(request), +}); fetchClient.use(middleware); /**