mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
feat(ui): show guardrail usage units and cost on the Guardrails Monitor
The overview table gains Usage Units and Cost columns plus a Guardrail Cost card, and the detail page gains a Usage & Cost section that breaks units and cost down by counter, team and key. Units the cost map could not price are called out next to the cost they are left out of. Both pages now read /guardrails/usage/* through $api.useQuery so the rows are typed from schema.d.ts; the hand-written PerformanceRow and the untyped fetch helpers are gone. fetchClient resolves fetch per request so integration tests that stub the global see typed-client calls too. Refs LIT-5652
This commit is contained in:
ryan-crabbe-berri
parent
2151dcbd73
commit
bde3f6ae46
16 changed files with 789 additions and 226 deletions
|
|
@ -2,12 +2,16 @@ import { describe, it, expect, vi, beforeEach } from "vitest";
|
|||
import userEvent from "@testing-library/user-event";
|
||||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { GuardrailDetail } from "./GuardrailDetail";
|
||||
|
||||
const mockGetGuardrailsUsageDetail = vi.fn();
|
||||
const mockUseGuardrailsUsageDetail = vi.fn();
|
||||
vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({
|
||||
useGuardrailsUsageDetail: (...args: unknown[]) => mockUseGuardrailsUsageDetail(...args),
|
||||
}));
|
||||
|
||||
const mockGetGuardrailsUsageLogs = vi.fn();
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGuardrailsUsageDetail: (...args: unknown[]) => mockGetGuardrailsUsageDetail(...args),
|
||||
getGuardrailsUsageLogs: (...args: unknown[]) => mockGetGuardrailsUsageLogs(...args),
|
||||
}));
|
||||
|
||||
|
|
@ -19,7 +23,8 @@ vi.mock("./EvaluationSettingsModal", () => ({
|
|||
EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ? <div data-testid="evaluation-modal" /> : null),
|
||||
}));
|
||||
|
||||
const detail = {
|
||||
const detail: GuardrailUsageDetail = {
|
||||
guardrail_id: "pii-detector",
|
||||
guardrail_name: "pii-detector",
|
||||
description: "Blocks personally identifiable information",
|
||||
status: "warning",
|
||||
|
|
@ -29,12 +34,25 @@ const detail = {
|
|||
failRate: 20,
|
||||
avgScore: 0.4,
|
||||
avgLatency: 180,
|
||||
trend: "stable",
|
||||
time_series: [],
|
||||
usage_units: { sensitiveInformationPolicyUnits: 4 },
|
||||
usage_units_daily: [],
|
||||
usage_units_by_team: { "": { sensitiveInformationPolicyUnits: 4 } },
|
||||
usage_units_by_key: { "hash-1": { sensitiveInformationPolicyUnits: 4 } },
|
||||
cost: 0.0004,
|
||||
cost_by_unit: { sensitiveInformationPolicyUnits: 0.0004 },
|
||||
cost_by_team: { "": 0.0004 },
|
||||
cost_by_key: { "hash-1": 0.0004 },
|
||||
untracked_usage_units: {},
|
||||
};
|
||||
|
||||
const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null });
|
||||
|
||||
const defaultProps = {
|
||||
guardrailId: "pii-detector",
|
||||
onBack: vi.fn(),
|
||||
accessToken: "test-token",
|
||||
accessToken: "test-token" as string | null,
|
||||
startDate: "2026-07-01",
|
||||
endDate: "2026-07-24",
|
||||
};
|
||||
|
|
@ -49,19 +67,19 @@ function renderDetail(props: Partial<typeof defaultProps> = {}) {
|
|||
describe("GuardrailDetail", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockGetGuardrailsUsageDetail.mockResolvedValue(detail);
|
||||
mockUseGuardrailsUsageDetail.mockReturnValue(loaded(detail));
|
||||
mockGetGuardrailsUsageLogs.mockResolvedValue({ logs: [], total: 0 });
|
||||
});
|
||||
|
||||
it("should show a busy indicator while the detail request is in flight", () => {
|
||||
mockGetGuardrailsUsageDetail.mockReturnValue(new Promise(() => {}));
|
||||
mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: true, error: null });
|
||||
renderDetail();
|
||||
expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument();
|
||||
expect(screen.queryByText("pii-detector")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show an error message and a way back when the detail request fails", async () => {
|
||||
mockGetGuardrailsUsageDetail.mockRejectedValue(new Error("boom"));
|
||||
mockUseGuardrailsUsageDetail.mockReturnValue({ data: undefined, isLoading: false, error: new Error("boom") });
|
||||
renderDetail();
|
||||
expect(await screen.findByText("Failed to load guardrail details.")).toBeInTheDocument();
|
||||
expect(screen.getByRole("button", { name: /back to overview/i })).toBeInTheDocument();
|
||||
|
|
@ -69,14 +87,13 @@ describe("GuardrailDetail", () => {
|
|||
|
||||
it("should request the detail and the logs for the guardrail and date range", async () => {
|
||||
renderDetail();
|
||||
await waitFor(() =>
|
||||
expect(mockGetGuardrailsUsageDetail).toHaveBeenCalledWith(
|
||||
"test-token",
|
||||
"pii-detector",
|
||||
"2026-07-01",
|
||||
"2026-07-24",
|
||||
),
|
||||
);
|
||||
expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith({
|
||||
accessToken: "test-token",
|
||||
guardrailId: "pii-detector",
|
||||
startDate: "2026-07-01",
|
||||
endDate: "2026-07-24",
|
||||
});
|
||||
await waitFor(() => expect(mockGetGuardrailsUsageLogs).toHaveBeenCalled());
|
||||
expect(mockGetGuardrailsUsageLogs).toHaveBeenCalledWith(
|
||||
"test-token",
|
||||
expect.objectContaining({ guardrailId: "pii-detector", startDate: "2026-07-01", endDate: "2026-07-24" }),
|
||||
|
|
@ -100,11 +117,18 @@ describe("GuardrailDetail", () => {
|
|||
});
|
||||
|
||||
it("should show a placeholder when no latency has been recorded", async () => {
|
||||
mockGetGuardrailsUsageDetail.mockResolvedValue({ ...detail, avgLatency: null });
|
||||
mockUseGuardrailsUsageDetail.mockReturnValue(loaded({ ...detail, avgLatency: null }));
|
||||
renderDetail();
|
||||
expect(await screen.findByText("No data")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the usage and cost breakdown for the guardrail on the overview tab", async () => {
|
||||
renderDetail();
|
||||
const section = await screen.findByRole("region", { name: "Usage and cost" });
|
||||
expect(section).toHaveTextContent("$0.0004");
|
||||
expect(section).toHaveTextContent("Sensitive Information Policy");
|
||||
});
|
||||
|
||||
it("should call onBack when 'Back to Overview' is clicked", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onBack = vi.fn();
|
||||
|
|
@ -138,8 +162,9 @@ describe("GuardrailDetail", () => {
|
|||
});
|
||||
|
||||
it("should not request anything without an access token", () => {
|
||||
mockUseGuardrailsUsageDetail.mockReturnValue(loaded(undefined));
|
||||
renderDetail({ accessToken: null });
|
||||
expect(mockGetGuardrailsUsageDetail).not.toHaveBeenCalled();
|
||||
expect(mockUseGuardrailsUsageDetail).toHaveBeenCalledWith(expect.objectContaining({ accessToken: null }));
|
||||
expect(mockGetGuardrailsUsageLogs).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
import { useQuery } from "@tanstack/react-query";
|
||||
import { ArrowLeft, Settings, Shield, TriangleAlert } from "lucide-react";
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { getGuardrailsUsageDetail, getGuardrailsUsageLogs } from "@/components/networking";
|
||||
import { getGuardrailsUsageLogs } from "@/components/networking";
|
||||
import { useGuardrailsUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { StatusBadge, type StatusTone } from "@/components/shared/table_cells/status_badge";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
import { EvaluationSettingsModal } from "./EvaluationSettingsModal";
|
||||
import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown";
|
||||
import { LogViewer } from "@/components/GuardrailsMonitor/LogViewer";
|
||||
import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard";
|
||||
import type { LogEntry } from "@/components/GuardrailsMonitor/mockData";
|
||||
|
|
@ -36,11 +38,7 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start
|
|||
data: detailData,
|
||||
isLoading: detailLoading,
|
||||
error: detailError,
|
||||
} = useQuery({
|
||||
queryKey: ["guardrails-usage-detail", guardrailId, startDate, endDate],
|
||||
queryFn: () => getGuardrailsUsageDetail(accessToken!, guardrailId, startDate, endDate),
|
||||
enabled: !!accessToken && !!guardrailId,
|
||||
});
|
||||
} = useGuardrailsUsageDetail({ accessToken, guardrailId, startDate, endDate });
|
||||
const { data: logsData, isLoading: logsLoading } = useQuery({
|
||||
queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize],
|
||||
queryFn: () =>
|
||||
|
|
@ -194,6 +192,8 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start
|
|||
/>
|
||||
</div>
|
||||
|
||||
{detailData && <GuardrailUsageBreakdown detail={detailData} />}
|
||||
|
||||
{logViewer("all")}
|
||||
</TabsContent>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,114 @@
|
|||
import { render, screen, within } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown";
|
||||
|
||||
const detail: GuardrailUsageDetail = {
|
||||
guardrail_id: "bedrock-pii-mask",
|
||||
guardrail_name: "bedrock-pii-mask",
|
||||
type: "pii",
|
||||
provider: "Bedrock",
|
||||
requestsEvaluated: 5,
|
||||
failRate: 0,
|
||||
avgScore: null,
|
||||
avgLatency: 120,
|
||||
status: "healthy",
|
||||
trend: "stable",
|
||||
description: null,
|
||||
time_series: [],
|
||||
usage_units: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300, someFutureCounter: 7 },
|
||||
usage_units_daily: [],
|
||||
usage_units_by_team: {
|
||||
"team-a": { contentPolicyUnits: 900, sensitiveInformationPolicyUnits: 300 },
|
||||
"": { contentPolicyUnits: 100, someFutureCounter: 7 },
|
||||
},
|
||||
usage_units_by_key: {
|
||||
"hash-1": { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 300 },
|
||||
"hash-2": { someFutureCounter: 7 },
|
||||
},
|
||||
cost: 0.18,
|
||||
cost_by_unit: { contentPolicyUnits: 0.15, sensitiveInformationPolicyUnits: 0.03, someFutureCounter: null },
|
||||
cost_by_team: { "team-a": 0.165, "": 0.015 },
|
||||
cost_by_key: { "hash-1": 0.18, "hash-2": null },
|
||||
untracked_usage_units: { someFutureCounter: 7 },
|
||||
};
|
||||
|
||||
const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) });
|
||||
|
||||
describe("GuardrailUsageBreakdown", () => {
|
||||
it("totals the units and the cost, and says how many units the cost leaves out", () => {
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
const cost = screen.getByRole("group", { name: "Cost" });
|
||||
expect(cost).toHaveTextContent("$0.1800");
|
||||
expect(cost).toHaveTextContent("7 units unpriced");
|
||||
|
||||
const units = screen.getByRole("group", { name: "Usage Units" });
|
||||
expect(units).toHaveTextContent("1,307");
|
||||
expect(units).toHaveTextContent("3 counters");
|
||||
});
|
||||
|
||||
it("lists each counter with its units, cost and unpriced share", () => {
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
const content = rowNamed("Content Policy");
|
||||
expect(within(content).getByText("1,000")).toBeInTheDocument();
|
||||
expect(within(content).getByText("$0.1500")).toBeInTheDocument();
|
||||
expect(within(content).getByText("—")).toBeInTheDocument();
|
||||
|
||||
const future = rowNamed("Some Future Counter");
|
||||
expect(within(future).getByText("7", { selector: ".text-warning" })).toBeInTheDocument();
|
||||
expect(within(future).getByText("—")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("breaks units and cost down by team and by key, naming the rows without one", () => {
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("heading", { name: "By key" })).toBeInTheDocument();
|
||||
const teamA = rowNamed("team-a");
|
||||
expect(within(teamA).getByText("1,200")).toBeInTheDocument();
|
||||
expect(within(teamA).getByText("$0.1650")).toBeInTheDocument();
|
||||
|
||||
const noTeam = rowNamed("No team");
|
||||
expect(within(noTeam).getByText("107")).toBeInTheDocument();
|
||||
expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument();
|
||||
|
||||
const unpricedKey = rowNamed("hash-2");
|
||||
expect(within(unpricedKey).getByText("7")).toBeInTheDocument();
|
||||
expect(within(unpricedKey).getByText("—")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("orders teams and keys by units, largest first", () => {
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
const rows = screen.getAllByRole("row").map((row) => row.textContent ?? "");
|
||||
expect(rows.findIndex((text) => text.includes("team-a"))).toBeLessThan(
|
||||
rows.findIndex((text) => text.includes("No team")),
|
||||
);
|
||||
expect(rows.findIndex((text) => text.includes("hash-1"))).toBeLessThan(
|
||||
rows.findIndex((text) => text.includes("hash-2")),
|
||||
);
|
||||
});
|
||||
|
||||
it("says so when the window has no billable units instead of rendering empty tables", () => {
|
||||
render(
|
||||
<GuardrailUsageBreakdown
|
||||
detail={{
|
||||
...detail,
|
||||
usage_units: {},
|
||||
usage_units_by_team: {},
|
||||
usage_units_by_key: {},
|
||||
cost: null,
|
||||
cost_by_unit: {},
|
||||
cost_by_team: {},
|
||||
cost_by_key: {},
|
||||
untracked_usage_units: {},
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByText("No billable usage units were recorded in this period.")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("table")).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
import type { ColumnDef } from "@tanstack/react-table";
|
||||
import { CircleDollarSign } from "lucide-react";
|
||||
import React from "react";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import { DataTable } from "@/components/shared/DataTable";
|
||||
import { IdCell } from "@/components/shared/table_cells/id_cell";
|
||||
import { MoneyCell } from "@/components/shared/table_cells/money_cell";
|
||||
|
||||
interface CounterRow {
|
||||
counter: string;
|
||||
units: number;
|
||||
cost: number | null;
|
||||
unpriced: number;
|
||||
}
|
||||
|
||||
interface GroupRow {
|
||||
id: string;
|
||||
units: number;
|
||||
cost: number | null;
|
||||
}
|
||||
|
||||
const counterRows = (detail: GuardrailUsageDetail): CounterRow[] =>
|
||||
Object.entries(detail.usage_units).map(([counter, units]) => ({
|
||||
counter,
|
||||
units,
|
||||
cost: detail.cost_by_unit[counter] ?? null,
|
||||
unpriced: detail.untracked_usage_units[counter] ?? 0,
|
||||
}));
|
||||
|
||||
const groupRows = (
|
||||
unitsByGroup: GuardrailUsageDetail["usage_units_by_team"],
|
||||
costByGroup: GuardrailUsageDetail["cost_by_team"],
|
||||
): GroupRow[] =>
|
||||
Object.entries(unitsByGroup)
|
||||
.map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null }))
|
||||
.sort((a, b) => b.units - a.units);
|
||||
|
||||
const counterColumns: ColumnDef<CounterRow>[] = [
|
||||
{ header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) },
|
||||
{
|
||||
header: "Units",
|
||||
accessorKey: "units",
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) => row.original.units.toLocaleString(),
|
||||
},
|
||||
{
|
||||
header: "Cost",
|
||||
accessorKey: "cost",
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) => <MoneyCell value={row.original.cost} emptyText="—" showZero />,
|
||||
},
|
||||
{
|
||||
header: "Unpriced Units",
|
||||
accessorKey: "unpriced",
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) =>
|
||||
row.original.unpriced > 0 ? (
|
||||
<span className="text-warning">{row.original.unpriced.toLocaleString()}</span>
|
||||
) : (
|
||||
<span className="text-muted-foreground">—</span>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
const groupColumns = (label: string, emptyLabel: string): ColumnDef<GroupRow>[] => [
|
||||
{
|
||||
header: label,
|
||||
accessorKey: "id",
|
||||
cell: ({ row }) =>
|
||||
row.original.id ? (
|
||||
<IdCell value={row.original.id} variant="plain" copyable />
|
||||
) : (
|
||||
<span className="text-muted-foreground">{emptyLabel}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
header: "Units",
|
||||
accessorKey: "units",
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) => row.original.units.toLocaleString(),
|
||||
},
|
||||
{
|
||||
header: "Cost",
|
||||
accessorKey: "cost",
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) => <MoneyCell value={row.original.cost} emptyText="—" showZero />,
|
||||
},
|
||||
];
|
||||
|
||||
const teamColumns = groupColumns("Team", "No team");
|
||||
const keyColumns = groupColumns("Key", "No key");
|
||||
|
||||
const TableHeading = ({ title }: { title: string }) => (
|
||||
<h6 className="text-sm font-semibold text-foreground">{title}</h6>
|
||||
);
|
||||
|
||||
export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDetail }) {
|
||||
const counters = counterRows(detail);
|
||||
const unpriced = unpricedSummary(detail.untracked_usage_units);
|
||||
|
||||
return (
|
||||
<section className="space-y-4" aria-label="Usage and cost">
|
||||
<div>
|
||||
<h5 className="mb-0 text-base font-semibold text-foreground">Usage & Cost</h5>
|
||||
<p className="mt-0.5 text-xs text-muted-foreground">
|
||||
Billable units the provider reported for this guardrail and what LiteLLM priced them at
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{counters.length === 0 ? (
|
||||
<p className="text-sm text-muted-foreground">No billable usage units were recorded in this period.</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="grid grid-cols-2 gap-4 md:grid-cols-3">
|
||||
<MetricCard
|
||||
label="Cost"
|
||||
value={formatCost(detail.cost)}
|
||||
valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={unpriced ?? undefined}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Usage Units"
|
||||
value={totalUnits(detail.usage_units).toLocaleString()}
|
||||
subtitle={`${counters.length} ${counters.length === 1 ? "counter" : "counters"}`}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<DataTable
|
||||
columns={counterColumns}
|
||||
data={counters}
|
||||
getRowId={(row) => row.counter}
|
||||
size="compact"
|
||||
toolbar={() => <TableHeading title="By counter" />}
|
||||
/>
|
||||
|
||||
<div className="grid gap-4 lg:grid-cols-2">
|
||||
<DataTable
|
||||
columns={teamColumns}
|
||||
data={groupRows(detail.usage_units_by_team, detail.cost_by_team)}
|
||||
getRowId={(row) => row.id || "no-team"}
|
||||
size="compact"
|
||||
toolbar={() => <TableHeading title="By team" />}
|
||||
/>
|
||||
<DataTable
|
||||
columns={keyColumns}
|
||||
data={groupRows(detail.usage_units_by_key, detail.cost_by_key)}
|
||||
getRowId={(row) => row.id || "no-key"}
|
||||
size="compact"
|
||||
toolbar={() => <TableHeading title="By key" />}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -2,14 +2,15 @@ import { render, screen, waitFor } from "@testing-library/react";
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import GuardrailsMonitorView from "./GuardrailsMonitorView";
|
||||
import * as networking from "@/components/networking";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGuardrailsUsageOverview: vi.fn(),
|
||||
formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)),
|
||||
}));
|
||||
|
||||
const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview);
|
||||
const mockUseGuardrailsUsageOverview = vi.fn();
|
||||
vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({
|
||||
useGuardrailsUsageOverview: (...args: unknown[]) => mockUseGuardrailsUsageOverview(...args),
|
||||
}));
|
||||
|
||||
function wrapper({ children }: { children: React.ReactNode }) {
|
||||
const queryClient = new QueryClient({
|
||||
|
|
@ -22,23 +23,20 @@ function wrapper({ children }: { children: React.ReactNode }) {
|
|||
|
||||
describe("GuardrailsMonitorView", () => {
|
||||
it("should render overview and fetch guardrails usage when accessToken is provided", async () => {
|
||||
mockGetGuardrailsUsageOverview.mockResolvedValue({
|
||||
rows: [],
|
||||
chart: [],
|
||||
totalRequests: 0,
|
||||
totalBlocked: 0,
|
||||
passRate: 100,
|
||||
});
|
||||
mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: true, error: null });
|
||||
|
||||
render(<GuardrailsMonitorView accessToken="test-token" />, { wrapper });
|
||||
|
||||
expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument();
|
||||
await waitFor(() => {
|
||||
expect(mockGetGuardrailsUsageOverview).toHaveBeenCalled();
|
||||
expect(mockUseGuardrailsUsageOverview).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ accessToken: "test-token", startDate: expect.any(String) }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("should render without crashing when accessToken is null", async () => {
|
||||
mockUseGuardrailsUsageOverview.mockReturnValue({ data: undefined, isLoading: false, error: null });
|
||||
render(<GuardrailsMonitorView accessToken={null} />, { wrapper });
|
||||
expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeInTheDocument();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { render, screen, waitFor, within } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import * as networking from "@/components/networking";
|
||||
import type {
|
||||
GuardrailUsageOverview,
|
||||
GuardrailUsageOverviewRow,
|
||||
} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { GuardrailsOverview } from "./GuardrailsOverview";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGuardrailsUsageOverview: vi.fn(),
|
||||
const useGuardrailsUsageOverviewMock = vi.fn();
|
||||
vi.mock("@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage", () => ({
|
||||
useGuardrailsUsageOverview: (...args: unknown[]) => useGuardrailsUsageOverviewMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock("./ScoreChart", () => ({
|
||||
|
|
@ -17,16 +20,63 @@ vi.mock("./EvaluationSettingsModal", () => ({
|
|||
EvaluationSettingsModal: ({ open }: { open: boolean }) => (open ? <div>Evaluation settings modal</div> : null),
|
||||
}));
|
||||
|
||||
const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview);
|
||||
const row = (overrides: Partial<GuardrailUsageOverviewRow>): GuardrailUsageOverviewRow => ({
|
||||
id: "guardrail",
|
||||
name: "Guardrail",
|
||||
type: "content_filter",
|
||||
provider: "LiteLLM",
|
||||
requestsEvaluated: 0,
|
||||
failRate: 0,
|
||||
avgScore: null,
|
||||
avgLatency: null,
|
||||
status: "healthy",
|
||||
trend: "stable",
|
||||
usageUnits: {},
|
||||
cost: null,
|
||||
untrackedUsageUnits: {},
|
||||
...overrides,
|
||||
});
|
||||
|
||||
function wrapper({ children }: { children: React.ReactNode }) {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: { retry: false },
|
||||
},
|
||||
});
|
||||
return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>;
|
||||
}
|
||||
const overview: GuardrailUsageOverview = {
|
||||
rows: [
|
||||
row({
|
||||
id: "guardrail-low",
|
||||
name: "Low Failure Guardrail",
|
||||
requestsEvaluated: 1200,
|
||||
failRate: 2.5,
|
||||
avgLatency: 45,
|
||||
trend: "down",
|
||||
}),
|
||||
row({
|
||||
id: "guardrail-high",
|
||||
name: "High Failure Guardrail",
|
||||
provider: "Bedrock",
|
||||
requestsEvaluated: 300,
|
||||
failRate: 18,
|
||||
status: "warning",
|
||||
trend: "up",
|
||||
usageUnits: { contentPolicyUnits: 1000, sensitiveInformationPolicyUnits: 250 },
|
||||
cost: 0.15,
|
||||
untrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 },
|
||||
}),
|
||||
row({
|
||||
id: "guardrail-free",
|
||||
name: "Free Bedrock Guardrail",
|
||||
provider: "Bedrock",
|
||||
requestsEvaluated: 10,
|
||||
failRate: 0,
|
||||
usageUnits: { contentPolicyUnits: 40 },
|
||||
cost: 0,
|
||||
}),
|
||||
],
|
||||
chart: [],
|
||||
totalRequests: 1510,
|
||||
totalBlocked: 84,
|
||||
passRate: 94.4,
|
||||
totalUsageUnits: { contentPolicyUnits: 1040, sensitiveInformationPolicyUnits: 250 },
|
||||
totalCost: 0.15,
|
||||
totalUntrackedUsageUnits: { sensitiveInformationPolicyUnits: 250 },
|
||||
};
|
||||
|
||||
function renderOverview(onSelectGuardrail = vi.fn()) {
|
||||
return render(
|
||||
|
|
@ -36,41 +86,24 @@ function renderOverview(onSelectGuardrail = vi.fn()) {
|
|||
endDate="2026-08-12"
|
||||
onSelectGuardrail={onSelectGuardrail}
|
||||
/>,
|
||||
{ wrapper },
|
||||
);
|
||||
}
|
||||
|
||||
const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) });
|
||||
|
||||
describe("GuardrailsOverview", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockGetGuardrailsUsageOverview.mockResolvedValue({
|
||||
rows: [
|
||||
{
|
||||
id: "guardrail-low",
|
||||
name: "Low Failure Guardrail",
|
||||
type: "content_filter",
|
||||
provider: "LiteLLM",
|
||||
requestsEvaluated: 1200,
|
||||
failRate: 2.5,
|
||||
avgLatency: 45,
|
||||
status: "healthy",
|
||||
trend: "down",
|
||||
},
|
||||
{
|
||||
id: "guardrail-high",
|
||||
name: "High Failure Guardrail",
|
||||
type: "content_filter",
|
||||
provider: "Bedrock",
|
||||
requestsEvaluated: 300,
|
||||
failRate: 18,
|
||||
status: "warning",
|
||||
trend: "up",
|
||||
},
|
||||
],
|
||||
chart: [],
|
||||
totalRequests: 1500,
|
||||
totalBlocked: 84,
|
||||
passRate: 94.4,
|
||||
useGuardrailsUsageOverviewMock.mockReturnValue({ data: overview, isLoading: false, error: null });
|
||||
});
|
||||
|
||||
it("asks for the usage overview of the selected window", () => {
|
||||
renderOverview();
|
||||
|
||||
expect(useGuardrailsUsageOverviewMock).toHaveBeenCalledWith({
|
||||
accessToken: "test-token",
|
||||
startDate: "2026-08-01",
|
||||
endDate: "2026-08-12",
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -78,15 +111,7 @@ describe("GuardrailsOverview", () => {
|
|||
const onSelectGuardrail = vi.fn();
|
||||
const user = userEvent.setup();
|
||||
|
||||
render(
|
||||
<GuardrailsOverview
|
||||
accessToken="test-token"
|
||||
startDate="2026-08-01"
|
||||
endDate="2026-08-12"
|
||||
onSelectGuardrail={onSelectGuardrail}
|
||||
/>,
|
||||
{ wrapper },
|
||||
);
|
||||
renderOverview(onSelectGuardrail);
|
||||
|
||||
expect(await screen.findByRole("columnheader", { name: "Guardrail" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("columnheader", { name: /Requests/ })).toBeInTheDocument();
|
||||
|
|
@ -105,6 +130,46 @@ describe("GuardrailsOverview", () => {
|
|||
expect(onSelectGuardrail).toHaveBeenCalledWith("guardrail-low");
|
||||
});
|
||||
|
||||
it("shows each guardrail's usage units and cost, marking the units cost leaves out", async () => {
|
||||
renderOverview();
|
||||
|
||||
expect(await screen.findByRole("columnheader", { name: "Usage Units" })).toBeInTheDocument();
|
||||
expect(screen.getByRole("columnheader", { name: /Cost/ })).toBeInTheDocument();
|
||||
|
||||
const priced = rowNamed("High Failure Guardrail");
|
||||
expect(within(priced).getByText("1,250")).toBeInTheDocument();
|
||||
expect(within(priced).getByText("$0.1500")).toBeInTheDocument();
|
||||
expect(within(priced).getByLabelText("250 units unpriced")).toBeInTheDocument();
|
||||
|
||||
const free = rowNamed("Free Bedrock Guardrail");
|
||||
expect(within(free).getByText("40")).toBeInTheDocument();
|
||||
expect(within(free).getByText("$0.0000")).toBeInTheDocument();
|
||||
expect(within(free).queryByLabelText(/unpriced/)).not.toBeInTheDocument();
|
||||
|
||||
const unmetered = rowNamed("Low Failure Guardrail");
|
||||
expect(within(unmetered).getAllByText("—")).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("breaks the usage units down per counter on hover", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderOverview();
|
||||
|
||||
await user.hover(within(rowNamed("High Failure Guardrail")).getByText("1,250"));
|
||||
|
||||
expect(await screen.findByText("Content Policy: 1,000")).toBeInTheDocument();
|
||||
expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("sorts by cost when its header is clicked", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderOverview();
|
||||
|
||||
await user.click(await screen.findByRole("button", { name: /Cost/ }));
|
||||
|
||||
await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail"));
|
||||
expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail");
|
||||
});
|
||||
|
||||
it("renders the page header and the export action", async () => {
|
||||
renderOverview();
|
||||
|
||||
|
|
@ -117,15 +182,36 @@ describe("GuardrailsOverview", () => {
|
|||
it("renders every summary metric card", async () => {
|
||||
renderOverview();
|
||||
|
||||
expect(await screen.findByText("1,500")).toBeInTheDocument();
|
||||
expect(await screen.findByText("1,510")).toBeInTheDocument();
|
||||
expect(screen.getByText("Total Evaluations")).toBeInTheDocument();
|
||||
expect(screen.getByText("Blocked Requests")).toBeInTheDocument();
|
||||
expect(screen.getByText("84")).toBeInTheDocument();
|
||||
expect(screen.getByText("Pass Rate")).toBeInTheDocument();
|
||||
expect(screen.getByText("94.4%")).toBeInTheDocument();
|
||||
expect(screen.getByText("23ms")).toBeInTheDocument();
|
||||
expect(screen.getByText("15ms")).toBeInTheDocument();
|
||||
expect(screen.getByText("Active Guardrails")).toBeInTheDocument();
|
||||
expect(screen.getByText("2")).toBeInTheDocument();
|
||||
expect(screen.getByText("3")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("totals guardrail cost across the window and says how many units it leaves out", async () => {
|
||||
renderOverview();
|
||||
|
||||
const card = await screen.findByRole("group", { name: "Guardrail Cost" });
|
||||
expect(card).toHaveTextContent("$0.1500");
|
||||
expect(card).toHaveTextContent("250 units unpriced");
|
||||
});
|
||||
|
||||
it("shows a dash for guardrail cost when nothing in the window was priced", async () => {
|
||||
useGuardrailsUsageOverviewMock.mockReturnValue({
|
||||
data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} },
|
||||
isLoading: false,
|
||||
error: null,
|
||||
});
|
||||
renderOverview();
|
||||
|
||||
const card = await screen.findByRole("group", { name: "Guardrail Cost" });
|
||||
expect(card).toHaveTextContent("—");
|
||||
expect(card).not.toHaveTextContent("unpriced");
|
||||
});
|
||||
|
||||
it("renders the table toolbar heading and its description", async () => {
|
||||
|
|
@ -147,14 +233,18 @@ describe("GuardrailsOverview", () => {
|
|||
});
|
||||
|
||||
it("marks the overview busy while the usage request is in flight", async () => {
|
||||
mockGetGuardrailsUsageOverview.mockReturnValue(new Promise(() => {}));
|
||||
useGuardrailsUsageOverviewMock.mockReturnValue({ data: undefined, isLoading: true, error: null });
|
||||
renderOverview();
|
||||
|
||||
await waitFor(() => expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument());
|
||||
});
|
||||
|
||||
it("shows a failure message when the usage request rejects", async () => {
|
||||
mockGetGuardrailsUsageOverview.mockRejectedValue(new Error("network down"));
|
||||
useGuardrailsUsageOverviewMock.mockReturnValue({
|
||||
data: undefined,
|
||||
isLoading: false,
|
||||
error: new Error("network down"),
|
||||
});
|
||||
renderOverview();
|
||||
|
||||
expect(await screen.findByText("Failed to load data. Try again.")).toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
import { useQuery } from "@tanstack/react-query";
|
||||
import type { ColumnDef, OnChangeFn, SortingState } from "@tanstack/react-table";
|
||||
import { Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react";
|
||||
import { CircleDollarSign, Download, HeartPulse, Settings, TrendingUp, TriangleAlert } from "lucide-react";
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { DataTable, DataTableSortHeader } from "@/components/shared/DataTable";
|
||||
import { getGuardrailsUsageOverview } from "@/components/networking";
|
||||
import { type PerformanceRow } from "@/components/GuardrailsMonitor/mockData";
|
||||
import { MoneyCell } from "@/components/shared/table_cells/money_cell";
|
||||
import { CellTooltip } from "@/components/shared/table_cells/cell_tooltip";
|
||||
import {
|
||||
type GuardrailUsageOverviewRow,
|
||||
useGuardrailsUsageOverview,
|
||||
} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { PageHeader } from "@/components/shared/PageHeader";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
|
|
@ -20,7 +24,7 @@ interface GuardrailsOverviewProps {
|
|||
dateRangeControl?: React.ReactNode;
|
||||
}
|
||||
|
||||
type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "falsePositiveRate" | "falseNegativeRate";
|
||||
type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "cost";
|
||||
|
||||
const providerColors: Record<string, string> = {
|
||||
Bedrock: "bg-warning/15 text-warning border-warning/20",
|
||||
|
|
@ -30,14 +34,48 @@ const providerColors: Record<string, string> = {
|
|||
Custom: "bg-muted text-muted-foreground border-border",
|
||||
};
|
||||
|
||||
function computeMetricsFromRows(data: PerformanceRow[]) {
|
||||
const totalRequests = data.reduce((sum, r) => sum + r.requestsEvaluated, 0);
|
||||
const totalBlocked = data.reduce((sum, r) => sum + Math.round((r.requestsEvaluated * r.failRate) / 100), 0);
|
||||
const passRate = totalRequests > 0 ? ((1 - totalBlocked / totalRequests) * 100).toFixed(1) : "0";
|
||||
const withLat = data.filter((r) => r.avgLatency != null);
|
||||
const avgLatency =
|
||||
withLat.length > 0 ? Math.round(withLat.reduce((sum, r) => sum + (r.avgLatency ?? 0), 0) / withLat.length) : 0;
|
||||
return { totalRequests, totalBlocked, passRate, avgLatency, count: data.length };
|
||||
const EMPTY_METRICS = {
|
||||
totalRequests: 0,
|
||||
totalBlocked: 0,
|
||||
passRate: "0",
|
||||
avgLatency: 0,
|
||||
count: 0,
|
||||
totalCost: null as number | null,
|
||||
unpriced: null as string | null,
|
||||
};
|
||||
|
||||
function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) {
|
||||
const counters = Object.entries(units);
|
||||
if (counters.length === 0) return <span className="text-muted-foreground">—</span>;
|
||||
return (
|
||||
<CellTooltip
|
||||
content={
|
||||
<ul className="space-y-0.5">
|
||||
{counters.map(([counter, n]) => (
|
||||
<li key={counter}>
|
||||
{counterLabel(counter)}: {n.toLocaleString()}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
}
|
||||
trigger={<span className="tabular-nums">{totalUnits(units).toLocaleString()}</span>}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function CostCell({ row }: { row: GuardrailUsageOverviewRow }) {
|
||||
const unpriced = unpricedSummary(row.untrackedUsageUnits);
|
||||
return (
|
||||
<span className="inline-flex w-full items-center justify-end gap-1">
|
||||
{unpriced && (
|
||||
<CellTooltip
|
||||
content={`${unpriced}: these units have no known price and are left out of the cost`}
|
||||
trigger={<TriangleAlert aria-label={unpriced} className="size-3.5 shrink-0 text-warning" />}
|
||||
/>
|
||||
)}
|
||||
<MoneyCell value={row.cost} emptyText="—" showZero />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
export function GuardrailsOverview({
|
||||
|
|
@ -55,26 +93,22 @@ export function GuardrailsOverview({
|
|||
data: guardrailsData,
|
||||
isLoading: guardrailsLoading,
|
||||
error: guardrailsError,
|
||||
} = useQuery({
|
||||
queryKey: ["guardrails-usage-overview", startDate, endDate],
|
||||
queryFn: () => getGuardrailsUsageOverview(accessToken!, startDate, endDate),
|
||||
enabled: !!accessToken,
|
||||
});
|
||||
} = useGuardrailsUsageOverview({ accessToken, startDate, endDate });
|
||||
|
||||
const activeData: PerformanceRow[] = guardrailsData?.rows ?? [];
|
||||
const activeData: GuardrailUsageOverviewRow[] = useMemo(() => guardrailsData?.rows ?? [], [guardrailsData]);
|
||||
const metrics = useMemo(() => {
|
||||
if (guardrailsData) {
|
||||
return {
|
||||
totalRequests: guardrailsData.totalRequests ?? 0,
|
||||
totalBlocked: guardrailsData.totalBlocked ?? 0,
|
||||
passRate: String(guardrailsData.passRate ?? 0),
|
||||
avgLatency: activeData.length
|
||||
? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length)
|
||||
: 0,
|
||||
count: activeData.length,
|
||||
};
|
||||
}
|
||||
return computeMetricsFromRows(activeData);
|
||||
if (!guardrailsData) return EMPTY_METRICS;
|
||||
return {
|
||||
totalRequests: guardrailsData.totalRequests,
|
||||
totalBlocked: guardrailsData.totalBlocked,
|
||||
passRate: String(guardrailsData.passRate),
|
||||
avgLatency: activeData.length
|
||||
? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length)
|
||||
: 0,
|
||||
count: activeData.length,
|
||||
totalCost: guardrailsData.totalCost,
|
||||
unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits),
|
||||
};
|
||||
}, [guardrailsData, activeData]);
|
||||
const chartData = guardrailsData?.chart;
|
||||
const sorted = useMemo(() => {
|
||||
|
|
@ -88,7 +122,7 @@ export function GuardrailsOverview({
|
|||
const isLoading = guardrailsLoading;
|
||||
const error = guardrailsError;
|
||||
|
||||
const columns: ColumnDef<PerformanceRow>[] = [
|
||||
const columns: ColumnDef<GuardrailUsageOverviewRow>[] = [
|
||||
{
|
||||
header: "Guardrail",
|
||||
accessorKey: "name",
|
||||
|
|
@ -166,6 +200,20 @@ export function GuardrailsOverview({
|
|||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
header: "Usage Units",
|
||||
accessorKey: "usageUnits",
|
||||
enableSorting: false,
|
||||
meta: { numeric: true },
|
||||
cell: ({ row }) => <UsageUnitsCell units={row.original.usageUnits} />,
|
||||
},
|
||||
{
|
||||
header: ({ column }) => <DataTableSortHeader column={column} title="Cost" />,
|
||||
accessorKey: "cost",
|
||||
meta: { numeric: true },
|
||||
sortDescFirst: false,
|
||||
cell: ({ row }) => <CostCell row={row.original} />,
|
||||
},
|
||||
{
|
||||
header: "Status",
|
||||
accessorKey: "status",
|
||||
|
|
@ -187,7 +235,7 @@ export function GuardrailsOverview({
|
|||
},
|
||||
];
|
||||
|
||||
const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency"];
|
||||
const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"];
|
||||
const sorting = useMemo<SortingState>(() => [{ id: sortBy, desc: sortDir === "desc" }], [sortBy, sortDir]);
|
||||
const handleSortingChange: OnChangeFn<SortingState> = (updater) => {
|
||||
const nextSorting = typeof updater === "function" ? updater(sorting) : updater;
|
||||
|
|
@ -236,6 +284,13 @@ export function GuardrailsOverview({
|
|||
metrics.avgLatency > 150 ? "text-destructive" : metrics.avgLatency > 50 ? "text-warning" : "text-success"
|
||||
}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Guardrail Cost"
|
||||
value={formatCost(metrics.totalCost)}
|
||||
valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={metrics.unpriced ?? undefined}
|
||||
/>
|
||||
<MetricCard label="Active Guardrails" value={metrics.count} />
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,23 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
|
|||
|
||||
const fetchMock = vi.fn();
|
||||
|
||||
const requestedUrls = () => fetchMock.mock.calls.map(([url]) => String(url));
|
||||
const requestUrl = (input: RequestInfo | URL) => (input instanceof Request ? input.url : String(input));
|
||||
|
||||
const requestedUrls = () => fetchMock.mock.calls.map(([input]) => requestUrl(input));
|
||||
|
||||
const emptyOverview = {
|
||||
rows: [],
|
||||
chart: [],
|
||||
totalRequests: 0,
|
||||
totalBlocked: 0,
|
||||
passRate: 100,
|
||||
totalUsageUnits: {},
|
||||
totalCost: null,
|
||||
totalUntrackedUsageUnits: {},
|
||||
};
|
||||
|
||||
const jsonResponse = (body: unknown) =>
|
||||
new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } });
|
||||
|
||||
const renderAs = (userRole: string) => {
|
||||
useAuthorizedMock.mockReturnValue({ accessToken: "sk-test", userId: "u1", userRole });
|
||||
|
|
@ -25,12 +41,9 @@ describe("Guardrails Monitor page access by role", () => {
|
|||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.clearAllMocks();
|
||||
fetchMock.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
statusText: "OK",
|
||||
json: async () => ({ rows: [], chart: [], totalRequests: 0, totalBlocked: 0, passRate: 100 }),
|
||||
});
|
||||
fetchMock.mockImplementation(async (input: RequestInfo | URL) =>
|
||||
jsonResponse(requestUrl(input).includes("/guardrails/usage/overview") ? emptyOverview : []),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
import { renderHook } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useGuardrailsUsageDetail, useGuardrailsUsageOverview } from "./useGuardrailsUsage";
|
||||
|
||||
const useQueryMock = vi.fn();
|
||||
vi.mock("@/lib/http/api", () => ({
|
||||
$api: { useQuery: (...args: unknown[]) => useQueryMock(...args) },
|
||||
}));
|
||||
|
||||
const lastCall = () => {
|
||||
const calls = useQueryMock.mock.calls;
|
||||
return calls[calls.length - 1] as [string, string, unknown, { enabled: boolean }];
|
||||
};
|
||||
|
||||
describe("useGuardrailsUsageOverview", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useQueryMock.mockReturnValue({ data: undefined });
|
||||
});
|
||||
|
||||
it("queries GET /guardrails/usage/overview with the window as query params", () => {
|
||||
renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "2026-09-01", endDate: "2026-09-04" }));
|
||||
|
||||
expect(lastCall().slice(0, 3)).toEqual([
|
||||
"get",
|
||||
"/guardrails/usage/overview",
|
||||
{ params: { query: { start_date: "2026-09-01", end_date: "2026-09-04" } } },
|
||||
]);
|
||||
expect(lastCall()[3].enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("omits blank dates so the proxy applies its default window", () => {
|
||||
renderHook(() => useGuardrailsUsageOverview({ accessToken: "sk", startDate: "", endDate: "" }));
|
||||
|
||||
expect(lastCall()[2]).toEqual({ params: { query: { start_date: undefined, end_date: undefined } } });
|
||||
});
|
||||
|
||||
it("stays disabled without an access token", () => {
|
||||
renderHook(() => useGuardrailsUsageOverview({ accessToken: null, startDate: "2026-09-01", endDate: "2026-09-04" }));
|
||||
|
||||
expect(lastCall()[3].enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("useGuardrailsUsageDetail", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
useQueryMock.mockReturnValue({ data: undefined });
|
||||
});
|
||||
|
||||
it("queries GET /guardrails/usage/detail/{guardrail_id} with the id as a path param", () => {
|
||||
renderHook(() =>
|
||||
useGuardrailsUsageDetail({
|
||||
accessToken: "sk",
|
||||
guardrailId: "bedrock-pii-mask",
|
||||
startDate: "2026-09-01",
|
||||
endDate: "2026-09-04",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(lastCall().slice(0, 3)).toEqual([
|
||||
"get",
|
||||
"/guardrails/usage/detail/{guardrail_id}",
|
||||
{
|
||||
params: {
|
||||
path: { guardrail_id: "bedrock-pii-mask" },
|
||||
query: { start_date: "2026-09-01", end_date: "2026-09-04" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(lastCall()[3].enabled).toBe(true);
|
||||
});
|
||||
|
||||
it("stays disabled without a guardrail id", () => {
|
||||
renderHook(() =>
|
||||
useGuardrailsUsageDetail({ accessToken: "sk", guardrailId: "", startDate: "2026-09-01", endDate: "2026-09-04" }),
|
||||
);
|
||||
|
||||
expect(lastCall()[3].enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import { $api } from "@/lib/http/api";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
export type GuardrailUsageOverview = components["schemas"]["UsageOverviewResponse"];
|
||||
export type GuardrailUsageOverviewRow = components["schemas"]["UsageOverviewRow"];
|
||||
export type GuardrailUsageDetail = components["schemas"]["UsageDetailResponse"];
|
||||
|
||||
export interface GuardrailsUsageWindow {
|
||||
accessToken: string | null;
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
}
|
||||
|
||||
const dateQuery = (startDate: string, endDate: string) => ({
|
||||
start_date: startDate || undefined,
|
||||
end_date: endDate || undefined,
|
||||
});
|
||||
|
||||
export const useGuardrailsUsageOverview = ({ accessToken, startDate, endDate }: GuardrailsUsageWindow) =>
|
||||
$api.useQuery(
|
||||
"get",
|
||||
"/guardrails/usage/overview",
|
||||
{ params: { query: dateQuery(startDate, endDate) } },
|
||||
{ enabled: Boolean(accessToken) },
|
||||
);
|
||||
|
||||
export const useGuardrailsUsageDetail = ({
|
||||
accessToken,
|
||||
guardrailId,
|
||||
startDate,
|
||||
endDate,
|
||||
}: GuardrailsUsageWindow & { guardrailId: string }) =>
|
||||
$api.useQuery(
|
||||
"get",
|
||||
"/guardrails/usage/detail/{guardrail_id}",
|
||||
{ params: { path: { guardrail_id: guardrailId }, query: dateQuery(startDate, endDate) } },
|
||||
{ enabled: Boolean(accessToken && guardrailId) },
|
||||
);
|
||||
|
|
@ -10,7 +10,7 @@ interface MetricCardProps {
|
|||
|
||||
export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) {
|
||||
return (
|
||||
<div className="h-full bg-card border border-border rounded-lg p-5 flex flex-col">
|
||||
<div role="group" aria-label={label} className="h-full bg-card border border-border rounded-lg p-5 flex flex-col">
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
<span className="text-sm font-medium text-muted-foreground">{label}</span>
|
||||
{icon && <span className="text-muted-foreground">{icon}</span>}
|
||||
|
|
|
|||
|
|
@ -2,39 +2,6 @@
|
|||
* Types for Guardrails Monitor dashboard (data from usage API).
|
||||
*/
|
||||
|
||||
export interface PerformanceRow {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
provider: string;
|
||||
requestsEvaluated: number;
|
||||
failRate: number;
|
||||
avgScore?: number;
|
||||
avgLatency?: number;
|
||||
p95Latency?: number;
|
||||
falsePositiveRate?: number;
|
||||
falseNegativeRate?: number;
|
||||
status: "healthy" | "warning" | "critical";
|
||||
trend: "up" | "down" | "stable";
|
||||
}
|
||||
|
||||
export interface GuardrailDetailRecord {
|
||||
name: string;
|
||||
type: string;
|
||||
provider: string;
|
||||
requestsEvaluated: number;
|
||||
failRate: number;
|
||||
avgScore?: number;
|
||||
avgLatency?: number;
|
||||
p95Latency?: number;
|
||||
falsePositiveRate?: number;
|
||||
falsePositiveCount?: number;
|
||||
falseNegativeRate?: number;
|
||||
falseNegativeCount?: number;
|
||||
status: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface LogEntry {
|
||||
id: string;
|
||||
timestamp: string;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits";
|
||||
|
||||
describe("formatCost", () => {
|
||||
it("renders a dash when nothing was priced", () => {
|
||||
expect(formatCost(null)).toBe("—");
|
||||
expect(formatCost(undefined)).toBe("—");
|
||||
});
|
||||
|
||||
it("keeps an explicit zero as a real price rather than a dash", () => {
|
||||
expect(formatCost(0)).toBe("$0.0000");
|
||||
});
|
||||
|
||||
it("shows four decimals for the sub-cent amounts guardrail units cost", () => {
|
||||
expect(formatCost(0.0003)).toBe("$0.0003");
|
||||
expect(formatCost(12.5)).toBe("$12.5000");
|
||||
});
|
||||
|
||||
it("flags amounts below the displayed precision instead of rounding them to zero", () => {
|
||||
expect(formatCost(0.00001)).toBe("< $0.0001");
|
||||
});
|
||||
});
|
||||
|
||||
describe("totalUnits", () => {
|
||||
it("sums every counter", () => {
|
||||
expect(totalUnits({ contentPolicyUnits: 3, sensitiveInformationPolicyUnits: 4 })).toBe(7);
|
||||
});
|
||||
|
||||
it("is zero for no counters", () => {
|
||||
expect(totalUnits({})).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("counterLabel", () => {
|
||||
it("turns a Bedrock counter name into words without the Units suffix", () => {
|
||||
expect(counterLabel("sensitiveInformationPolicyUnits")).toBe("Sensitive Information Policy");
|
||||
expect(counterLabel("contentPolicyUnits")).toBe("Content Policy");
|
||||
});
|
||||
|
||||
it("leaves a name it cannot split alone apart from capitalising it", () => {
|
||||
expect(counterLabel("units")).toBe("Units");
|
||||
});
|
||||
});
|
||||
|
||||
describe("unpricedSummary", () => {
|
||||
it("is null when every unit was priced", () => {
|
||||
expect(unpricedSummary({})).toBeNull();
|
||||
expect(unpricedSummary({ contentPolicyUnits: 0 })).toBeNull();
|
||||
});
|
||||
|
||||
it("counts unpriced units across counters with a pluralised label", () => {
|
||||
expect(unpricedSummary({ contentPolicyUnits: 1200, someFutureCounter: 34 })).toBe("1,234 units unpriced");
|
||||
expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils";
|
||||
|
||||
export type UsageUnits = Readonly<Record<string, number>>;
|
||||
|
||||
export const formatCost = (cost: number | null | undefined): string => {
|
||||
if (cost == null) return "—";
|
||||
return cost === 0 ? `$${formatNumberWithCommas(0, 4)}` : getSpendString(cost, 4);
|
||||
};
|
||||
|
||||
export const totalUnits = (units: UsageUnits): number => Object.values(units).reduce((sum, n) => sum + n, 0);
|
||||
|
||||
export const counterLabel = (counter: string): string =>
|
||||
counter
|
||||
.replace(/Units$/, "")
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||
.replace(/^./, (c) => c.toUpperCase());
|
||||
|
||||
export const unpricedSummary = (untracked: UsageUnits): string | null => {
|
||||
const total = totalUnits(untracked);
|
||||
return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null;
|
||||
};
|
||||
|
|
@ -3956,63 +3956,6 @@ export const rejectGuardrailSubmission = async (
|
|||
};
|
||||
|
||||
// Guardrails / Policies usage (dashboard)
|
||||
export const getGuardrailsUsageOverview = async (accessToken: string, startDate?: string, endDate?: string) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/guardrails/usage/overview` : `/guardrails/usage/overview`;
|
||||
const params = new URLSearchParams();
|
||||
if (startDate) params.append("start_date", startDate);
|
||||
if (endDate) params.append("end_date", endDate);
|
||||
if (params.toString()) url += `?${params.toString()}`;
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
throw new Error(deriveErrorMessage(errorData));
|
||||
}
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error("Failed to get guardrails usage overview:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const getGuardrailsUsageDetail = async (
|
||||
accessToken: string,
|
||||
guardrailId: string,
|
||||
startDate?: string,
|
||||
endDate?: string,
|
||||
) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/guardrails/usage/detail/${encodeURIComponent(guardrailId)}`
|
||||
: `/guardrails/usage/detail/${encodeURIComponent(guardrailId)}`;
|
||||
const params = new URLSearchParams();
|
||||
if (startDate) params.append("start_date", startDate);
|
||||
if (endDate) params.append("end_date", endDate);
|
||||
if (params.toString()) url += `?${params.toString()}`;
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
throw new Error(deriveErrorMessage(errorData));
|
||||
}
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error("Failed to get guardrails usage detail:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const getGuardrailsUsageLogs = async (
|
||||
accessToken: string,
|
||||
options: {
|
||||
|
|
|
|||
|
|
@ -43,11 +43,15 @@ const middleware: Middleware = {
|
|||
*
|
||||
* The base URL is injected, not fixed at import: every request is built against
|
||||
* whatever registerBaseUrlGetter supplies at call time (a split-origin proxy or
|
||||
* worker URL), falling back to the current origin. The middleware injects the
|
||||
* auth header and maps non-2xx responses to ApiError so query functions can just
|
||||
* read `.data`.
|
||||
* worker URL), falling back to the current origin. `fetch` is looked up per
|
||||
* request for the same reason, so a test that stubs the global sees these calls
|
||||
* too. The middleware injects the auth header and maps non-2xx responses to
|
||||
* ApiError so query functions can just read `.data`.
|
||||
*/
|
||||
export const fetchClient = createFetchClient<paths>({ Request: BaseAwareRequest });
|
||||
export const fetchClient = createFetchClient<paths>({
|
||||
Request: BaseAwareRequest,
|
||||
fetch: (request) => globalThis.fetch(request),
|
||||
});
|
||||
fetchClient.use(middleware);
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue