mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat(ui): add configuration tabs to the Cost Optimization page (#33899)
* feat(ui): add configuration tabs to Cost Optimization page
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* feat(ui): reuse AutoRouter v2 and Router Settings prompt-caching panel in Cost Optimization; clarify Headroom compression
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* feat(ui): add experimental dashboard banner with feedback discussion link to Cost Optimization
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* feat(ui): add savings methodology note and per-key/team compression enterprise callout to Cost Optimization
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
* test(ui): assert active tab state in Cost Optimization tab-switch test
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---------
Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
(cherry picked from commit 34561482ed)
This commit is contained in:
devin-ai-integration[bot]
Yuneng Jiang
parent
5d66f1cbd1
commit
0b8c13fa9c
10 changed files with 699 additions and 229 deletions
|
|
@ -0,0 +1,28 @@
|
|||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import { Form } from "antd";
|
||||
|
||||
import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab";
|
||||
|
||||
interface AutorouterTabProps {
|
||||
accessToken: string | null;
|
||||
userId: string | null;
|
||||
userRole: string;
|
||||
}
|
||||
|
||||
const AutorouterTab: React.FC<AutorouterTabProps> = ({ accessToken, userRole }) => {
|
||||
const [form] = Form.useForm();
|
||||
|
||||
if (!accessToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
<AddAutoRouterTab form={form} handleOk={() => form.resetFields()} accessToken={accessToken} userRole={userRole} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AutorouterTab;
|
||||
|
|
@ -1,109 +1,34 @@
|
|||
import { render } from "@testing-library/react";
|
||||
import { fireEvent, render } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
|
||||
const mockUsePaginatedDailyActivity = vi.fn();
|
||||
|
||||
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
|
||||
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
userDailyActivityCall: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/advanced_date_picker", () => ({
|
||||
__esModule: true,
|
||||
default: () => <div data-testid="date-picker" />,
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/charts", () => ({
|
||||
AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => (
|
||||
<div data-testid="area-chart" data-categories={categories.join(",")} data-series={JSON.stringify(data)} />
|
||||
),
|
||||
DonutChart: ({ data, label }: { data: unknown; label: string }) => (
|
||||
<div data-testid="donut-chart" data-label={label} data-slices={JSON.stringify(data)} />
|
||||
),
|
||||
}));
|
||||
vi.mock("./UsageTab", () => ({ __esModule: true, default: () => <div data-testid="usage-tab" /> }));
|
||||
vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () => <div data-testid="compression-tab" /> }));
|
||||
vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () => <div data-testid="autorouter-tab" /> }));
|
||||
vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () => <div data-testid="caching-tab" /> }));
|
||||
|
||||
import CostOptimizationView from "./CostOptimizationView";
|
||||
|
||||
const baseMetrics = (overrides: Partial<SpendMetrics>): SpendMetrics => ({
|
||||
spend: 0,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
api_requests: 0,
|
||||
successful_requests: 0,
|
||||
failed_requests: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
|
||||
date,
|
||||
metrics: baseMetrics(metrics),
|
||||
breakdown: {
|
||||
models: {},
|
||||
model_groups: {},
|
||||
mcp_servers: {},
|
||||
providers: {},
|
||||
api_keys: {},
|
||||
entities: {},
|
||||
},
|
||||
});
|
||||
|
||||
const renderWith = (results: DailyData[]) => {
|
||||
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
|
||||
return render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
|
||||
};
|
||||
const renderView = () => render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
|
||||
|
||||
describe("CostOptimizationView", () => {
|
||||
it("sums compression and caching dollars across days into the summary cards", () => {
|
||||
const { getByText } = renderWith([
|
||||
day("2026-07-12", {
|
||||
compression_savings_spend: 0.04,
|
||||
prompt_caching_savings_spend: 0.006,
|
||||
compression_saved_tokens: 40000,
|
||||
}),
|
||||
day("2026-07-13", {
|
||||
compression_savings_spend: 0.1,
|
||||
prompt_caching_savings_spend: 0.01,
|
||||
compression_saved_tokens: 100000,
|
||||
}),
|
||||
]);
|
||||
it("renders all four cost-optimization tabs", () => {
|
||||
const { getByText } = renderView();
|
||||
|
||||
// compression 0.14 + caching 0.016 = 0.156
|
||||
expect(getByText("$0.1560")).toBeInTheDocument();
|
||||
expect(getByText("$0.1400")).toBeInTheDocument();
|
||||
expect(getByText("$0.0160")).toBeInTheDocument();
|
||||
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
|
||||
expect(getByText("Usage")).toBeInTheDocument();
|
||||
expect(getByText("Prompt Compression")).toBeInTheDocument();
|
||||
expect(getByText("Autorouter")).toBeInTheDocument();
|
||||
expect(getByText("Prompt Caching")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("builds a per-day time series and per-driver donut from the daily rows", () => {
|
||||
const { getByTestId } = renderWith([
|
||||
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
|
||||
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
|
||||
]);
|
||||
it("defaults to the Usage tab and switches the active tab on click", () => {
|
||||
const { getByRole } = renderView();
|
||||
|
||||
const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]");
|
||||
expect(series).toHaveLength(2);
|
||||
expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 });
|
||||
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 });
|
||||
expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "false");
|
||||
|
||||
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
|
||||
expect(slices).toEqual([
|
||||
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
|
||||
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
|
||||
]);
|
||||
});
|
||||
fireEvent.click(getByRole("tab", { name: "Prompt Compression" }));
|
||||
|
||||
it("omits a driver slice when that driver has no savings", () => {
|
||||
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
|
||||
|
||||
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
|
||||
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
|
||||
expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "false");
|
||||
expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "true");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
import React from "react";
|
||||
import { PiggyBank } from "lucide-react";
|
||||
import { Alert, Tabs } from "antd";
|
||||
|
||||
import { AreaChart, DonutChart } from "@/components/shared/charts";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { userDailyActivityCall } from "@/components/networking";
|
||||
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import { all_admin_roles } from "@/utils/roles";
|
||||
import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity";
|
||||
import UsageTab from "./UsageTab";
|
||||
import PromptCompressionTab from "./PromptCompressionTab";
|
||||
import AutorouterTab from "./AutorouterTab";
|
||||
import PromptCachingTab from "./PromptCachingTab";
|
||||
|
||||
interface CostOptimizationViewProps {
|
||||
accessToken: string | null;
|
||||
|
|
@ -18,138 +15,62 @@ interface CostOptimizationViewProps {
|
|||
userRole: string;
|
||||
}
|
||||
|
||||
type DateRange = { from?: Date; to?: Date };
|
||||
|
||||
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const usd = (value: number): string => {
|
||||
const decimals = value > 0 && value < 1 ? 4 : 2;
|
||||
return `$${formatNumberWithCommas(value, decimals)}`;
|
||||
};
|
||||
|
||||
const shortDate = (iso: string): string =>
|
||||
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
|
||||
|
||||
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
|
||||
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
|
||||
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
|
||||
|
||||
const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-2xl font-semibold text-foreground">{value}</p>
|
||||
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
|
||||
const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken, userId, userRole }) => {
|
||||
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
|
||||
const initialTo = useMemo(() => new Date(), []);
|
||||
const [dateValue, setDateValue] = useState<DateRange>({ from: initialFrom, to: initialTo });
|
||||
|
||||
const startTime = dateValue.from ?? null;
|
||||
const endTime = dateValue.to ?? null;
|
||||
const isAdmin = all_admin_roles.includes(userRole);
|
||||
const effectiveUserId = isAdmin ? null : userId;
|
||||
|
||||
const { data, loading, isFetchingMore } = usePaginatedDailyActivity({
|
||||
fetchFn: userDailyActivityCall,
|
||||
args: [accessToken, startTime, endTime, effectiveUserId],
|
||||
enabled: !!accessToken && !!startTime && !!endTime,
|
||||
});
|
||||
|
||||
const results = data.results as DailyData[];
|
||||
|
||||
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
|
||||
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
|
||||
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
|
||||
const totalSaved = compressionTotal + cachingTotal;
|
||||
|
||||
const overTime = useMemo(
|
||||
() =>
|
||||
results.map((d) => ({
|
||||
date: shortDate(d.date),
|
||||
Compression: compressionOf(d.metrics),
|
||||
"Prompt caching": cachingOf(d.metrics),
|
||||
})),
|
||||
[results],
|
||||
);
|
||||
|
||||
const byDriver = useMemo(
|
||||
() =>
|
||||
[
|
||||
{ driver: "Compression", usd: compressionTotal },
|
||||
{ driver: "Prompt caching", usd: cachingTotal },
|
||||
].filter((d) => d.usd > 0),
|
||||
[compressionTotal, cachingTotal],
|
||||
);
|
||||
const items = [
|
||||
{
|
||||
key: "usage",
|
||||
label: "Usage",
|
||||
children: <UsageTab accessToken={accessToken} userId={userId} userRole={userRole} />,
|
||||
},
|
||||
{
|
||||
key: "compression",
|
||||
label: "Prompt Compression",
|
||||
children: <PromptCompressionTab accessToken={accessToken} />,
|
||||
},
|
||||
{
|
||||
key: "autorouter",
|
||||
label: "Autorouter",
|
||||
children: <AutorouterTab accessToken={accessToken} userId={userId} userRole={userRole} />,
|
||||
},
|
||||
{
|
||||
key: "caching",
|
||||
label: "Prompt Caching",
|
||||
children: <PromptCachingTab accessToken={accessToken} />,
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6 p-6">
|
||||
<div className="flex flex-wrap items-start justify-between gap-4">
|
||||
<div>
|
||||
<div className="flex items-center gap-2">
|
||||
<PiggyBank className="size-6 text-emerald-600" strokeWidth={1.75} />
|
||||
<h1 className="text-xl font-semibold text-foreground">Cost Optimization</h1>
|
||||
</div>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Money saved by prompt compression and prompt caching across your requests
|
||||
</p>
|
||||
<div>
|
||||
<div className="flex items-center gap-2">
|
||||
<PiggyBank className="size-6 text-emerald-600" strokeWidth={1.75} />
|
||||
<h1 className="text-xl font-semibold text-foreground">Cost Optimization</h1>
|
||||
</div>
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={(v) => setDateValue(v)} />
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Track and configure the mechanisms that save you money: prompt compression, prompt caching, and auto routing
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
|
||||
<SummaryCard
|
||||
label="Total saved"
|
||||
value={usd(totalSaved)}
|
||||
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
|
||||
/>
|
||||
<SummaryCard
|
||||
label="Compression savings"
|
||||
value={usd(compressionTotal)}
|
||||
hint={`${formatNumberWithCommas(savedTokensTotal)} tokens compressed`}
|
||||
/>
|
||||
<SummaryCard label="Prompt caching savings" value={usd(cachingTotal)} hint="Cache read discount" />
|
||||
</div>
|
||||
<Alert
|
||||
type="info"
|
||||
showIcon
|
||||
message="This is an experimental dashboard"
|
||||
description={
|
||||
<span>
|
||||
Have feedback? Join the discussion{" "}
|
||||
<a
|
||||
href="https://github.com/BerriAI/litellm/discussions/32172"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-600 underline"
|
||||
>
|
||||
here
|
||||
</a>
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
|
||||
<Card className="lg:col-span-2">
|
||||
<CardHeader>
|
||||
<CardTitle>Savings over time</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<AreaChart
|
||||
data={overTime}
|
||||
index="date"
|
||||
categories={["Compression", "Prompt caching"]}
|
||||
colors={["emerald", "blue"]}
|
||||
valueFormatter={usd}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Savings by driver</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<DonutChart
|
||||
className="h-80"
|
||||
data={byDriver}
|
||||
index="driver"
|
||||
category="usd"
|
||||
colors={["emerald", "blue"]}
|
||||
valueFormatter={usd}
|
||||
showLabel
|
||||
label={usd(totalSaved)}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
<Tabs defaultActiveKey="usage" items={items} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
"use client";
|
||||
|
||||
import React, { useCallback, useEffect, useState } from "react";
|
||||
|
||||
import { getGeneralSettingsCall } from "@/components/networking";
|
||||
import NotificationsManager from "@/components/molecules/notifications_manager";
|
||||
import {
|
||||
PromptCachingPanel,
|
||||
generalSettingsItem,
|
||||
} from "@/app/(dashboard)/router-settings/_components/general_settings";
|
||||
|
||||
interface PromptCachingTabProps {
|
||||
accessToken: string | null;
|
||||
}
|
||||
|
||||
const PromptCachingTab: React.FC<PromptCachingTabProps> = ({ accessToken }) => {
|
||||
const [settings, setSettings] = useState<generalSettingsItem[]>([]);
|
||||
|
||||
const loadSettings = useCallback(() => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
getGeneralSettingsCall(accessToken)
|
||||
.then((data: generalSettingsItem[]) => setSettings(data))
|
||||
.catch((error) => {
|
||||
console.error("Failed to load prompt caching settings:", error);
|
||||
NotificationsManager.fromBackend("Failed to load prompt caching settings");
|
||||
});
|
||||
}, [accessToken]);
|
||||
|
||||
useEffect(() => {
|
||||
loadSettings();
|
||||
}, [loadSettings]);
|
||||
|
||||
const handleChange = (fieldName: string, newValue: unknown) => {
|
||||
setSettings((prev) =>
|
||||
prev.map((setting) => (setting.field_name === fieldName ? { ...setting, field_value: newValue } : setting)),
|
||||
);
|
||||
};
|
||||
|
||||
if (!accessToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
<PromptCachingPanel accessToken={accessToken} settings={settings} onChange={handleChange} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PromptCachingTab;
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
"use client";
|
||||
|
||||
import React, { useCallback, useEffect, useState } from "react";
|
||||
import { Button, Form, Input, Switch } from "antd";
|
||||
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { createGuardrailCall, getGuardrailsList } from "@/components/networking";
|
||||
import NotificationsManager from "@/components/molecules/notifications_manager";
|
||||
import {
|
||||
buildCompressionGuardrailPayload,
|
||||
compressionGuardrailsOf,
|
||||
GuardrailListItem,
|
||||
GuardrailListResponse,
|
||||
} from "./helpers";
|
||||
|
||||
interface PromptCompressionTabProps {
|
||||
accessToken: string | null;
|
||||
}
|
||||
|
||||
interface CompressionFormValues {
|
||||
name: string;
|
||||
apiBase: string;
|
||||
defaultOn: boolean;
|
||||
}
|
||||
|
||||
const PromptCompressionTab: React.FC<PromptCompressionTabProps> = ({ accessToken }) => {
|
||||
const [form] = Form.useForm<CompressionFormValues>();
|
||||
const [guardrails, setGuardrails] = useState<GuardrailListItem[]>([]);
|
||||
const [isLoading, setIsLoading] = useState<boolean>(true);
|
||||
const [isSaving, setIsSaving] = useState<boolean>(false);
|
||||
|
||||
const loadGuardrails = useCallback(() => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
getGuardrailsList(accessToken)
|
||||
.then((response) => setGuardrails(compressionGuardrailsOf(response as GuardrailListResponse)))
|
||||
.catch((error) => {
|
||||
console.error("Failed to load compression guardrails:", error);
|
||||
NotificationsManager.fromBackend("Failed to load compression guardrails");
|
||||
})
|
||||
.finally(() => setIsLoading(false));
|
||||
}, [accessToken]);
|
||||
|
||||
useEffect(() => {
|
||||
loadGuardrails();
|
||||
}, [loadGuardrails]);
|
||||
|
||||
const handleAdd = async (values: CompressionFormValues) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
setIsSaving(true);
|
||||
try {
|
||||
await createGuardrailCall(
|
||||
accessToken,
|
||||
buildCompressionGuardrailPayload({
|
||||
name: values.name,
|
||||
apiBase: values.apiBase,
|
||||
defaultOn: values.defaultOn ?? true,
|
||||
}),
|
||||
);
|
||||
NotificationsManager.success("Compression guardrail created");
|
||||
form.resetFields();
|
||||
await loadGuardrails();
|
||||
} catch (error) {
|
||||
console.error("Failed to create compression guardrail:", error);
|
||||
NotificationsManager.fromBackend("Failed to create compression guardrail");
|
||||
} finally {
|
||||
setIsSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Headroom prompt compression</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="mb-4 text-sm text-muted-foreground">
|
||||
Headroom is a native LiteLLM guardrail that compresses your prompts before they reach the model, so you pay
|
||||
for fewer input tokens. The tokens it removes are priced and shown on the Usage tab as compression savings.{" "}
|
||||
<a
|
||||
href="https://docs.litellm.ai/docs/proxy/headroom"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-600 underline"
|
||||
>
|
||||
Headroom setup docs
|
||||
</a>
|
||||
</p>
|
||||
{isLoading && <p className="text-sm text-muted-foreground">Loading...</p>}
|
||||
{!isLoading && guardrails.length === 0 && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
No prompt compression guardrails configured yet. Add one below to start saving on input tokens
|
||||
</p>
|
||||
)}
|
||||
{!isLoading && guardrails.length > 0 && (
|
||||
<ul className="divide-y divide-gray-200">
|
||||
{guardrails.map((guardrail) => (
|
||||
<li key={guardrail.guardrail_id} className="flex items-center justify-between py-3">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-foreground">{guardrail.guardrail_name}</p>
|
||||
<p className="text-xs text-muted-foreground">{guardrail.litellm_params?.api_base ?? ""}</p>
|
||||
</div>
|
||||
<span
|
||||
className={`rounded-full px-2 py-0.5 text-xs font-medium ${
|
||||
guardrail.litellm_params?.default_on
|
||||
? "bg-emerald-100 text-emerald-800"
|
||||
: "bg-gray-100 text-gray-600"
|
||||
}`}
|
||||
>
|
||||
{guardrail.litellm_params?.default_on ? "Always on" : "Opt-in"}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Add Headroom compression guardrail</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<Form
|
||||
form={form}
|
||||
layout="vertical"
|
||||
requiredMark={false}
|
||||
onFinish={handleAdd}
|
||||
initialValues={{ defaultOn: true }}
|
||||
>
|
||||
<Form.Item name="name" label="Name" rules={[{ required: true, message: "Name is required" }]}>
|
||||
<Input placeholder="headroom-compression" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
name="apiBase"
|
||||
label="Headroom API base"
|
||||
tooltip="Base URL of your Headroom compression service (LiteLLM calls its /v1/compress endpoint)"
|
||||
extra="The URL where your Headroom compression service is hosted"
|
||||
rules={[{ required: true, message: "API base is required" }]}
|
||||
>
|
||||
<Input placeholder="https://your-headroom-endpoint" />
|
||||
</Form.Item>
|
||||
<Form.Item name="defaultOn" label="Apply to all requests" valuePropName="checked">
|
||||
<Switch />
|
||||
</Form.Item>
|
||||
<div className="mb-4 rounded-lg border border-yellow-200 bg-yellow-50 p-3">
|
||||
<p className="text-sm text-yellow-800">
|
||||
Applying compression to all requests is available to all users. Enabling it selectively per key or team
|
||||
is a LiteLLM Enterprise feature. Get a trial key{" "}
|
||||
<a
|
||||
href="https://www.litellm.ai/#pricing"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline"
|
||||
>
|
||||
here
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex justify-end">
|
||||
<Button type="primary" htmlType="submit" loading={isSaving}>
|
||||
Add guardrail
|
||||
</Button>
|
||||
</div>
|
||||
</Form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PromptCompressionTab;
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { render } from "@testing-library/react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
|
||||
const mockUsePaginatedDailyActivity = vi.fn();
|
||||
|
||||
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
|
||||
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
userDailyActivityCall: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/advanced_date_picker", () => ({
|
||||
__esModule: true,
|
||||
default: () => <div data-testid="date-picker" />,
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/charts", () => ({
|
||||
AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => (
|
||||
<div data-testid="area-chart" data-categories={categories.join(",")} data-series={JSON.stringify(data)} />
|
||||
),
|
||||
DonutChart: ({ data, label }: { data: unknown; label: string }) => (
|
||||
<div data-testid="donut-chart" data-label={label} data-slices={JSON.stringify(data)} />
|
||||
),
|
||||
}));
|
||||
|
||||
import UsageTab from "./UsageTab";
|
||||
|
||||
const baseMetrics = (overrides: Partial<SpendMetrics>): SpendMetrics => ({
|
||||
spend: 0,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
api_requests: 0,
|
||||
successful_requests: 0,
|
||||
failed_requests: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
|
||||
date,
|
||||
metrics: baseMetrics(metrics),
|
||||
breakdown: {
|
||||
models: {},
|
||||
model_groups: {},
|
||||
mcp_servers: {},
|
||||
providers: {},
|
||||
api_keys: {},
|
||||
entities: {},
|
||||
},
|
||||
});
|
||||
|
||||
const renderWith = (results: DailyData[]) => {
|
||||
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
|
||||
return render(<UsageTab accessToken="test-token" userId="u1" userRole="proxy_admin" />);
|
||||
};
|
||||
|
||||
describe("UsageTab", () => {
|
||||
it("sums compression and caching dollars across days into the summary cards", () => {
|
||||
const { getByText } = renderWith([
|
||||
day("2026-07-12", {
|
||||
compression_savings_spend: 0.04,
|
||||
prompt_caching_savings_spend: 0.006,
|
||||
compression_saved_tokens: 40000,
|
||||
}),
|
||||
day("2026-07-13", {
|
||||
compression_savings_spend: 0.1,
|
||||
prompt_caching_savings_spend: 0.01,
|
||||
compression_saved_tokens: 100000,
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(getByText("$0.1560")).toBeInTheDocument();
|
||||
expect(getByText("$0.1400")).toBeInTheDocument();
|
||||
expect(getByText("$0.0160")).toBeInTheDocument();
|
||||
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("builds a per-day time series and per-driver donut from the daily rows", () => {
|
||||
const { getByTestId } = renderWith([
|
||||
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
|
||||
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
|
||||
]);
|
||||
|
||||
const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]");
|
||||
expect(series).toHaveLength(2);
|
||||
expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 });
|
||||
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 });
|
||||
|
||||
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
|
||||
expect(slices).toEqual([
|
||||
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
|
||||
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits a driver slice when that driver has no savings", () => {
|
||||
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
|
||||
|
||||
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
|
||||
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { Collapse } from "antd";
|
||||
|
||||
import { AreaChart, DonutChart } from "@/components/shared/charts";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { userDailyActivityCall } from "@/components/networking";
|
||||
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import { all_admin_roles } from "@/utils/roles";
|
||||
import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity";
|
||||
|
||||
interface UsageTabProps {
|
||||
accessToken: string | null;
|
||||
userId: string | null;
|
||||
userRole: string;
|
||||
}
|
||||
|
||||
type DateRange = { from?: Date; to?: Date };
|
||||
|
||||
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const usd = (value: number): string => {
|
||||
const decimals = value > 0 && value < 1 ? 4 : 2;
|
||||
return `$${formatNumberWithCommas(value, decimals)}`;
|
||||
};
|
||||
|
||||
const shortDate = (iso: string): string =>
|
||||
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
|
||||
|
||||
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
|
||||
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
|
||||
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
|
||||
|
||||
const MethodologyNote = () => (
|
||||
<Collapse
|
||||
ghost
|
||||
items={[
|
||||
{
|
||||
key: "methodology",
|
||||
label: <span className="text-sm font-medium">How savings are calculated</span>,
|
||||
children: (
|
||||
<div className="space-y-3 text-sm text-muted-foreground">
|
||||
<p>
|
||||
Savings are computed for each request when it is logged, using the provider's reported usage and the
|
||||
model's pricing, then summed into a daily rollup. Totals below are read from that rollup over the
|
||||
selected date range, so the numbers never require a scan of raw request logs.
|
||||
</p>
|
||||
<p>
|
||||
Compression savings are the tokens Headroom removed before the call, priced at the model's input
|
||||
rate: <code>compression_saved_tokens * input_cost_per_token</code>
|
||||
</p>
|
||||
<p>
|
||||
Prompt caching savings are the tokens the provider served from cache (Anthropic{" "}
|
||||
<code>cache_read_input_tokens</code>, or OpenAI-style <code>prompt_tokens_details.cached_tokens</code>),
|
||||
priced at the discount between the normal input rate and the cache-read rate:{" "}
|
||||
<code>cache_read_input_tokens * max(input_cost_per_token - cache_read_input_token_cost, 0)</code>
|
||||
</p>
|
||||
<p>
|
||||
Total saved is the sum of both drivers. Models without a separate cache-read price in the pricing map
|
||||
contribute zero caching savings rather than erroring.
|
||||
</p>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
]}
|
||||
/>
|
||||
);
|
||||
|
||||
const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-2xl font-semibold text-foreground">{value}</p>
|
||||
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
|
||||
const UsageTab: React.FC<UsageTabProps> = ({ accessToken, userId, userRole }) => {
|
||||
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
|
||||
const initialTo = useMemo(() => new Date(), []);
|
||||
const [dateValue, setDateValue] = useState<DateRange>({ from: initialFrom, to: initialTo });
|
||||
|
||||
const startTime = dateValue.from ?? null;
|
||||
const endTime = dateValue.to ?? null;
|
||||
const isAdmin = all_admin_roles.includes(userRole);
|
||||
const effectiveUserId = isAdmin ? null : userId;
|
||||
|
||||
const { data, loading, isFetchingMore } = usePaginatedDailyActivity({
|
||||
fetchFn: userDailyActivityCall,
|
||||
args: [accessToken, startTime, endTime, effectiveUserId],
|
||||
enabled: !!accessToken && !!startTime && !!endTime,
|
||||
});
|
||||
|
||||
const results = data.results as DailyData[];
|
||||
|
||||
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
|
||||
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
|
||||
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
|
||||
const totalSaved = compressionTotal + cachingTotal;
|
||||
|
||||
const overTime = useMemo(
|
||||
() =>
|
||||
results.map((d) => ({
|
||||
date: shortDate(d.date),
|
||||
Compression: compressionOf(d.metrics),
|
||||
"Prompt caching": cachingOf(d.metrics),
|
||||
})),
|
||||
[results],
|
||||
);
|
||||
|
||||
const byDriver = useMemo(
|
||||
() =>
|
||||
[
|
||||
{ driver: "Compression", usd: compressionTotal },
|
||||
{ driver: "Prompt caching", usd: cachingTotal },
|
||||
].filter((d) => d.usd > 0),
|
||||
[compressionTotal, cachingTotal],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6">
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
<MethodologyNote />
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={(v) => setDateValue(v)} />
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
|
||||
<SummaryCard
|
||||
label="Total saved"
|
||||
value={usd(totalSaved)}
|
||||
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
|
||||
/>
|
||||
<SummaryCard
|
||||
label="Compression savings"
|
||||
value={usd(compressionTotal)}
|
||||
hint={`${formatNumberWithCommas(savedTokensTotal)} tokens compressed`}
|
||||
/>
|
||||
<SummaryCard label="Prompt caching savings" value={usd(cachingTotal)} hint="Cache read discount" />
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
|
||||
<Card className="lg:col-span-2">
|
||||
<CardHeader>
|
||||
<CardTitle>Savings over time</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<AreaChart
|
||||
data={overTime}
|
||||
index="date"
|
||||
categories={["Compression", "Prompt caching"]}
|
||||
colors={["emerald", "blue"]}
|
||||
valueFormatter={usd}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Savings by driver</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<DonutChart
|
||||
className="h-80"
|
||||
data={byDriver}
|
||||
index="driver"
|
||||
category="usd"
|
||||
colors={["emerald", "blue"]}
|
||||
valueFormatter={usd}
|
||||
showLabel
|
||||
label={usd(totalSaved)}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default UsageTab;
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildCompressionGuardrailPayload, compressionGuardrailsOf } from "./helpers";
|
||||
|
||||
describe("compressionGuardrailsOf", () => {
|
||||
it("keeps only headroom-provider guardrails and drops others", () => {
|
||||
const filtered = compressionGuardrailsOf({
|
||||
guardrails: [
|
||||
{ guardrail_id: "1", guardrail_name: "headroom-compression", litellm_params: { guardrail: "headroom" } },
|
||||
{ guardrail_id: "2", guardrail_name: "pii-masker", litellm_params: { guardrail: "presidio" } },
|
||||
{ guardrail_id: "3", guardrail_name: "no-params", litellm_params: null },
|
||||
],
|
||||
});
|
||||
|
||||
expect(filtered.map((g) => g.guardrail_id)).toEqual(["1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCompressionGuardrailPayload", () => {
|
||||
it("builds a headroom guardrail payload with trimmed fields", () => {
|
||||
const payload = buildCompressionGuardrailPayload({
|
||||
name: " headroom-compression ",
|
||||
apiBase: " https://compress ",
|
||||
defaultOn: false,
|
||||
});
|
||||
|
||||
expect(payload).toEqual({
|
||||
guardrail_name: "headroom-compression",
|
||||
litellm_params: {
|
||||
guardrail: "headroom",
|
||||
mode: "pre_call",
|
||||
api_base: "https://compress",
|
||||
default_on: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
export interface GuardrailLitellmParams {
|
||||
guardrail?: string | null;
|
||||
api_base?: string | null;
|
||||
default_on?: boolean | null;
|
||||
}
|
||||
|
||||
export interface GuardrailListItem {
|
||||
guardrail_id: string;
|
||||
guardrail_name: string | null;
|
||||
litellm_params?: GuardrailLitellmParams | null;
|
||||
}
|
||||
|
||||
export interface GuardrailListResponse {
|
||||
guardrails?: GuardrailListItem[];
|
||||
}
|
||||
|
||||
export const COMPRESSION_GUARDRAIL_PROVIDER = "headroom";
|
||||
|
||||
export const isCompressionGuardrail = (guardrail: GuardrailListItem): boolean =>
|
||||
(guardrail.litellm_params?.guardrail ?? "").toLowerCase() === COMPRESSION_GUARDRAIL_PROVIDER;
|
||||
|
||||
export const compressionGuardrailsOf = (response: GuardrailListResponse): GuardrailListItem[] =>
|
||||
(response.guardrails ?? []).filter(isCompressionGuardrail);
|
||||
|
||||
export interface CompressionGuardrailInput {
|
||||
name: string;
|
||||
apiBase: string;
|
||||
defaultOn: boolean;
|
||||
}
|
||||
|
||||
export const buildCompressionGuardrailPayload = (input: CompressionGuardrailInput): Record<string, unknown> => ({
|
||||
guardrail_name: input.name.trim(),
|
||||
litellm_params: {
|
||||
guardrail: COMPRESSION_GUARDRAIL_PROVIDER,
|
||||
mode: "pre_call",
|
||||
api_base: input.apiBase.trim(),
|
||||
default_on: input.defaultOn,
|
||||
},
|
||||
});
|
||||
|
|
@ -33,7 +33,7 @@ interface GeneralSettingsPageProps {
|
|||
userID: string | null;
|
||||
}
|
||||
|
||||
interface generalSettingsItem {
|
||||
export interface generalSettingsItem {
|
||||
field_name: string;
|
||||
field_type: string;
|
||||
field_value: any;
|
||||
|
|
@ -90,7 +90,7 @@ const SettingValueEditor: React.FC<{
|
|||
return null;
|
||||
};
|
||||
|
||||
const PromptCachingPanel: React.FC<{
|
||||
export const PromptCachingPanel: React.FC<{
|
||||
accessToken: string;
|
||||
settings: generalSettingsItem[];
|
||||
onChange: (fieldName: string, newValue: any) => void;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue