scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 48

merge: resolve cost optimization tab refactor

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-07-20 15:53:52 +00:00
commit 95802957b6
16 changed files with 1168 additions and 601 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -1,8 +1,8 @@
import base64
import json # <--- NEW
import json
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
@ -25,6 +25,8 @@ else:
LANGFUSE_CLOUD_EU_ENDPOINT = "https://cloud.langfuse.com/api/public/otel"
LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel"
LANGFUSE_INGESTION_VERSION_HEADER = "x-langfuse-ingestion-version"
LANGFUSE_INGESTION_VERSION = "4"
class LangfuseOtelLogger(OpenTelemetry):
@ -326,7 +328,9 @@ class LangfuseOtelLogger(OpenTelemetry):
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=f"Authorization={auth_header}",
headers=LangfuseOtelLogger._format_otel_headers(
LangfuseOtelLogger._build_langfuse_otel_headers(auth_header)
),
)
@staticmethod
@ -338,6 +342,26 @@ class LangfuseOtelLogger(OpenTelemetry):
auth_header = base64.b64encode(auth_string.encode()).decode()
return f"Basic {auth_header}"
@staticmethod
def _build_langfuse_otel_headers(auth_header: str) -> Dict[str, str]:
"""
Build the OTLP header set Langfuse expects.
`x-langfuse-ingestion-version: 4` selects Langfuse's v4 ingestion path;
without it spans fall back to the older transformation path.
"""
return {
"Authorization": auth_header,
LANGFUSE_INGESTION_VERSION_HEADER: LANGFUSE_INGESTION_VERSION,
}
@staticmethod
def _format_otel_headers(headers: Dict[str, str]) -> str:
"""
Serialize a header mapping into the comma-separated OTLP header string
"""
return ",".join(f"{key}={value}" for key, value in headers.items())
def construct_dynamic_otel_headers(
self, standard_callback_dynamic_params: StandardCallbackDynamicParams
) -> Optional[dict]:
@ -358,7 +382,7 @@ class LangfuseOtelLogger(OpenTelemetry):
public_key=dynamic_langfuse_public_key,
secret_key=dynamic_langfuse_secret_key,
)
dynamic_headers["Authorization"] = auth_header
dynamic_headers.update(LangfuseOtelLogger._build_langfuse_otel_headers(auth_header))
return dynamic_headers

35
tests/e2e/conftest.py
View file

@ -14,14 +14,14 @@ shared fixtures build on it.
"""
import functools
import sys
import os
from collections.abc import Iterator
from pathlib import Path
import pytest
import requests
from e2e_config import CONTROL_PLANE_BASE_URL, PROXY_BASE_URL
from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup
from junit_properties import attach_result_properties
from lifecycle import ProxyClientProvider, ResourceManager
from proxy_client import ProxyClient, build_proxy_client
@ -107,26 +107,17 @@ def pytest_runtest_call(item: pytest.Item) -> None:
def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
"""Once the whole e2e session is done (all suites), truncate the spend logs so
the DB doesn't accumulate test rows. Sessions where no e2e test body ran leave
the DB alone so a `DATABASE_URL` pointing at a shared instance is never wiped
without an e2e run. Best-effort: a cleanup failure (no DB reachable) must not
fail the run. The spend_tracking dir goes on sys.path only for this import and
is removed after, so a broader `pytest tests/` run is not left with a mutated
path."""
if not session.stash.get(_E2E_TEST_RAN, False):
return
spend_dir = str(Path(__file__).parent / "quota_management" / "spend_tracking")
sys.path.insert(0, spend_dir)
try:
from spend_e2e_client import reset_spend_logs # pyright: ignore
reset_spend_logs()
except Exception as exc: # noqa: BLE001 - cleanup is best-effort
print(f"spend-log cleanup best-effort failed: {exc}")
finally:
if spend_dir in sys.path:
sys.path.remove(spend_dir)
"""Once the whole e2e session is done (all suites), optionally truncate the
spend logs so the DB doesn't accumulate test rows. The truncate is destructive
and irreversible, so it runs only when the operator explicitly opts in
(`E2E_RESET_SPEND_LOGS=1`) and an e2e test body actually ran; otherwise a
`DATABASE_URL` pointing at a shared or staging instance is left untouched.
Best-effort: a cleanup failure (no DB reachable) must not fail the run."""
run_spend_log_cleanup(
opt_in=os.environ.get(RESET_OPT_IN_ENV),
e2e_test_ran=session.stash.get(_E2E_TEST_RAN, False),
truncate=reset_spend_logs,
)
@pytest.fixture(scope="session")

56
tests/e2e/e2e_db.py Normal file
View file

@ -0,0 +1,56 @@
"""Shared, destructive DB helpers for the e2e harness.
Kept at the top level next to e2e_config and lifecycle so every suite imports it
by name (`from e2e_db import ...`); no suite reaches into another's directory by
mutating sys.path.
reset_spend_logs truncates LiteLLM_SpendLogs and cannot be undone, so the
session-finish cleanup routes through run_spend_log_cleanup, which fires the
truncate only on an explicit operator opt-in. "An e2e test ran" is necessary but
never sufficient: a DATABASE_URL pointing at a shared or staging instance must
not be wiped by a routine local run that merely exercised a test.
"""
import os
from collections.abc import Callable
RESET_OPT_IN_ENV = "E2E_RESET_SPEND_LOGS"
def run_spend_log_cleanup(
*, opt_in: str | None, e2e_test_ran: bool, truncate: Callable[[], None]
) -> bool:
"""Invoke `truncate` iff the destructive spend-log reset is both opted into
and warranted, returning whether the truncate was attempted.
The truncate fires only when the opt-in value is exactly "1" AND an e2e test
body actually ran. Any other opt-in value (unset, "0", "true", "") leaves the
DB untouched, so the destructive path is never armed by the env var's mere
presence or by a test run on its own. Best-effort: a truncate failure is
swallowed so cleanup never fails the session, so the returned bool reports
that the reset was attempted, not that the DB call succeeded.
"""
if opt_in != "1" or not e2e_test_ran:
return False
try:
truncate()
except Exception as exc: # noqa: BLE001 - cleanup is best-effort
print(f"spend-log cleanup best-effort failed: {exc}")
return True
def reset_spend_logs() -> None:
"""Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes
spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses
DATABASE_URL (default: the local docker postgres on its mapped host port; the
in-container `@db` host isn't resolvable from the host, so default to
localhost).
"""
import psycopg
url = os.environ.get(
"DATABASE_URL",
"postgresql://llmproxy:dbpassword9090@localhost:5432/litellm",
)
with psycopg.connect(url) as conn:
_ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"')

19
tests/e2e/quota_management/spend_tracking/spend_e2e_client.py
View file

@ -11,7 +11,6 @@ helpers from one place.
from __future__ import annotations
import os
import time
from collections.abc import Callable
from dataclasses import dataclass
@ -50,7 +49,6 @@ from models import (
__all__ = [
"SpendClient",
"build_client",
"reset_spend_logs",
"unique_marker",
"unwrap",
"is_ok",
@ -59,23 +57,6 @@ __all__ = [
]
def reset_spend_logs() -> None:
"""Truncate LiteLLM_SpendLogs for a clean slate. No proxy endpoint deletes
spend logs (/global/spend/reset keeps them), so go to the DB directly. Uses
DATABASE_URL (default: the local docker postgres on its mapped host port; note
the in-container `@db` host isn't resolvable from the host, so default to
localhost).
"""
import psycopg
url = os.environ.get(
"DATABASE_URL",
"postgresql://llmproxy:dbpassword9090@localhost:5432/litellm",
)
with psycopg.connect(url) as conn:
_ = conn.execute('TRUNCATE TABLE "LiteLLM_SpendLogs"')
def _chat_body(
model: str,
content: str,

12
tests/test_litellm/integrations/test_langfuse_otel.py
View file

@ -456,7 +456,7 @@ class TestLangfuseOtelKeyDynamicConfig:
import base64
expected_auth = base64.b64encode(b"key_public:key_secret").decode()
assert config.headers == f"Authorization=Basic {expected_auth}"
assert config.headers == f"Authorization=Basic {expected_auth},x-langfuse-ingestion-version=4"
def test_construct_dynamic_otel_config_host_without_protocol(self):
with self._clean_env():
@ -521,7 +521,10 @@ class TestLangfuseOtelKeyDynamicConfig:
import base64
expected_auth = base64.b64encode(b"key_public:key_secret").decode()
assert exporter._headers == {"Authorization": f"Basic {expected_auth}"}
assert exporter._headers == {
"Authorization": f"Basic {expected_auth}",
"x-langfuse-ingestion-version": "4",
}
def test_key_dynamic_params_reuse_cached_provider(self):
with self._clean_env():
@ -574,7 +577,10 @@ class TestLangfuseOtelKeyDynamicConfig:
provider = next(iter(logger._tracer_provider_cache.values()))
exporter = provider._active_span_processor._span_processors[0].span_exporter
assert isinstance(exporter, OTLPSpanExporter)
assert exporter._headers == {"Authorization": f"Basic {secret}"}
assert exporter._headers == {
"Authorization": f"Basic {secret}",
"x-langfuse-ingestion-version": "4",
}
class TestLangfuseOtelResponsesAPI:

39
tests/test_service_logger_otel.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.types.services import ServiceTypes
from litellm._service_logger import ServiceLogging
from litellm.types.utils import StandardCallbackDynamicParams
class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase):
@ -108,6 +109,44 @@ class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase):
"Generic OTEL logger should have received the log exactly once.",
)
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing")
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics")
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs")
async def test_langfuse_otel_env_config_includes_v4_ingestion_header(
self, mock_logs, mock_metrics, mock_tracing
):
logger = LangfuseOtelLogger()
headers = OpenTelemetry._get_headers_dictionary(logger.config.headers)
self.assertEqual(
headers["x-langfuse-ingestion-version"],
"4",
)
self.assertTrue(headers["Authorization"].startswith("Basic "))
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing")
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics")
@patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs")
async def test_langfuse_otel_dynamic_headers_include_v4_ingestion_header(
self, mock_logs, mock_metrics, mock_tracing
):
logger = LangfuseOtelLogger()
headers = logger.construct_dynamic_otel_headers(
StandardCallbackDynamicParams(
langfuse_public_key="pk-lf-dynamic",
langfuse_secret_key="sk-lf-dynamic",
)
)
self.assertIsNotNone(headers)
self.assertEqual(
headers["x-langfuse-ingestion-version"],
"4",
)
self.assertTrue(headers["Authorization"].startswith("Basic "))
if __name__ == "__main__":
unittest.main()

28
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx Normal file
View file

@ -0,0 +1,28 @@
"use client";
import React from "react";
import { Form } from "antd";
import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab";
interface AutorouterTabProps {
accessToken: string | null;
userId: string | null;
userRole: string;
}
const AutorouterTab: React.FC<AutorouterTabProps> = ({ accessToken, userRole }) => {
const [form] = Form.useForm();
if (!accessToken) {
return null;
}
return (
<div className="w-full">
<AddAutoRouterTab form={form} handleOk={() => form.resetFields()} accessToken={accessToken} userRole={userRole} />
</div>
);
};
export default AutorouterTab;

209
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx
View file

@ -1,207 +1,34 @@
import { fireEvent, render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
const mockUsePaginatedDailyActivity = vi.fn();
const mockUseQuery = vi.fn();
const mockUiSpendLogsCall = vi.fn();
vi.mock("@tanstack/react-query", () => ({
useQuery: (args: unknown) => mockUseQuery(args),
}));
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
}));
vi.mock("@/components/networking", () => ({
userDailyActivityCall: vi.fn(),
getCostOptimizationUsageLogs: vi.fn(),
uiSpendLogsCall: (args: unknown) => mockUiSpendLogsCall(args),
}));
vi.mock("@/components/view_logs/LogDetailsDrawer", () => ({
LogDetailsDrawer: ({ open, logEntry }: { open: boolean; logEntry: { request_id: string } | null }) =>
open && logEntry ? <div data-testid="log-details-drawer">{logEntry.request_id}</div> : null,
}));
vi.mock("@/components/shared/advanced_date_picker", () => ({
__esModule: true,
default: () => <div data-testid="date-picker" />,
}));
vi.mock("@/components/shared/charts", () => ({
AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => (
<div data-testid="area-chart" data-categories={categories.join(",")} data-series={JSON.stringify(data)} />
),
DonutChart: ({ data, label }: { data: unknown; label: string }) => (
<div data-testid="donut-chart" data-label={label} data-slices={JSON.stringify(data)} />
),
}));
vi.mock("./UsageTab", () => ({ __esModule: true, default: () => <div data-testid="usage-tab" /> }));
vi.mock("./PromptCompressionTab", () => ({ __esModule: true, default: () => <div data-testid="compression-tab" /> }));
vi.mock("./AutorouterTab", () => ({ __esModule: true, default: () => <div data-testid="autorouter-tab" /> }));
vi.mock("./PromptCachingTab", () => ({ __esModule: true, default: () => <div data-testid="caching-tab" /> }));
import CostOptimizationView from "./CostOptimizationView";
const baseMetrics = (overrides: Partial<SpendMetrics>): SpendMetrics => ({
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
...overrides,
});
const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
date,
metrics: baseMetrics(metrics),
breakdown: {
models: {},
model_groups: {},
mcp_servers: {},
providers: {},
api_keys: {},
entities: {},
},
});
const detailLog = {
request_id: "req-123456789",
api_key: "key",
team_id: "team",
model: "test-model",
model_id: "model-id",
call_type: "completion",
spend: 0.02,
total_tokens: 150,
prompt_tokens: 100,
completion_tokens: 50,
startTime: "2026-07-13T12:00:00Z",
endTime: "2026-07-13T12:00:01Z",
messages: [],
response: {},
cache_hit: "",
metadata: {},
};
const renderWith = (results: DailyData[]) => {
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
mockUseQuery.mockImplementation((args: { queryKey: string[] }) =>
args.queryKey[0] === "cost-optimization-usage-logs"
? {
data: {
logs: [
{
request_id: "req-123456789",
timestamp: "2026-07-13T12:00:00Z",
model: "test-model",
total_tokens: 150,
optimization_type: "both",
spend: 0.02,
savings: 0.14,
original_cost: 0.16,
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.04,
tokens_saved: 100,
cache_read_tokens: 50,
},
],
total: 1,
page: 1,
page_size: 50,
total_pages: 1,
},
isLoading: false,
isFetching: false,
error: null,
}
: {
data: { data: [detailLog], total: 1 },
isLoading: false,
isFetching: false,
error: null,
},
);
return render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
};
const renderView = () => render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
describe("CostOptimizationView", () => {
it("sums compression and caching dollars across days into the summary cards", () => {
const { getByText } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
compression_saved_tokens: 40000,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
compression_saved_tokens: 100000,
}),
]);
it("renders all four cost-optimization tabs", () => {
const { getByText } = renderView();
// compression 0.14 + caching 0.016 = 0.156
expect(getByText("$0.1560")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
expect(getByText("$0.0160")).toBeInTheDocument();
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
expect(getByText("Usage")).toBeInTheDocument();
expect(getByText("Prompt Compression")).toBeInTheDocument();
expect(getByText("Autorouter")).toBeInTheDocument();
expect(getByText("Prompt Caching")).toBeInTheDocument();
});
it("builds a per-day time series and per-driver donut from the daily rows", () => {
const { getByTestId } = renderWith([
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
]);
it("defaults to the Usage tab and switches the active tab on click", () => {
const { getByRole } = renderView();
const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]");
expect(series).toHaveLength(2);
expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 });
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 });
expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "true");
expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "false");
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
]);
});
fireEvent.click(getByRole("tab", { name: "Prompt Compression" }));
it("omits a driver slice when that driver has no savings", () => {
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
});
it("renders recent optimized requests with savings and optimization type", () => {
const { getByText } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
expect(getByText("Recent Optimized Requests")).toBeInTheDocument();
expect(getByText("req-123456789")).toBeInTheDocument();
expect(getByText("Both")).toBeInTheDocument();
expect(getByText("$0.1600")).toBeInTheDocument();
expect(getByText("$0.0200")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
});
it("fetches and opens request details when an optimized request is clicked", async () => {
const { getByText, getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
fireEvent.click(getByText("req-123456789"));
const detailQuery = mockUseQuery.mock.calls
.map(([args]) => args as { queryKey: string[]; queryFn: () => Promise<unknown> })
.find((args) => args.queryKey[0] === "cost-optimization-spend-log");
expect(detailQuery).toBeDefined();
await detailQuery?.queryFn();
expect(mockUiSpendLogsCall).toHaveBeenCalledWith(
expect.objectContaining({
accessToken: "test-token",
params: { request_id: "req-123456789" },
}),
);
expect(getByTestId("log-details-drawer")).toHaveTextContent("req-123456789");
expect(getByRole("tab", { name: "Usage" })).toHaveAttribute("aria-selected", "false");
expect(getByRole("tab", { name: "Prompt Compression" })).toHaveAttribute("aria-selected", "true");
});
});

413
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx
View file

@ -1,26 +1,13 @@
"use client";
import React, { useMemo, useState } from "react";
import React from "react";
import { PiggyBank } from "lucide-react";
import { useQuery } from "@tanstack/react-query";
import moment from "moment";
import { Alert, Tabs } from "antd";
import { AreaChart, DonutChart } from "@/components/shared/charts";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import {
getCostOptimizationUsageLogs,
uiSpendLogsCall,
type OptimizedRequestLog,
type OptimizedRequestLogsResponse,
userDailyActivityCall,
} from "@/components/networking";
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
import { formatNumberWithCommas } from "@/utils/dataUtils";
import { all_admin_roles } from "@/utils/roles";
import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity";
import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer";
import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns";
import UsageTab from "./UsageTab";
import PromptCompressionTab from "./PromptCompressionTab";
import AutorouterTab from "./AutorouterTab";
import PromptCachingTab from "./PromptCachingTab";
interface CostOptimizationViewProps {
accessToken: string | null;
@ -28,356 +15,62 @@ interface CostOptimizationViewProps {
userRole: string;
}
type DateRange = { from?: Date; to?: Date };
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
const usd = (value: number): string => {
const decimals = value > 0 && value < 1 ? 4 : 2;
return `$${formatNumberWithCommas(value, decimals)}`;
};
const shortDate = (iso: string): string =>
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => (
<Card>
<CardHeader>
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
</CardHeader>
<CardContent>
<p className="text-2xl font-semibold text-foreground">{value}</p>
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
</CardContent>
</Card>
);
const optimizationTypeLabel = (type: OptimizedRequestLog["optimization_type"]): string => {
if (type === "both") return "Both";
if (type === "caching") return "Caching";
return "Compression";
};
const optimizationTypeClass = (type: OptimizedRequestLog["optimization_type"]): string => {
if (type === "both") return "bg-purple-50 text-purple-700 border-purple-200";
if (type === "caching") return "bg-green-50 text-green-700 border-green-200";
return "bg-blue-50 text-blue-700 border-blue-200";
};
const optimizedLogsDescription = (
loading: boolean,
fetching: boolean,
error: unknown,
data: OptimizedRequestLogsResponse | undefined,
): string => {
if (loading || fetching) return "Loading...";
if (error) return "Failed to load optimized requests";
if (data?.total) return `Showing ${data.logs.length} of ${data.total} requests`;
return "No optimized requests for this period";
};
const OptimizedRequestsTable = ({
data,
logsPage,
onPrevious,
onNext,
onRowClick,
}: {
data: OptimizedRequestLogsResponse;
logsPage: number;
onPrevious: () => void;
onNext: () => void;
onRowClick: (log: OptimizedRequestLog) => void;
}) => (
<>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="border-b text-left text-xs text-muted-foreground">
<th className="px-3 py-3 font-medium">Request ID</th>
<th className="px-3 py-3 font-medium">Timestamp</th>
<th className="px-3 py-3 font-medium">Model</th>
<th className="px-3 py-3 text-right font-medium">Tokens</th>
<th className="px-3 py-3 font-medium">Type</th>
<th className="px-3 py-3 text-right font-medium">Original Cost</th>
<th className="px-3 py-3 text-right font-medium">Optimized Cost</th>
<th className="px-3 py-3 text-right font-medium">Savings</th>
</tr>
</thead>
<tbody>
{data.logs.map((log) => (
<tr
key={log.request_id}
className="cursor-pointer border-b transition-colors hover:bg-muted/50 last:border-0"
onClick={() => onRowClick(log)}
onKeyDown={(event) => {
if (event.key === "Enter" || event.key === " ") {
event.preventDefault();
onRowClick(log);
}
}}
role="button"
tabIndex={0}
>
<td className="max-w-40 truncate px-3 py-3 font-mono text-xs" title={log.request_id}>
{log.request_id}
</td>
<td className="whitespace-nowrap px-3 py-3 text-muted-foreground">
{new Date(log.timestamp).toLocaleString()}
</td>
<td className="max-w-48 truncate px-3 py-3" title={log.model}>
{log.model}
</td>
<td className="px-3 py-3 text-right">{formatNumberWithCommas(log.total_tokens)}</td>
<td className="px-3 py-3">
<span
className={`inline-flex rounded border px-2 py-0.5 text-xs font-medium ${optimizationTypeClass(log.optimization_type)}`}
>
{optimizationTypeLabel(log.optimization_type)}
</span>
</td>
<td className="px-3 py-3 text-right text-muted-foreground line-through">{usd(log.original_cost)}</td>
<td className="px-3 py-3 text-right">{usd(log.spend)}</td>
<td className="px-3 py-3 text-right font-medium text-emerald-600">{usd(log.savings)}</td>
</tr>
))}
</tbody>
</table>
</div>
{data.total_pages > 1 && (
<div className="mt-4 flex items-center justify-between">
<p className="text-xs text-muted-foreground">
Page {data.page} of {data.total_pages}
</p>
<div className="flex gap-2">
<button
type="button"
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
disabled={logsPage === 1}
onClick={onPrevious}
>
Previous
</button>
<button
type="button"
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
disabled={logsPage >= data.total_pages}
onClick={onNext}
>
Next
</button>
</div>
</div>
)}
</>
);
const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken, userId, userRole }) => {
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
const initialTo = useMemo(() => new Date(), []);
const [dateValue, setDateValue] = useState<DateRange>({ from: initialFrom, to: initialTo });
const startTime = dateValue.from ?? null;
const endTime = dateValue.to ?? null;
const isAdmin = all_admin_roles.includes(userRole);
const effectiveUserId = isAdmin ? null : userId;
const { data, loading, isFetchingMore } = usePaginatedDailyActivity({
fetchFn: userDailyActivityCall,
args: [accessToken, startTime, endTime, effectiveUserId],
enabled: !!accessToken && !!startTime && !!endTime,
});
const results = data.results as DailyData[];
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
const totalSaved = compressionTotal + cachingTotal;
const [logsPage, setLogsPage] = useState(1);
const startDate = startTime ? startTime.toISOString().slice(0, 10) : "";
const endDate = endTime ? endTime.toISOString().slice(0, 10) : "";
const [selectedRequestId, setSelectedRequestId] = useState<string | null>(null);
const [drawerOpen, setDrawerOpen] = useState(false);
const {
data: optimizedLogsData,
isLoading: optimizedLogsLoading,
isFetching: optimizedLogsFetching,
error: optimizedLogsError,
} = useQuery({
queryKey: ["cost-optimization-usage-logs", startDate, endDate, logsPage],
queryFn: () =>
getCostOptimizationUsageLogs({
accessToken: accessToken!,
startDate,
endDate,
page: logsPage,
pageSize: 50,
}),
enabled: !!accessToken && !!startDate && !!endDate,
});
const logsDescription = optimizedLogsDescription(
optimizedLogsLoading,
optimizedLogsFetching,
optimizedLogsError,
optimizedLogsData,
);
const drawerStartTime = startTime
? moment(startTime).utc().format("YYYY-MM-DD HH:mm:ss")
: moment().subtract(24, "hours").utc().format("YYYY-MM-DD HH:mm:ss");
const drawerEndTime = endTime
? moment(endTime).utc().endOf("day").format("YYYY-MM-DD HH:mm:ss")
: moment().utc().format("YYYY-MM-DD HH:mm:ss");
const { data: fullLogResponse } = useQuery({
queryKey: ["cost-optimization-spend-log", selectedRequestId, drawerStartTime, drawerEndTime],
queryFn: async () => {
if (!accessToken || !selectedRequestId) return null;
const response = await uiSpendLogsCall({
accessToken,
start_date: drawerStartTime,
end_date: drawerEndTime,
page: 1,
page_size: 10,
params: { request_id: selectedRequestId },
});
return response as { data: ViewLogsLogEntry[]; total: number };
const items = [
{
key: "usage",
label: "Usage",
children: <UsageTab accessToken={accessToken} userId={userId} userRole={userRole} />,
},
enabled: Boolean(accessToken && selectedRequestId && drawerOpen),
});
const selectedLog: ViewLogsLogEntry | null = fullLogResponse?.data?.[0] ?? null;
const overTime = useMemo(
() =>
results.map((d) => ({
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
})),
[results],
);
const byDriver = useMemo(
() =>
[
{ driver: "Compression", usd: compressionTotal },
{ driver: "Prompt caching", usd: cachingTotal },
].filter((d) => d.usd > 0),
[compressionTotal, cachingTotal],
);
{
key: "compression",
label: "Prompt Compression",
children: <PromptCompressionTab accessToken={accessToken} />,
},
{
key: "autorouter",
label: "Autorouter",
children: <AutorouterTab accessToken={accessToken} userId={userId} userRole={userRole} />,
},
{
key: "caching",
label: "Prompt Caching",
children: <PromptCachingTab accessToken={accessToken} />,
},
];
return (
<div className="w-full space-y-6 p-6">
<div className="flex flex-wrap items-start justify-between gap-4">
<div>
<div className="flex items-center gap-2">
<PiggyBank className="size-6 text-emerald-600" strokeWidth={1.75} />
<h1 className="text-xl font-semibold text-foreground">Cost Optimization</h1>
</div>
<p className="mt-1 text-sm text-muted-foreground">
Money saved by prompt compression and prompt caching across your requests
</p>
<div>
<div className="flex items-center gap-2">
<PiggyBank className="size-6 text-emerald-600" strokeWidth={1.75} />
<h1 className="text-xl font-semibold text-foreground">Cost Optimization</h1>
</div>
<AdvancedDatePicker
value={dateValue}
onValueChange={(v) => {
setDateValue(v);
setLogsPage(1);
}}
/>
<p className="mt-1 text-sm text-muted-foreground">
Track and configure the mechanisms that save you money: prompt compression, prompt caching, and auto routing
</p>
</div>
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<SummaryCard
label="Total saved"
value={usd(totalSaved)}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
/>
<SummaryCard
label="Compression savings"
value={usd(compressionTotal)}
hint={`${formatNumberWithCommas(savedTokensTotal)} tokens compressed`}
/>
<SummaryCard label="Prompt caching savings" value={usd(cachingTotal)} hint="Cache read discount" />
</div>
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
<Card className="lg:col-span-2">
<CardHeader>
<CardTitle>Savings over time</CardTitle>
</CardHeader>
<CardContent>
<AreaChart
data={overTime}
index="date"
categories={["Compression", "Prompt caching"]}
colors={["emerald", "blue"]}
valueFormatter={usd}
/>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Savings by driver</CardTitle>
</CardHeader>
<CardContent>
<DonutChart
className="h-80"
data={byDriver}
index="driver"
category="usd"
colors={["emerald", "blue"]}
valueFormatter={usd}
showLabel
label={usd(totalSaved)}
/>
</CardContent>
</Card>
</div>
<Card>
<CardHeader>
<CardTitle>Recent Optimized Requests</CardTitle>
<p className="text-sm text-muted-foreground">{logsDescription}</p>
</CardHeader>
<CardContent>
{optimizedLogsLoading && (
<div className="py-10 text-center text-sm text-muted-foreground">Loading optimized requests...</div>
)}
{!optimizedLogsLoading && optimizedLogsError && (
<div className="py-10 text-center text-sm text-red-600">Failed to load optimized requests.</div>
)}
{!optimizedLogsLoading && !optimizedLogsError && optimizedLogsData?.logs.length ? (
<OptimizedRequestsTable
data={optimizedLogsData}
logsPage={logsPage}
onPrevious={() => setLogsPage((page) => Math.max(1, page - 1))}
onNext={() => setLogsPage((page) => Math.min(optimizedLogsData.total_pages, page + 1))}
onRowClick={(log) => {
setSelectedRequestId(log.request_id);
setDrawerOpen(true);
}}
/>
) : null}
{!optimizedLogsLoading && !optimizedLogsError && !optimizedLogsData?.logs.length && (
<div className="py-10 text-center text-sm text-muted-foreground">No optimized requests to display.</div>
)}
</CardContent>
</Card>
<LogDetailsDrawer
open={drawerOpen}
onClose={() => {
setDrawerOpen(false);
setSelectedRequestId(null);
}}
logEntry={selectedLog}
accessToken={accessToken}
allLogs={selectedLog ? [selectedLog] : []}
startTime={drawerStartTime}
<Alert
type="info"
showIcon
message="This is an experimental dashboard"
description={
<span>
Have feedback? Join the discussion{" "}
<a
href="https://github.com/BerriAI/litellm/discussions/32172"
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 underline"
>
here
</a>
</span>
}
/>
<Tabs defaultActiveKey="usage" items={items} />
</div>
);
};

52
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.tsx Normal file
View file

@ -0,0 +1,52 @@
"use client";
import React, { useCallback, useEffect, useState } from "react";
import { getGeneralSettingsCall } from "@/components/networking";
import NotificationsManager from "@/components/molecules/notifications_manager";
import {
PromptCachingPanel,
generalSettingsItem,
} from "@/app/(dashboard)/router-settings/_components/general_settings";
interface PromptCachingTabProps {
accessToken: string | null;
}
const PromptCachingTab: React.FC<PromptCachingTabProps> = ({ accessToken }) => {
const [settings, setSettings] = useState<generalSettingsItem[]>([]);
const loadSettings = useCallback(() => {
if (!accessToken) {
return;
}
getGeneralSettingsCall(accessToken)
.then((data: generalSettingsItem[]) => setSettings(data))
.catch((error) => {
console.error("Failed to load prompt caching settings:", error);
NotificationsManager.fromBackend("Failed to load prompt caching settings");
});
}, [accessToken]);
useEffect(() => {
loadSettings();
}, [loadSettings]);
const handleChange = (fieldName: string, newValue: unknown) => {
setSettings((prev) =>
prev.map((setting) => (setting.field_name === fieldName ? { ...setting, field_value: newValue } : setting)),
);
};
if (!accessToken) {
return null;
}
return (
<div className="w-full">
<PromptCachingPanel accessToken={accessToken} settings={settings} onChange={handleChange} />
</div>
);
};
export default PromptCachingTab;

176
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCompressionTab.tsx Normal file
View file

@ -0,0 +1,176 @@
"use client";
import React, { useCallback, useEffect, useState } from "react";
import { Button, Form, Input, Switch } from "antd";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { createGuardrailCall, getGuardrailsList } from "@/components/networking";
import NotificationsManager from "@/components/molecules/notifications_manager";
import {
buildCompressionGuardrailPayload,
compressionGuardrailsOf,
GuardrailListItem,
GuardrailListResponse,
} from "./helpers";
interface PromptCompressionTabProps {
accessToken: string | null;
}
interface CompressionFormValues {
name: string;
apiBase: string;
defaultOn: boolean;
}
const PromptCompressionTab: React.FC<PromptCompressionTabProps> = ({ accessToken }) => {
const [form] = Form.useForm<CompressionFormValues>();
const [guardrails, setGuardrails] = useState<GuardrailListItem[]>([]);
const [isLoading, setIsLoading] = useState<boolean>(true);
const [isSaving, setIsSaving] = useState<boolean>(false);
const loadGuardrails = useCallback(() => {
if (!accessToken) {
return;
}
getGuardrailsList(accessToken)
.then((response) => setGuardrails(compressionGuardrailsOf(response as GuardrailListResponse)))
.catch((error) => {
console.error("Failed to load compression guardrails:", error);
NotificationsManager.fromBackend("Failed to load compression guardrails");
})
.finally(() => setIsLoading(false));
}, [accessToken]);
useEffect(() => {
loadGuardrails();
}, [loadGuardrails]);
const handleAdd = async (values: CompressionFormValues) => {
if (!accessToken) {
return;
}
setIsSaving(true);
try {
await createGuardrailCall(
accessToken,
buildCompressionGuardrailPayload({
name: values.name,
apiBase: values.apiBase,
defaultOn: values.defaultOn ?? true,
}),
);
NotificationsManager.success("Compression guardrail created");
form.resetFields();
await loadGuardrails();
} catch (error) {
console.error("Failed to create compression guardrail:", error);
NotificationsManager.fromBackend("Failed to create compression guardrail");
} finally {
setIsSaving(false);
}
};
return (
<div className="w-full space-y-6">
<Card>
<CardHeader>
<CardTitle>Headroom prompt compression</CardTitle>
</CardHeader>
<CardContent>
<p className="mb-4 text-sm text-muted-foreground">
Headroom is a native LiteLLM guardrail that compresses your prompts before they reach the model, so you pay
for fewer input tokens. The tokens it removes are priced and shown on the Usage tab as compression savings.{" "}
<a
href="https://docs.litellm.ai/docs/proxy/headroom"
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 underline"
>
Headroom setup docs
</a>
</p>
{isLoading && <p className="text-sm text-muted-foreground">Loading...</p>}
{!isLoading && guardrails.length === 0 && (
<p className="text-sm text-muted-foreground">
No prompt compression guardrails configured yet. Add one below to start saving on input tokens
</p>
)}
{!isLoading && guardrails.length > 0 && (
<ul className="divide-y divide-gray-200">
{guardrails.map((guardrail) => (
<li key={guardrail.guardrail_id} className="flex items-center justify-between py-3">
<div>
<p className="text-sm font-medium text-foreground">{guardrail.guardrail_name}</p>
<p className="text-xs text-muted-foreground">{guardrail.litellm_params?.api_base ?? ""}</p>
</div>
<span
className={`rounded-full px-2 py-0.5 text-xs font-medium ${
guardrail.litellm_params?.default_on
? "bg-emerald-100 text-emerald-800"
: "bg-gray-100 text-gray-600"
}`}
>
{guardrail.litellm_params?.default_on ? "Always on" : "Opt-in"}
</span>
</li>
))}
</ul>
)}
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Add Headroom compression guardrail</CardTitle>
</CardHeader>
<CardContent>
<Form
form={form}
layout="vertical"
requiredMark={false}
onFinish={handleAdd}
initialValues={{ defaultOn: true }}
>
<Form.Item name="name" label="Name" rules={[{ required: true, message: "Name is required" }]}>
<Input placeholder="headroom-compression" />
</Form.Item>
<Form.Item
name="apiBase"
label="Headroom API base"
tooltip="Base URL of your Headroom compression service (LiteLLM calls its /v1/compress endpoint)"
extra="The URL where your Headroom compression service is hosted"
rules={[{ required: true, message: "API base is required" }]}
>
<Input placeholder="https://your-headroom-endpoint" />
</Form.Item>
<Form.Item name="defaultOn" label="Apply to all requests" valuePropName="checked">
<Switch />
</Form.Item>
<div className="mb-4 rounded-lg border border-yellow-200 bg-yellow-50 p-3">
<p className="text-sm text-yellow-800">
Applying compression to all requests is available to all users. Enabling it selectively per key or team
is a LiteLLM Enterprise feature. Get a trial key{" "}
<a
href="https://www.litellm.ai/#pricing"
target="_blank"
rel="noopener noreferrer"
className="underline"
>
here
</a>
</p>
</div>
<div className="flex justify-end">
<Button type="primary" htmlType="submit" loading={isSaving}>
Add guardrail
</Button>
</div>
</Form>
</CardContent>
</Card>
</div>
);
};
export default PromptCompressionTab;

206
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx Normal file
View file

@ -0,0 +1,206 @@
import { fireEvent, render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
const mockUsePaginatedDailyActivity = vi.fn();
const mockUseQuery = vi.fn();
const mockUiSpendLogsCall = vi.fn();
vi.mock("@tanstack/react-query", () => ({
useQuery: (args: unknown) => mockUseQuery(args),
}));
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
}));
vi.mock("@/components/networking", () => ({
userDailyActivityCall: vi.fn(),
getCostOptimizationUsageLogs: vi.fn(),
uiSpendLogsCall: (args: unknown) => mockUiSpendLogsCall(args),
}));
vi.mock("@/components/view_logs/LogDetailsDrawer", () => ({
LogDetailsDrawer: ({ open, logEntry }: { open: boolean; logEntry: { request_id: string } | null }) =>
open && logEntry ? <div data-testid="log-details-drawer">{logEntry.request_id}</div> : null,
}));
vi.mock("@/components/shared/advanced_date_picker", () => ({
__esModule: true,
default: () => <div data-testid="date-picker" />,
}));
vi.mock("@/components/shared/charts", () => ({
AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => (
<div data-testid="area-chart" data-categories={categories.join(",")} data-series={JSON.stringify(data)} />
),
DonutChart: ({ data, label }: { data: unknown; label: string }) => (
<div data-testid="donut-chart" data-label={label} data-slices={JSON.stringify(data)} />
),
}));
import UsageTab from "./UsageTab";
const baseMetrics = (overrides: Partial<SpendMetrics>): SpendMetrics => ({
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
...overrides,
});
const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
date,
metrics: baseMetrics(metrics),
breakdown: {
models: {},
model_groups: {},
mcp_servers: {},
providers: {},
api_keys: {},
entities: {},
},
});
const detailLog = {
request_id: "req-123456789",
api_key: "key",
team_id: "team",
model: "test-model",
model_id: "model-id",
call_type: "completion",
spend: 0.02,
total_tokens: 150,
prompt_tokens: 100,
completion_tokens: 50,
startTime: "2026-07-13T12:00:00Z",
endTime: "2026-07-13T12:00:01Z",
messages: [],
response: {},
cache_hit: "",
metadata: {},
};
const renderWith = (results: DailyData[]) => {
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
mockUseQuery.mockImplementation((args: { queryKey: string[] }) =>
args.queryKey[0] === "cost-optimization-usage-logs"
? {
data: {
logs: [
{
request_id: "req-123456789",
timestamp: "2026-07-13T12:00:00Z",
model: "test-model",
total_tokens: 150,
optimization_type: "both",
spend: 0.02,
savings: 0.14,
original_cost: 0.16,
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.04,
tokens_saved: 100,
cache_read_tokens: 50,
},
],
total: 1,
page: 1,
page_size: 50,
total_pages: 1,
},
isLoading: false,
isFetching: false,
error: null,
}
: {
data: { data: [detailLog], total: 1 },
isLoading: false,
isFetching: false,
error: null,
},
);
return render(<UsageTab accessToken="test-token" userId="u1" userRole="proxy_admin" />);
};
describe("UsageTab", () => {
it("sums compression and caching dollars across days into the summary cards", () => {
const { getByText } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
compression_saved_tokens: 40000,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
compression_saved_tokens: 100000,
}),
]);
expect(getByText("$0.1560")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
expect(getByText("$0.0160")).toBeInTheDocument();
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
});
it("builds a per-day time series and per-driver donut from the daily rows", () => {
const { getByTestId } = renderWith([
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
]);
const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]");
expect(series).toHaveLength(2);
expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 });
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 });
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
]);
});
it("omits a driver slice when that driver has no savings", () => {
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
});
it("renders recent optimized requests with savings and optimization type", () => {
const { getByText } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
expect(getByText("Recent Optimized Requests")).toBeInTheDocument();
expect(getByText("req-123456789")).toBeInTheDocument();
expect(getByText("Both")).toBeInTheDocument();
expect(getByText("$0.1600")).toBeInTheDocument();
expect(getByText("$0.0200")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
});
it("fetches and opens request details when an optimized request is clicked", async () => {
const { getByText, getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
fireEvent.click(getByText("req-123456789"));
const detailQuery = mockUseQuery.mock.calls
.map(([args]) => args as { queryKey: string[]; queryFn: () => Promise<unknown> })
.find((args) => args.queryKey[0] === "cost-optimization-spend-log");
expect(detailQuery).toBeDefined();
await detailQuery?.queryFn();
expect(mockUiSpendLogsCall).toHaveBeenCalledWith(
expect.objectContaining({
accessToken: "test-token",
params: { request_id: "req-123456789" },
}),
);
expect(getByTestId("log-details-drawer")).toHaveTextContent("req-123456789");
});
});

412
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx Normal file
View file

@ -0,0 +1,412 @@
"use client";
import React, { useMemo, useState } from "react";
import { Collapse } from "antd";
import { useQuery } from "@tanstack/react-query";
import moment from "moment";
import { AreaChart, DonutChart } from "@/components/shared/charts";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import {
getCostOptimizationUsageLogs,
uiSpendLogsCall,
type OptimizedRequestLog,
type OptimizedRequestLogsResponse,
userDailyActivityCall,
} from "@/components/networking";
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
import { formatNumberWithCommas } from "@/utils/dataUtils";
import { all_admin_roles } from "@/utils/roles";
import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity";
import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer";
import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns";
interface UsageTabProps {
accessToken: string | null;
userId: string | null;
userRole: string;
}
type DateRange = { from?: Date; to?: Date };
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
const usd = (value: number): string => {
const decimals = value > 0 && value < 1 ? 4 : 2;
return `$${formatNumberWithCommas(value, decimals)}`;
};
const shortDate = (iso: string): string =>
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
const MethodologyNote = () => (
<Collapse
ghost
items={[
{
key: "methodology",
label: <span className="text-sm font-medium">How savings are calculated</span>,
children: (
<div className="space-y-3 text-sm text-muted-foreground">
<p>
Savings are computed for each request when it is logged, using the provider&apos;s reported usage and the
model&apos;s pricing, then summed into a daily rollup. Totals below are read from that rollup over the
selected date range, so the numbers never require a scan of raw request logs.
</p>
<p>
Compression savings are the tokens Headroom removed before the call, priced at the model&apos;s input
rate: <code>compression_saved_tokens * input_cost_per_token</code>
</p>
<p>
Prompt caching savings are the tokens the provider served from cache (Anthropic{" "}
<code>cache_read_input_tokens</code>, or OpenAI-style <code>prompt_tokens_details.cached_tokens</code>),
priced at the discount between the normal input rate and the cache-read rate:{" "}
<code>cache_read_input_tokens * max(input_cost_per_token - cache_read_input_token_cost, 0)</code>
</p>
<p>
Total saved is the sum of both drivers. Models without a separate cache-read price in the pricing map
contribute zero caching savings rather than erroring.
</p>
</div>
),
},
]}
/>
);
const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => (
<Card>
<CardHeader>
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
</CardHeader>
<CardContent>
<p className="text-2xl font-semibold text-foreground">{value}</p>
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
</CardContent>
</Card>
);
const optimizationTypeLabel = (type: OptimizedRequestLog["optimization_type"]): string => {
if (type === "both") return "Both";
if (type === "caching") return "Caching";
return "Compression";
};
const optimizationTypeClass = (type: OptimizedRequestLog["optimization_type"]): string => {
if (type === "both") return "bg-purple-50 text-purple-700 border-purple-200";
if (type === "caching") return "bg-green-50 text-green-700 border-green-200";
return "bg-blue-50 text-blue-700 border-blue-200";
};
const optimizedLogsDescription = (
loading: boolean,
fetching: boolean,
error: unknown,
data: OptimizedRequestLogsResponse | undefined,
): string => {
if (loading || fetching) return "Loading...";
if (error) return "Failed to load optimized requests";
if (data?.total) return `Showing ${data.logs.length} of ${data.total} requests`;
return "No optimized requests for this period";
};
const OptimizedRequestsTable = ({
data,
logsPage,
onPrevious,
onNext,
onRowClick,
}: {
data: OptimizedRequestLogsResponse;
logsPage: number;
onPrevious: () => void;
onNext: () => void;
onRowClick: (log: OptimizedRequestLog) => void;
}) => (
<>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="border-b text-left text-xs text-muted-foreground">
<th className="px-3 py-3 font-medium">Request ID</th>
<th className="px-3 py-3 font-medium">Timestamp</th>
<th className="px-3 py-3 font-medium">Model</th>
<th className="px-3 py-3 text-right font-medium">Tokens</th>
<th className="px-3 py-3 font-medium">Type</th>
<th className="px-3 py-3 text-right font-medium">Original Cost</th>
<th className="px-3 py-3 text-right font-medium">Optimized Cost</th>
<th className="px-3 py-3 text-right font-medium">Savings</th>
</tr>
</thead>
<tbody>
{data.logs.map((log) => (
<tr
key={log.request_id}
className="cursor-pointer border-b transition-colors hover:bg-muted/50 last:border-0"
onClick={() => onRowClick(log)}
onKeyDown={(event) => {
if (event.key === "Enter" || event.key === " ") {
event.preventDefault();
onRowClick(log);
}
}}
role="button"
tabIndex={0}
>
<td className="max-w-40 truncate px-3 py-3 font-mono text-xs" title={log.request_id}>
{log.request_id}
</td>
<td className="whitespace-nowrap px-3 py-3 text-muted-foreground">
{new Date(log.timestamp).toLocaleString()}
</td>
<td className="max-w-48 truncate px-3 py-3" title={log.model}>
{log.model}
</td>
<td className="px-3 py-3 text-right">{formatNumberWithCommas(log.total_tokens)}</td>
<td className="px-3 py-3">
<span
className={`inline-flex rounded border px-2 py-0.5 text-xs font-medium ${optimizationTypeClass(log.optimization_type)}`}
>
{optimizationTypeLabel(log.optimization_type)}
</span>
</td>
<td className="px-3 py-3 text-right text-muted-foreground line-through">{usd(log.original_cost)}</td>
<td className="px-3 py-3 text-right">{usd(log.spend)}</td>
<td className="px-3 py-3 text-right font-medium text-emerald-600">{usd(log.savings)}</td>
</tr>
))}
</tbody>
</table>
</div>
{data.total_pages > 1 && (
<div className="mt-4 flex items-center justify-between">
<p className="text-xs text-muted-foreground">
Page {data.page} of {data.total_pages}
</p>
<div className="flex gap-2">
<button
type="button"
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
disabled={logsPage === 1}
onClick={onPrevious}
>
Previous
</button>
<button
type="button"
className="rounded border px-3 py-1 text-sm disabled:cursor-not-allowed disabled:opacity-50"
disabled={logsPage >= data.total_pages}
onClick={onNext}
>
Next
</button>
</div>
</div>
)}
</>
);
const UsageTab: React.FC<UsageTabProps> = ({ accessToken, userId, userRole }) => {
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
const initialTo = useMemo(() => new Date(), []);
const [dateValue, setDateValue] = useState<DateRange>({ from: initialFrom, to: initialTo });
const startTime = dateValue.from ?? null;
const endTime = dateValue.to ?? null;
const isAdmin = all_admin_roles.includes(userRole);
const effectiveUserId = isAdmin ? null : userId;
const { data, loading, isFetchingMore } = usePaginatedDailyActivity({
fetchFn: userDailyActivityCall,
args: [accessToken, startTime, endTime, effectiveUserId],
enabled: !!accessToken && !!startTime && !!endTime,
});
const results = data.results as DailyData[];
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
const totalSaved = compressionTotal + cachingTotal;
const [logsPage, setLogsPage] = useState(1);
const startDate = startTime ? startTime.toISOString().slice(0, 10) : "";
const endDate = endTime ? endTime.toISOString().slice(0, 10) : "";
const [selectedRequestId, setSelectedRequestId] = useState<string | null>(null);
const [drawerOpen, setDrawerOpen] = useState(false);
const {
data: optimizedLogsData,
isLoading: optimizedLogsLoading,
isFetching: optimizedLogsFetching,
error: optimizedLogsError,
} = useQuery({
queryKey: ["cost-optimization-usage-logs", startDate, endDate, logsPage],
queryFn: () =>
getCostOptimizationUsageLogs({
accessToken: accessToken!,
startDate,
endDate,
page: logsPage,
pageSize: 50,
}),
enabled: !!accessToken && !!startDate && !!endDate,
});
const logsDescription = optimizedLogsDescription(
optimizedLogsLoading,
optimizedLogsFetching,
optimizedLogsError,
optimizedLogsData,
);
const drawerStartTime = startTime
? moment(startTime).utc().format("YYYY-MM-DD HH:mm:ss")
: moment().subtract(24, "hours").utc().format("YYYY-MM-DD HH:mm:ss");
const drawerEndTime = endTime
? moment(endTime).utc().endOf("day").format("YYYY-MM-DD HH:mm:ss")
: moment().utc().format("YYYY-MM-DD HH:mm:ss");
const { data: fullLogResponse } = useQuery({
queryKey: ["cost-optimization-spend-log", selectedRequestId, drawerStartTime, drawerEndTime],
queryFn: async () => {
if (!accessToken || !selectedRequestId) return null;
const response = await uiSpendLogsCall({
accessToken,
start_date: drawerStartTime,
end_date: drawerEndTime,
page: 1,
page_size: 10,
params: { request_id: selectedRequestId },
});
return response as { data: ViewLogsLogEntry[]; total: number };
},
enabled: Boolean(accessToken && selectedRequestId && drawerOpen),
});
const selectedLog: ViewLogsLogEntry | null = fullLogResponse?.data?.[0] ?? null;
const overTime = useMemo(
() =>
results.map((d) => ({
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
})),
[results],
);
const byDriver = useMemo(
() =>
[
{ driver: "Compression", usd: compressionTotal },
{ driver: "Prompt caching", usd: cachingTotal },
].filter((d) => d.usd > 0),
[compressionTotal, cachingTotal],
);
return (
<div className="w-full space-y-6">
<div className="flex flex-wrap items-center justify-between gap-4">
<MethodologyNote />
<AdvancedDatePicker
value={dateValue}
onValueChange={(v) => {
setDateValue(v);
setLogsPage(1);
}}
/>
</div>
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<SummaryCard
label="Total saved"
value={usd(totalSaved)}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
/>
<SummaryCard
label="Compression savings"
value={usd(compressionTotal)}
hint={`${formatNumberWithCommas(savedTokensTotal)} tokens compressed`}
/>
<SummaryCard label="Prompt caching savings" value={usd(cachingTotal)} hint="Cache read discount" />
</div>
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
<Card className="lg:col-span-2">
<CardHeader>
<CardTitle>Savings over time</CardTitle>
</CardHeader>
<CardContent>
<AreaChart
data={overTime}
index="date"
categories={["Compression", "Prompt caching"]}
colors={["emerald", "blue"]}
valueFormatter={usd}
/>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Savings by driver</CardTitle>
</CardHeader>
<CardContent>
<DonutChart
className="h-80"
data={byDriver}
index="driver"
category="usd"
colors={["emerald", "blue"]}
valueFormatter={usd}
showLabel
label={usd(totalSaved)}
/>
</CardContent>
</Card>
</div>
<Card>
<CardHeader>
<CardTitle>Recent Optimized Requests</CardTitle>
<p className="text-sm text-muted-foreground">{logsDescription}</p>
</CardHeader>
<CardContent>
{optimizedLogsLoading && (
<div className="py-10 text-center text-sm text-muted-foreground">Loading optimized requests...</div>
)}
{!optimizedLogsLoading && optimizedLogsError && (
<div className="py-10 text-center text-sm text-red-600">Failed to load optimized requests.</div>
)}
{!optimizedLogsLoading && !optimizedLogsError && optimizedLogsData?.logs.length ? (
<OptimizedRequestsTable
data={optimizedLogsData}
logsPage={logsPage}
onPrevious={() => setLogsPage((page) => Math.max(1, page - 1))}
onNext={() => setLogsPage((page) => Math.min(optimizedLogsData.total_pages, page + 1))}
onRowClick={(log) => {
setSelectedRequestId(log.request_id);
setDrawerOpen(true);
}}
/>
) : null}
{!optimizedLogsLoading && !optimizedLogsError && !optimizedLogsData?.logs.length && (
<div className="py-10 text-center text-sm text-muted-foreground">No optimized requests to display.</div>
)}
</CardContent>
</Card>
<LogDetailsDrawer
open={drawerOpen}
onClose={() => {
setDrawerOpen(false);
setSelectedRequestId(null);
}}
logEntry={selectedLog}
accessToken={accessToken}
allLogs={selectedLog ? [selectedLog] : []}
startTime={drawerStartTime}
/>
</div>
);
};
export default UsageTab;

37
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.test.ts Normal file
View file

@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import { buildCompressionGuardrailPayload, compressionGuardrailsOf } from "./helpers";
describe("compressionGuardrailsOf", () => {
it("keeps only headroom-provider guardrails and drops others", () => {
const filtered = compressionGuardrailsOf({
guardrails: [
{ guardrail_id: "1", guardrail_name: "headroom-compression", litellm_params: { guardrail: "headroom" } },
{ guardrail_id: "2", guardrail_name: "pii-masker", litellm_params: { guardrail: "presidio" } },
{ guardrail_id: "3", guardrail_name: "no-params", litellm_params: null },
],
});
expect(filtered.map((g) => g.guardrail_id)).toEqual(["1"]);
});
});
describe("buildCompressionGuardrailPayload", () => {
it("builds a headroom guardrail payload with trimmed fields", () => {
const payload = buildCompressionGuardrailPayload({
name: " headroom-compression ",
apiBase: " https://compress ",
defaultOn: false,
});
expect(payload).toEqual({
guardrail_name: "headroom-compression",
litellm_params: {
guardrail: "headroom",
mode: "pre_call",
api_base: "https://compress",
default_on: false,
},
});
});
});

39
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/helpers.ts Normal file
View file

@ -0,0 +1,39 @@
export interface GuardrailLitellmParams {
guardrail?: string | null;
api_base?: string | null;
default_on?: boolean | null;
}
export interface GuardrailListItem {
guardrail_id: string;
guardrail_name: string | null;
litellm_params?: GuardrailLitellmParams | null;
}
export interface GuardrailListResponse {
guardrails?: GuardrailListItem[];
}
export const COMPRESSION_GUARDRAIL_PROVIDER = "headroom";
export const isCompressionGuardrail = (guardrail: GuardrailListItem): boolean =>
(guardrail.litellm_params?.guardrail ?? "").toLowerCase() === COMPRESSION_GUARDRAIL_PROVIDER;
export const compressionGuardrailsOf = (response: GuardrailListResponse): GuardrailListItem[] =>
(response.guardrails ?? []).filter(isCompressionGuardrail);
export interface CompressionGuardrailInput {
name: string;
apiBase: string;
defaultOn: boolean;
}
export const buildCompressionGuardrailPayload = (input: CompressionGuardrailInput): Record<string, unknown> => ({
guardrail_name: input.name.trim(),
litellm_params: {
guardrail: COMPRESSION_GUARDRAIL_PROVIDER,
mode: "pre_call",
api_base: input.apiBase.trim(),
default_on: input.defaultOn,
},
});

4
ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx
View file

@ -33,7 +33,7 @@ interface GeneralSettingsPageProps {
userID: string | null;
}
interface generalSettingsItem {
export interface generalSettingsItem {
field_name: string;
field_type: string;
field_value: any;
@ -90,7 +90,7 @@ const SettingValueEditor: React.FC<{
return null;
};
const PromptCachingPanel: React.FC<{
export const PromptCachingPanel: React.FC<{
accessToken: string;
settings: generalSettingsItem[];
onChange: (fieldName: string, newValue: any) => void;