mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
feat(ui): cost savings dashboard for prompt caching and compression
This commit is contained in:
Tin Chi Lo
parent
be609cff3a
commit
aa6d5ee9e3
15 changed files with 1362 additions and 49 deletions
|
|
@ -228,6 +228,11 @@ LAZY_FEATURES: Tuple[LazyFeature, ...] = (
|
|||
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
|
||||
path_prefixes=("/vantage",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="cost_savings",
|
||||
module_path="litellm.proxy.spend_tracking.cost_savings_endpoints",
|
||||
path_prefixes=("/cost_savings",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="usage_ai",
|
||||
module_path="litellm.proxy.management_endpoints.usage_endpoints",
|
||||
|
|
|
|||
|
|
@ -59,6 +59,10 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
|
|||
ToolDiscoveryQueue,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
extract_cache_read_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.compression_savings import (
|
||||
extract_compression_saved_tokens,
|
||||
)
|
||||
|
|
@ -71,31 +75,6 @@ else:
|
|||
ProxyLogging = Any
|
||||
|
||||
|
||||
def _extract_cache_read_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_read_input_tokens field.
|
||||
OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_read_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cached_tokens", 0) or 0)
|
||||
|
||||
|
||||
def _extract_cache_creation_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_creation_input_tokens field.
|
||||
OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens
|
||||
or prompt_tokens_details.cache_creation_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_creation_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0)
|
||||
|
||||
|
||||
class DBSpendUpdateWriter:
|
||||
"""
|
||||
Module responsible for
|
||||
|
|
@ -1851,8 +1830,8 @@ class DBSpendUpdateWriter:
|
|||
api_requests=1,
|
||||
successful_requests=1 if request_status == "success" else 0,
|
||||
failed_requests=1 if request_status != "success" else 0,
|
||||
cache_read_input_tokens=_extract_cache_read_tokens(usage_obj),
|
||||
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
|
||||
cache_read_input_tokens=extract_cache_read_tokens(usage_obj),
|
||||
cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),
|
||||
compression_saved_tokens=extract_compression_saved_tokens(_metadata),
|
||||
)
|
||||
return daily_transaction
|
||||
|
|
|
|||
30
litellm/proxy/spend_tracking/cache_savings.py
Normal file
30
litellm/proxy/spend_tracking/cache_savings.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""
|
||||
Single chokepoint for reading prompt-cache token counts out of a SpendLog
|
||||
``usage_object`` dict. Imported by the daily-spend DB writer and by
|
||||
cost-savings read endpoints so both resolve cache tokens by the same rule.
|
||||
"""
|
||||
|
||||
|
||||
def extract_cache_read_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_read_input_tokens field.
|
||||
OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_read_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cached_tokens", 0) or 0)
|
||||
|
||||
|
||||
def extract_cache_creation_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_creation_input_tokens field.
|
||||
OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens
|
||||
or prompt_tokens_details.cache_creation_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_creation_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0)
|
||||
383
litellm/proxy/spend_tracking/cost_savings_endpoints.py
Normal file
383
litellm/proxy/spend_tracking/cost_savings_endpoints.py
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
"""
|
||||
Cost savings analytics for prompt caching and prompt compression.
|
||||
|
||||
Dollarizes optimization token counts recorded in the daily spend aggregates
|
||||
(cache_read_input_tokens, cache_creation_input_tokens, compression_saved_tokens)
|
||||
using the model cost map:
|
||||
|
||||
- caching savings (net) = cache_read * (input_price - cache_read_price)
|
||||
- cache_creation * (cache_creation_price - input_price)
|
||||
- compression savings = compression_saved * input_price
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from itertools import groupby
|
||||
from typing import Annotated
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
_user_has_admin_view,
|
||||
require_caller_user_id_for_non_admin,
|
||||
)
|
||||
from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
extract_cache_read_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
|
||||
from litellm.types.proxy.cost_savings_endpoints import (
|
||||
CostSavingsActivityResponse,
|
||||
CostSavingsMetrics,
|
||||
DailyCostSavings,
|
||||
OptimizationType,
|
||||
OptimizedRequestSummary,
|
||||
RecentOptimizedRequestsResponse,
|
||||
)
|
||||
|
||||
router = APIRouter(tags=["Budget & Spend Tracking"])
|
||||
|
||||
RECENT_REQUESTS_SCAN_WINDOW = 500
|
||||
|
||||
_ACTIVITY_SQL = (
|
||||
"SELECT date, COALESCE(model, '') AS model, COALESCE(custom_llm_provider, '') AS custom_llm_provider, "
|
||||
"SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens, "
|
||||
"SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens, "
|
||||
"SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, "
|
||||
"SUM(spend)::float AS spend "
|
||||
'FROM "LiteLLM_DailyUserSpend" WHERE date >= $1 AND date <= $2{user_filter} '
|
||||
"GROUP BY date, COALESCE(model, ''), COALESCE(custom_llm_provider, '') "
|
||||
"ORDER BY date"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelPricing:
|
||||
input_cost_per_token: float
|
||||
cache_read_cost_per_token: float | None
|
||||
cache_creation_cost_per_token: float | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SavingsAmounts:
|
||||
cache_savings: float
|
||||
compression_savings: float
|
||||
|
||||
|
||||
class _CostMapEntry(BaseModel):
|
||||
input_cost_per_token: float | None = None
|
||||
cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
|
||||
|
||||
class _DailySavingsRow(BaseModel):
|
||||
date: str
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
cache_read_input_tokens: int
|
||||
cache_creation_input_tokens: int
|
||||
compression_saved_tokens: int
|
||||
spend: float
|
||||
|
||||
|
||||
class _SpendLogRow(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
request_id: str
|
||||
startTime: datetime
|
||||
model: str
|
||||
custom_llm_provider: str | None = None
|
||||
total_tokens: int | None = None
|
||||
spend: float | None = None
|
||||
metadata: object = None
|
||||
|
||||
|
||||
_DAILY_ROWS_ADAPTER = TypeAdapter(list[_DailySavingsRow])
|
||||
|
||||
|
||||
def _pricing_candidates(model: str, custom_llm_provider: str) -> tuple[str, ...]:
|
||||
if not custom_llm_provider:
|
||||
return (model,)
|
||||
return tuple(
|
||||
dict.fromkeys(
|
||||
(
|
||||
f"{custom_llm_provider}/{model}",
|
||||
model,
|
||||
model.removeprefix(f"{custom_llm_provider}/"),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def resolve_model_pricing(model: str, custom_llm_provider: str, cost_map: Mapping[str, object]) -> ModelPricing | None:
|
||||
for candidate in _pricing_candidates(model, custom_llm_provider):
|
||||
raw = cost_map.get(candidate)
|
||||
if not isinstance(raw, Mapping):
|
||||
continue
|
||||
try:
|
||||
entry = _CostMapEntry.model_validate(dict(raw))
|
||||
except ValidationError:
|
||||
continue
|
||||
if entry.input_cost_per_token:
|
||||
return ModelPricing(
|
||||
input_cost_per_token=entry.input_cost_per_token,
|
||||
cache_read_cost_per_token=entry.cache_read_input_token_cost,
|
||||
cache_creation_cost_per_token=entry.cache_creation_input_token_cost,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def compute_savings_amounts(
|
||||
cache_read_tokens: int,
|
||||
cache_creation_tokens: int,
|
||||
compression_saved_tokens: int,
|
||||
pricing: ModelPricing | None,
|
||||
) -> SavingsAmounts:
|
||||
if pricing is None:
|
||||
return SavingsAmounts(cache_savings=0.0, compression_savings=0.0)
|
||||
read_savings = (
|
||||
cache_read_tokens * (pricing.input_cost_per_token - pricing.cache_read_cost_per_token)
|
||||
if pricing.cache_read_cost_per_token is not None
|
||||
else 0.0
|
||||
)
|
||||
write_premium = (
|
||||
cache_creation_tokens * (pricing.cache_creation_cost_per_token - pricing.input_cost_per_token)
|
||||
if pricing.cache_creation_cost_per_token is not None
|
||||
else 0.0
|
||||
)
|
||||
return SavingsAmounts(
|
||||
cache_savings=read_savings - write_premium,
|
||||
compression_savings=compression_saved_tokens * pricing.input_cost_per_token,
|
||||
)
|
||||
|
||||
|
||||
def _is_unpriced(cache_read_tokens: int, compression_saved_tokens: int, pricing: ModelPricing | None) -> bool:
|
||||
if pricing is None:
|
||||
return cache_read_tokens > 0 or compression_saved_tokens > 0
|
||||
return cache_read_tokens > 0 and pricing.cache_read_cost_per_token is None
|
||||
|
||||
|
||||
def _metrics_for_rows(
|
||||
rows: list[_DailySavingsRow],
|
||||
pricing_by_key: Mapping[tuple[str, str], ModelPricing | None],
|
||||
) -> CostSavingsMetrics:
|
||||
amounts = [
|
||||
compute_savings_amounts(
|
||||
cache_read_tokens=row.cache_read_input_tokens,
|
||||
cache_creation_tokens=row.cache_creation_input_tokens,
|
||||
compression_saved_tokens=row.compression_saved_tokens,
|
||||
pricing=pricing_by_key[(row.model, row.custom_llm_provider)],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
cache_savings = sum(amount.cache_savings for amount in amounts)
|
||||
compression_savings = sum(amount.compression_savings for amount in amounts)
|
||||
return CostSavingsMetrics(
|
||||
cache_savings=cache_savings,
|
||||
compression_savings=compression_savings,
|
||||
total_savings=cache_savings + compression_savings,
|
||||
spend=sum(row.spend for row in rows),
|
||||
cache_read_input_tokens=sum(row.cache_read_input_tokens for row in rows),
|
||||
cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in rows),
|
||||
compression_saved_tokens=sum(row.compression_saved_tokens for row in rows),
|
||||
)
|
||||
|
||||
|
||||
def build_activity_response(
|
||||
rows: list[_DailySavingsRow], cost_map: Mapping[str, object]
|
||||
) -> CostSavingsActivityResponse:
|
||||
pricing_by_key = {
|
||||
(row.model, row.custom_llm_provider): resolve_model_pricing(row.model, row.custom_llm_provider, cost_map)
|
||||
for row in rows
|
||||
}
|
||||
results = [
|
||||
DailyCostSavings(date=date_value, metrics=_metrics_for_rows(list(day_rows), pricing_by_key))
|
||||
for date_value, day_rows in groupby(rows, key=lambda row: row.date)
|
||||
]
|
||||
unpriced_models = sorted(
|
||||
{
|
||||
row.model or "(unknown)"
|
||||
for row in rows
|
||||
if _is_unpriced(
|
||||
row.cache_read_input_tokens,
|
||||
row.compression_saved_tokens,
|
||||
pricing_by_key[(row.model, row.custom_llm_provider)],
|
||||
)
|
||||
}
|
||||
)
|
||||
return CostSavingsActivityResponse(
|
||||
results=results,
|
||||
totals=_metrics_for_rows(rows, pricing_by_key),
|
||||
unpriced_models=unpriced_models,
|
||||
)
|
||||
|
||||
|
||||
def _parse_request_metadata(raw: object) -> dict[str, object]:
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
if not isinstance(raw, str):
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except ValueError:
|
||||
return {}
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
|
||||
|
||||
def summarize_optimized_request(row: _SpendLogRow, cost_map: Mapping[str, object]) -> OptimizedRequestSummary | None:
|
||||
metadata = _parse_request_metadata(row.metadata)
|
||||
usage_object = metadata.get("usage_object")
|
||||
usage = usage_object if isinstance(usage_object, dict) else {}
|
||||
cache_read_tokens = extract_cache_read_tokens(usage)
|
||||
cache_creation_tokens = extract_cache_creation_tokens(usage)
|
||||
compression_saved_tokens = extract_compression_saved_tokens(metadata)
|
||||
if cache_read_tokens <= 0 and compression_saved_tokens <= 0:
|
||||
return None
|
||||
pricing = resolve_model_pricing(row.model, row.custom_llm_provider or "", cost_map)
|
||||
amounts = compute_savings_amounts(
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
compression_saved_tokens=compression_saved_tokens,
|
||||
pricing=pricing,
|
||||
)
|
||||
savings = amounts.cache_savings + amounts.compression_savings
|
||||
optimized_cost = row.spend or 0.0
|
||||
optimizations: list[OptimizationType] = [
|
||||
*(["caching"] if cache_read_tokens > 0 else []),
|
||||
*(["compression"] if compression_saved_tokens > 0 else []),
|
||||
]
|
||||
return OptimizedRequestSummary(
|
||||
request_id=row.request_id,
|
||||
start_time=row.startTime.isoformat(),
|
||||
model=row.model,
|
||||
total_tokens=row.total_tokens or 0,
|
||||
optimizations=optimizations,
|
||||
original_cost=optimized_cost + savings,
|
||||
optimized_cost=optimized_cost,
|
||||
savings=savings,
|
||||
)
|
||||
|
||||
|
||||
def _validated_date(value: str, param: str) -> str:
|
||||
try:
|
||||
parsed = date.fromisoformat(value)
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": f"Invalid {param}: expected YYYY-MM-DD, got {value!r}"},
|
||||
) from e
|
||||
return parsed.isoformat()
|
||||
|
||||
|
||||
def _scoped_user_id(user_api_key_dict: UserAPIKeyAuth) -> str | None:
|
||||
if _user_has_admin_view(user_api_key_dict):
|
||||
return None
|
||||
return require_caller_user_id_for_non_admin(user_api_key_dict)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/cost_savings/activity",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=CostSavingsActivityResponse,
|
||||
)
|
||||
@management_endpoint_wrapper
|
||||
async def get_cost_savings_activity(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
start_date: Annotated[str | None, fastapi.Query(description="Start date in YYYY-MM-DD format")] = None,
|
||||
end_date: Annotated[str | None, fastapi.Query(description="End date in YYYY-MM-DD format")] = None,
|
||||
) -> CostSavingsActivityResponse:
|
||||
"""
|
||||
Daily cost savings from prompt caching and prompt compression over a date window.
|
||||
|
||||
Admins see gateway-wide savings; other callers see savings for their own usage.
|
||||
Savings are computed from the daily spend aggregates joined with the model cost
|
||||
map at query time; models missing prices are reported in unpriced_models.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # circular import with proxy_server
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={"error": CommonProxyErrors.db_not_connected_error.value},
|
||||
)
|
||||
if start_date is None or end_date is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": "Please provide start_date and end_date"},
|
||||
)
|
||||
window_start = _validated_date(start_date, "start_date")
|
||||
window_end = _validated_date(end_date, "end_date")
|
||||
scoped_user_id = _scoped_user_id(user_api_key_dict)
|
||||
sql = _ACTIVITY_SQL.format(user_filter=" AND user_id = $3" if scoped_user_id is not None else "")
|
||||
params = [window_start, window_end, *([scoped_user_id] if scoped_user_id is not None else [])]
|
||||
raw_rows = await prisma_client.db.query_raw(sql, *params)
|
||||
rows = _DAILY_ROWS_ADAPTER.validate_python(raw_rows)
|
||||
return build_activity_response(rows, litellm.model_cost)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/cost_savings/recent_requests",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=RecentOptimizedRequestsResponse,
|
||||
)
|
||||
@management_endpoint_wrapper
|
||||
async def get_recent_optimized_requests(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
start_date: Annotated[str | None, fastapi.Query(description="Start date in YYYY-MM-DD format")] = None,
|
||||
end_date: Annotated[str | None, fastapi.Query(description="End date in YYYY-MM-DD format")] = None,
|
||||
limit: Annotated[int, fastapi.Query(ge=1, le=100)] = 20,
|
||||
) -> RecentOptimizedRequestsResponse:
|
||||
"""
|
||||
Most recent requests in the window that benefited from prompt caching or
|
||||
prompt compression, with their actual cost, counterfactual unoptimized cost,
|
||||
and savings.
|
||||
|
||||
Scans up to the scanned_requests most recent spend logs in the window;
|
||||
admins see all requests, other callers see their own.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # circular import with proxy_server
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={"error": CommonProxyErrors.db_not_connected_error.value},
|
||||
)
|
||||
if start_date is None or end_date is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": "Please provide start_date and end_date"},
|
||||
)
|
||||
window_start = datetime.combine(
|
||||
date.fromisoformat(_validated_date(start_date, "start_date")), datetime.min.time(), tzinfo=timezone.utc
|
||||
)
|
||||
window_end_exclusive = datetime.combine(
|
||||
date.fromisoformat(_validated_date(end_date, "end_date")) + timedelta(days=1),
|
||||
datetime.min.time(),
|
||||
tzinfo=timezone.utc,
|
||||
)
|
||||
scoped_user_id = _scoped_user_id(user_api_key_dict)
|
||||
user_scope = {"user": scoped_user_id} if scoped_user_id is not None else {}
|
||||
raw_rows = await prisma_client.db.litellm_spendlogs.find_many(
|
||||
where={"startTime": {"gte": window_start, "lt": window_end_exclusive}, **user_scope},
|
||||
order={"startTime": "desc"},
|
||||
take=RECENT_REQUESTS_SCAN_WINDOW,
|
||||
)
|
||||
summaries = [
|
||||
summary
|
||||
for summary in (
|
||||
summarize_optimized_request(_SpendLogRow.model_validate(raw_row), litellm.model_cost)
|
||||
for raw_row in raw_rows
|
||||
)
|
||||
if summary is not None
|
||||
]
|
||||
return RecentOptimizedRequestsResponse(requests=summaries[:limit], scanned_requests=len(raw_rows))
|
||||
49
litellm/types/proxy/cost_savings_endpoints.py
Normal file
49
litellm/types/proxy/cost_savings_endpoints.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
OptimizationType = Literal["caching", "compression"]
|
||||
|
||||
|
||||
class CostSavingsMetrics(BaseModel):
|
||||
cache_savings: float = 0.0
|
||||
compression_savings: float = 0.0
|
||||
total_savings: float = 0.0
|
||||
spend: float = 0.0
|
||||
cache_read_input_tokens: int = 0
|
||||
cache_creation_input_tokens: int = 0
|
||||
compression_saved_tokens: int = 0
|
||||
|
||||
|
||||
class DailyCostSavings(BaseModel):
|
||||
date: str
|
||||
metrics: CostSavingsMetrics
|
||||
|
||||
|
||||
class CostSavingsActivityResponse(BaseModel):
|
||||
results: list[DailyCostSavings]
|
||||
totals: CostSavingsMetrics = Field(default_factory=CostSavingsMetrics)
|
||||
unpriced_models: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Models with optimized tokens in the window but no usable prices in the model cost map; "
|
||||
"their savings are reported as 0",
|
||||
)
|
||||
|
||||
|
||||
class OptimizedRequestSummary(BaseModel):
|
||||
request_id: str
|
||||
start_time: str
|
||||
model: str
|
||||
total_tokens: int
|
||||
optimizations: list[OptimizationType]
|
||||
original_cost: float
|
||||
optimized_cost: float
|
||||
savings: float
|
||||
|
||||
|
||||
class RecentOptimizedRequestsResponse(BaseModel):
|
||||
requests: list[OptimizedRequestSummary]
|
||||
scanned_requests: int = Field(
|
||||
description="Number of most-recent requests in the window scanned for optimizations; "
|
||||
"optimized requests older than the scan window are not listed"
|
||||
)
|
||||
|
|
@ -0,0 +1,317 @@
|
|||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../.."))
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.proxy.spend_tracking.cost_savings_endpoints import (
|
||||
_DailySavingsRow,
|
||||
_SpendLogRow,
|
||||
build_activity_response,
|
||||
compute_savings_amounts,
|
||||
resolve_model_pricing,
|
||||
summarize_optimized_request,
|
||||
)
|
||||
|
||||
COST_MAP = {
|
||||
"anthropic/claude-x": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
},
|
||||
"gpt-x": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
},
|
||||
"no-cache-price-model": {"input_cost_per_token": 5e-06},
|
||||
"free-model": {"input_cost_per_token": 0.0},
|
||||
"malformed-model": {"input_cost_per_token": "not-a-number"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
class TestResolveModelPricing:
|
||||
def test_provider_qualified_key_wins(self):
|
||||
pricing = resolve_model_pricing("claude-x", "anthropic", COST_MAP)
|
||||
assert pricing is not None
|
||||
assert pricing.input_cost_per_token == 3e-06
|
||||
assert pricing.cache_read_cost_per_token == 3e-07
|
||||
assert pricing.cache_creation_cost_per_token == 3.75e-06
|
||||
|
||||
def test_bare_key_fallback(self):
|
||||
pricing = resolve_model_pricing("gpt-x", "openai", COST_MAP)
|
||||
assert pricing is not None
|
||||
assert pricing.input_cost_per_token == 2e-06
|
||||
assert pricing.cache_creation_cost_per_token is None
|
||||
|
||||
def test_provider_prefixed_stored_model_resolves_bare_key(self):
|
||||
pricing = resolve_model_pricing("openai/gpt-x", "openai", COST_MAP)
|
||||
assert pricing is not None
|
||||
assert pricing.input_cost_per_token == 2e-06
|
||||
|
||||
def test_unknown_model_returns_none(self):
|
||||
assert resolve_model_pricing("nope", "openai", COST_MAP) is None
|
||||
|
||||
def test_zero_input_price_returns_none(self):
|
||||
assert resolve_model_pricing("free-model", "", COST_MAP) is None
|
||||
|
||||
def test_malformed_entry_returns_none(self):
|
||||
assert resolve_model_pricing("malformed-model", "", COST_MAP) is None
|
||||
|
||||
|
||||
class TestComputeSavingsAmounts:
|
||||
def test_net_cache_savings_subtracts_write_premium(self):
|
||||
pricing = resolve_model_pricing("claude-x", "anthropic", COST_MAP)
|
||||
amounts = compute_savings_amounts(
|
||||
cache_read_tokens=1000, cache_creation_tokens=100, compression_saved_tokens=0, pricing=pricing
|
||||
)
|
||||
assert amounts.cache_savings == pytest.approx(1000 * (3e-06 - 3e-07) - 100 * (3.75e-06 - 3e-06))
|
||||
|
||||
def test_compression_savings_use_input_price(self):
|
||||
pricing = resolve_model_pricing("gpt-x", "", COST_MAP)
|
||||
amounts = compute_savings_amounts(
|
||||
cache_read_tokens=0, cache_creation_tokens=0, compression_saved_tokens=500, pricing=pricing
|
||||
)
|
||||
assert amounts.compression_savings == pytest.approx(500 * 2e-06)
|
||||
|
||||
def test_missing_cache_read_price_yields_zero_cache_savings(self):
|
||||
pricing = resolve_model_pricing("no-cache-price-model", "", COST_MAP)
|
||||
amounts = compute_savings_amounts(
|
||||
cache_read_tokens=1000, cache_creation_tokens=0, compression_saved_tokens=0, pricing=pricing
|
||||
)
|
||||
assert amounts.cache_savings == 0.0
|
||||
|
||||
def test_none_pricing_yields_zero(self):
|
||||
amounts = compute_savings_amounts(
|
||||
cache_read_tokens=1000, cache_creation_tokens=10, compression_saved_tokens=500, pricing=None
|
||||
)
|
||||
assert amounts.cache_savings == 0.0
|
||||
assert amounts.compression_savings == 0.0
|
||||
|
||||
|
||||
def _row(**overrides):
|
||||
defaults = {
|
||||
"date": "2026-07-15",
|
||||
"model": "claude-x",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"compression_saved_tokens": 0,
|
||||
"spend": 0.0,
|
||||
}
|
||||
return _DailySavingsRow(**{**defaults, **overrides})
|
||||
|
||||
|
||||
class TestBuildActivityResponse:
|
||||
def test_days_grouped_and_totals_summed(self):
|
||||
rows = [
|
||||
_row(date="2026-07-15", cache_read_input_tokens=1000, spend=1.0),
|
||||
_row(date="2026-07-15", model="gpt-x", custom_llm_provider="openai", compression_saved_tokens=500, spend=2.0),
|
||||
_row(date="2026-07-16", cache_read_input_tokens=2000, cache_creation_input_tokens=100, spend=3.0),
|
||||
]
|
||||
response = build_activity_response(rows, COST_MAP)
|
||||
assert [daily.date for daily in response.results] == ["2026-07-15", "2026-07-16"]
|
||||
day_one, day_two = response.results
|
||||
assert day_one.metrics.cache_savings == pytest.approx(1000 * (3e-06 - 3e-07))
|
||||
assert day_one.metrics.compression_savings == pytest.approx(500 * 2e-06)
|
||||
assert day_two.metrics.cache_savings == pytest.approx(2000 * (3e-06 - 3e-07) - 100 * 7.5e-07)
|
||||
assert response.totals.spend == pytest.approx(6.0)
|
||||
assert response.totals.total_savings == pytest.approx(
|
||||
day_one.metrics.total_savings + day_two.metrics.total_savings
|
||||
)
|
||||
assert response.totals.cache_read_input_tokens == 3000
|
||||
assert response.totals.compression_saved_tokens == 500
|
||||
assert response.unpriced_models == []
|
||||
|
||||
def test_unpriced_models_reported(self):
|
||||
rows = [
|
||||
_row(model="mystery-model", custom_llm_provider="", cache_read_input_tokens=100),
|
||||
_row(model="no-cache-price-model", custom_llm_provider="", cache_read_input_tokens=100),
|
||||
_row(model="mystery-idle", custom_llm_provider="", spend=1.0),
|
||||
]
|
||||
response = build_activity_response(rows, COST_MAP)
|
||||
assert response.unpriced_models == ["mystery-model", "no-cache-price-model"]
|
||||
assert response.totals.cache_savings == 0.0
|
||||
|
||||
|
||||
def _spend_log_row(metadata, **overrides):
|
||||
defaults = {
|
||||
"request_id": "req_1",
|
||||
"startTime": datetime(2026, 7, 15, 12, 0, 0, tzinfo=timezone.utc),
|
||||
"model": "claude-x",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"total_tokens": 1500,
|
||||
"spend": 0.01,
|
||||
"metadata": metadata,
|
||||
}
|
||||
return _SpendLogRow(**{**defaults, **overrides})
|
||||
|
||||
|
||||
class TestSummarizeOptimizedRequest:
|
||||
def test_anthropic_style_cache_read(self):
|
||||
row = _spend_log_row({"usage_object": {"cache_read_input_tokens": 1000}})
|
||||
summary = summarize_optimized_request(row, COST_MAP)
|
||||
assert summary is not None
|
||||
assert summary.optimizations == ["caching"]
|
||||
assert summary.savings == pytest.approx(1000 * (3e-06 - 3e-07))
|
||||
assert summary.original_cost == pytest.approx(summary.optimized_cost + summary.savings)
|
||||
|
||||
def test_openai_style_cached_tokens(self):
|
||||
row = _spend_log_row(
|
||||
{"usage_object": {"prompt_tokens_details": {"cached_tokens": 800}}},
|
||||
model="gpt-x",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
summary = summarize_optimized_request(row, COST_MAP)
|
||||
assert summary is not None
|
||||
assert summary.optimizations == ["caching"]
|
||||
assert summary.savings == pytest.approx(800 * (2e-06 - 1e-06))
|
||||
|
||||
def test_headroom_guardrail_compression(self):
|
||||
row = _spend_log_row(
|
||||
{
|
||||
"usage_object": {},
|
||||
"guardrail_information": [
|
||||
{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": 400}}
|
||||
],
|
||||
}
|
||||
)
|
||||
summary = summarize_optimized_request(row, COST_MAP)
|
||||
assert summary is not None
|
||||
assert summary.optimizations == ["compression"]
|
||||
assert summary.savings == pytest.approx(400 * 3e-06)
|
||||
|
||||
def test_compression_and_caching_both(self):
|
||||
row = _spend_log_row(
|
||||
{
|
||||
"usage_object": {"cache_read_input_tokens": 1000},
|
||||
"compression_savings": {"tokens_saved": 600},
|
||||
}
|
||||
)
|
||||
summary = summarize_optimized_request(row, COST_MAP)
|
||||
assert summary is not None
|
||||
assert summary.optimizations == ["caching", "compression"]
|
||||
assert summary.savings == pytest.approx(1000 * (3e-06 - 3e-07) + 600 * 3e-06)
|
||||
|
||||
def test_metadata_as_json_string(self):
|
||||
row = _spend_log_row('{"usage_object": {"cache_read_input_tokens": 100}}')
|
||||
summary = summarize_optimized_request(row, COST_MAP)
|
||||
assert summary is not None
|
||||
assert summary.optimizations == ["caching"]
|
||||
|
||||
def test_unoptimized_request_returns_none(self):
|
||||
assert summarize_optimized_request(_spend_log_row({"usage_object": {}}), COST_MAP) is None
|
||||
assert summarize_optimized_request(_spend_log_row(None), COST_MAP) is None
|
||||
|
||||
|
||||
def _override_auth(role, user_id="user-1"):
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role, user_id=user_id)
|
||||
|
||||
|
||||
class TestActivityEndpoint:
|
||||
def _setup(self, monkeypatch, rows):
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.query_raw = AsyncMock(return_value=rows)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(litellm, "model_cost", COST_MAP)
|
||||
return mock_prisma
|
||||
|
||||
def test_admin_gets_global_view(self, client, monkeypatch):
|
||||
mock_prisma = self._setup(
|
||||
monkeypatch,
|
||||
[
|
||||
{
|
||||
"date": "2026-07-15",
|
||||
"model": "claude-x",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"cache_read_input_tokens": 1000,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"compression_saved_tokens": 0,
|
||||
"spend": 1.0,
|
||||
}
|
||||
],
|
||||
)
|
||||
_override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
try:
|
||||
response = client.get(
|
||||
"/cost_savings/activity", params={"start_date": "2026-07-09", "end_date": "2026-07-15"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["totals"]["cache_savings"] == pytest.approx(1000 * (3e-06 - 3e-07))
|
||||
sql, *params = mock_prisma.db.query_raw.await_args.args
|
||||
assert "user_id" not in sql
|
||||
assert params == ["2026-07-09", "2026-07-15"]
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
def test_non_admin_scoped_to_own_user_id(self, client, monkeypatch):
|
||||
mock_prisma = self._setup(monkeypatch, [])
|
||||
_override_auth(LitellmUserRoles.INTERNAL_USER, user_id="user-42")
|
||||
try:
|
||||
response = client.get(
|
||||
"/cost_savings/activity", params={"start_date": "2026-07-09", "end_date": "2026-07-15"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
sql, *params = mock_prisma.db.query_raw.await_args.args
|
||||
assert "user_id = $3" in sql
|
||||
assert params == ["2026-07-09", "2026-07-15", "user-42"]
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
def test_invalid_date_rejected(self, client, monkeypatch):
|
||||
self._setup(monkeypatch, [])
|
||||
_override_auth(LitellmUserRoles.PROXY_ADMIN)
|
||||
try:
|
||||
response = client.get(
|
||||
"/cost_savings/activity", params={"start_date": "not-a-date", "end_date": "2026-07-15"}
|
||||
)
|
||||
assert response.status_code == 400
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
|
||||
class TestRecentRequestsEndpoint:
|
||||
def test_non_admin_scoped_and_filtered(self, client, monkeypatch):
|
||||
optimized = {
|
||||
"request_id": "req_hit",
|
||||
"startTime": datetime(2026, 7, 15, 12, 0, 0, tzinfo=timezone.utc),
|
||||
"model": "claude-x",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"total_tokens": 1500,
|
||||
"spend": 0.01,
|
||||
"metadata": {"usage_object": {"cache_read_input_tokens": 1000}},
|
||||
}
|
||||
plain = {**optimized, "request_id": "req_plain", "metadata": {"usage_object": {}}}
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.litellm_spendlogs.find_many = AsyncMock(
|
||||
return_value=[MagicMock(**optimized), MagicMock(**plain)]
|
||||
)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(litellm, "model_cost", COST_MAP)
|
||||
_override_auth(LitellmUserRoles.INTERNAL_USER, user_id="user-42")
|
||||
try:
|
||||
response = client.get(
|
||||
"/cost_savings/recent_requests",
|
||||
params={"start_date": "2026-07-09", "end_date": "2026-07-15"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert [request["request_id"] for request in body["requests"]] == ["req_hit"]
|
||||
assert body["scanned_requests"] == 2
|
||||
where = mock_prisma.db.litellm_spendlogs.find_many.await_args.kwargs["where"]
|
||||
assert where["user"] == "user-42"
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
|
@ -3128,7 +3128,7 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens
|
|||
|
||||
|
||||
def test_extract_cache_read_tokens_anthropic_top_level():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens
|
||||
|
||||
usage_obj = {
|
||||
"prompt_tokens": 100,
|
||||
|
|
@ -3136,34 +3136,34 @@ def test_extract_cache_read_tokens_anthropic_top_level():
|
|||
"prompt_tokens_details": {"cached_tokens": 80},
|
||||
}
|
||||
# Anthropic top-level value should win over prompt_tokens_details fallback.
|
||||
assert _extract_cache_read_tokens(usage_obj) == 80
|
||||
assert extract_cache_read_tokens(usage_obj) == 80
|
||||
|
||||
|
||||
def test_extract_cache_read_tokens_openai_compatible_fallback():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens
|
||||
|
||||
# Anthropic field absent — fall back to prompt_tokens_details.cached_tokens.
|
||||
usage_obj = {
|
||||
"prompt_tokens": 22583,
|
||||
"prompt_tokens_details": {"cached_tokens": 22016},
|
||||
}
|
||||
assert _extract_cache_read_tokens(usage_obj) == 22016
|
||||
assert extract_cache_read_tokens(usage_obj) == 22016
|
||||
|
||||
|
||||
def test_extract_cache_read_tokens_zero_when_missing():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens
|
||||
|
||||
assert _extract_cache_read_tokens({}) == 0
|
||||
assert _extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0
|
||||
assert extract_cache_read_tokens({}) == 0
|
||||
assert extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}})
|
||||
extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}})
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_anthropic_top_level():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
usage_obj = {
|
||||
|
|
@ -3172,12 +3172,12 @@ def test_extract_cache_creation_tokens_anthropic_top_level():
|
|||
"prompt_tokens_details": {"cache_write_tokens": 50},
|
||||
}
|
||||
# Anthropic top-level should short-circuit the fallback.
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 50
|
||||
assert extract_cache_creation_tokens(usage_obj) == 50
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_openai_cache_write_alias():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
# kimi-k2 emits cache_write_tokens.
|
||||
|
|
@ -3185,12 +3185,12 @@ def test_extract_cache_creation_tokens_openai_cache_write_alias():
|
|||
"prompt_tokens": 1000,
|
||||
"prompt_tokens_details": {"cache_write_tokens": 200},
|
||||
}
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 200
|
||||
assert extract_cache_creation_tokens(usage_obj) == 200
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_openai_cache_creation_alias():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
# Other OpenAI-compatible providers emit cache_creation_tokens.
|
||||
|
|
@ -3198,18 +3198,18 @@ def test_extract_cache_creation_tokens_openai_cache_creation_alias():
|
|||
"prompt_tokens": 1000,
|
||||
"prompt_tokens_details": {"cache_creation_tokens": 300},
|
||||
}
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 300
|
||||
assert extract_cache_creation_tokens(usage_obj) == 300
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_zero_when_missing():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
from litellm.proxy.spend_tracking.cache_savings import (
|
||||
extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
assert _extract_cache_creation_tokens({}) == 0
|
||||
assert _extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0
|
||||
assert extract_cache_creation_tokens({}) == 0
|
||||
assert extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_creation_tokens(
|
||||
extract_cache_creation_tokens(
|
||||
{"prompt_tokens_details": {"cache_write_tokens": None}}
|
||||
)
|
||||
== 0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import * as networking from "@/components/networking";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import CostSavingsView, { formatUsd } from "./CostSavingsView";
|
||||
|
||||
function renderView() {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false } },
|
||||
});
|
||||
return render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<CostSavingsView />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
if (typeof window !== "undefined" && !window.ResizeObserver) {
|
||||
window.ResizeObserver = class ResizeObserver {
|
||||
observe() {}
|
||||
unobserve() {}
|
||||
disconnect() {}
|
||||
} as any;
|
||||
}
|
||||
});
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
costSavingsActivityCall: vi.fn(),
|
||||
costSavingsRecentRequestsCall: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
|
||||
__esModule: true,
|
||||
default: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/shared/advanced_date_picker", async () => {
|
||||
const React = await import("react");
|
||||
const AdvancedDatePicker = () => React.createElement("div", { "data-testid": "advanced-date-picker" }, "Date Picker");
|
||||
AdvancedDatePicker.displayName = "AdvancedDatePicker";
|
||||
return { default: AdvancedDatePicker };
|
||||
});
|
||||
|
||||
const ACTIVITY_RESPONSE: networking.CostSavingsActivityResponse = {
|
||||
results: [
|
||||
{
|
||||
date: "2026-07-15",
|
||||
metrics: {
|
||||
cache_savings: 1.5,
|
||||
compression_savings: 0.5,
|
||||
total_savings: 2.0,
|
||||
spend: 10.0,
|
||||
cache_read_input_tokens: 1000,
|
||||
cache_creation_input_tokens: 0,
|
||||
compression_saved_tokens: 250,
|
||||
},
|
||||
},
|
||||
],
|
||||
totals: {
|
||||
cache_savings: 1.5,
|
||||
compression_savings: 0.5,
|
||||
total_savings: 2.0,
|
||||
spend: 10.0,
|
||||
cache_read_input_tokens: 1000,
|
||||
cache_creation_input_tokens: 0,
|
||||
compression_saved_tokens: 250,
|
||||
},
|
||||
unpriced_models: [],
|
||||
};
|
||||
|
||||
const RECENT_RESPONSE: networking.RecentOptimizedRequestsResponse = {
|
||||
requests: [
|
||||
{
|
||||
request_id: "req_abc123",
|
||||
start_time: "2026-07-15T12:00:00+00:00",
|
||||
model: "claude-x",
|
||||
total_tokens: 1500,
|
||||
optimizations: ["caching", "compression"],
|
||||
original_cost: 0.05,
|
||||
optimized_cost: 0.02,
|
||||
savings: 0.03,
|
||||
},
|
||||
],
|
||||
scanned_requests: 42,
|
||||
};
|
||||
|
||||
describe("formatUsd", () => {
|
||||
it("formats zero, cents, and sub-cent values", () => {
|
||||
expect(formatUsd(0)).toBe("$0");
|
||||
expect(formatUsd(12.345)).toBe("$12.35");
|
||||
expect(formatUsd(0.002625)).toBe("$0.002625");
|
||||
expect(formatUsd(1234.5)).toBe("$1,234.50");
|
||||
});
|
||||
});
|
||||
|
||||
describe("CostSavingsView", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(useAuthorized).mockReturnValue({
|
||||
accessToken: "sk-test",
|
||||
token: "token",
|
||||
userId: "user-1",
|
||||
userRole: "Admin",
|
||||
userEmail: null,
|
||||
premiumUser: false,
|
||||
disabledPersonalKeyCreation: false,
|
||||
showSSOBanner: false,
|
||||
} as any);
|
||||
vi.mocked(networking.costSavingsActivityCall).mockResolvedValue(ACTIVITY_RESPONSE);
|
||||
vi.mocked(networking.costSavingsRecentRequestsCall).mockResolvedValue(RECENT_RESPONSE);
|
||||
});
|
||||
|
||||
it("renders KPI totals from the activity response", async () => {
|
||||
renderView();
|
||||
await waitFor(() => {
|
||||
expect(screen.getAllByText("$2.00").length).toBeGreaterThan(0);
|
||||
});
|
||||
expect(screen.getByText("Total Savings")).toBeInTheDocument();
|
||||
expect(screen.getAllByText("$1.50").length).toBeGreaterThan(0);
|
||||
expect(screen.getAllByText("$0.50").length).toBeGreaterThan(0);
|
||||
expect(screen.getAllByText("$10.00").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("renders recent optimized requests with type badges and savings", async () => {
|
||||
renderView();
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("req_abc123")).toBeInTheDocument();
|
||||
});
|
||||
expect(screen.getByText("caching")).toBeInTheDocument();
|
||||
expect(screen.getByText("compression")).toBeInTheDocument();
|
||||
expect(screen.getByText("$0.03")).toBeInTheDocument();
|
||||
expect(screen.getByText(/scanned last 42 requests/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows a warning when models are missing prices", async () => {
|
||||
vi.mocked(networking.costSavingsActivityCall).mockResolvedValue({
|
||||
...ACTIVITY_RESPONSE,
|
||||
unpriced_models: ["mystery-model"],
|
||||
});
|
||||
renderView();
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Some models are missing prices")).toBeInTheDocument();
|
||||
});
|
||||
expect(screen.getByText(/mystery-model/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows the empty state when no requests were optimized", async () => {
|
||||
vi.mocked(networking.costSavingsRecentRequestsCall).mockResolvedValue({ requests: [], scanned_requests: 0 });
|
||||
renderView();
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/No optimized requests in this window/)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
"use client";
|
||||
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { Alert, Card, Table, Tag, Typography } from "antd";
|
||||
import type { ColumnsType } from "antd/es/table";
|
||||
import { ComponentProps, useState } from "react";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import ChartLoader from "@/components/shared/chart_loader";
|
||||
import { AreaChart, CustomLegend, DonutChart, type ChartColor } from "@/components/shared/charts";
|
||||
import {
|
||||
CostOptimizationType,
|
||||
CostSavingsMetrics,
|
||||
costSavingsActivityCall,
|
||||
costSavingsRecentRequestsCall,
|
||||
OptimizedRequestSummary,
|
||||
} from "@/components/networking";
|
||||
|
||||
type DateRangeValue = ComponentProps<typeof AdvancedDatePicker>["value"];
|
||||
|
||||
const SERIES_CATEGORIES = ["Caching", "Compression"] as const;
|
||||
const SERIES_COLORS: readonly ChartColor[] = ["emerald", "blue"];
|
||||
|
||||
const OPTIMIZATION_TAG_COLOR: Record<CostOptimizationType, string> = {
|
||||
caching: "green",
|
||||
compression: "blue",
|
||||
};
|
||||
|
||||
export function formatUsd(value: number): string {
|
||||
if (value === 0) return "$0";
|
||||
const abs = Math.abs(value);
|
||||
if (abs >= 0.01) {
|
||||
return `$${value.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`;
|
||||
}
|
||||
return `$${value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "")}`;
|
||||
}
|
||||
|
||||
function defaultDateRange(): DateRangeValue {
|
||||
return {
|
||||
from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
|
||||
to: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
const RECENT_REQUEST_COLUMNS: ColumnsType<OptimizedRequestSummary> = [
|
||||
{
|
||||
title: "Request ID",
|
||||
dataIndex: "request_id",
|
||||
key: "request_id",
|
||||
render: (value: string) => <span className="font-mono text-xs">{value}</span>,
|
||||
},
|
||||
{ title: "Model", dataIndex: "model", key: "model" },
|
||||
{
|
||||
title: "Tokens",
|
||||
dataIndex: "total_tokens",
|
||||
key: "total_tokens",
|
||||
render: (value: number) => value.toLocaleString(),
|
||||
},
|
||||
{
|
||||
title: "Type",
|
||||
dataIndex: "optimizations",
|
||||
key: "optimizations",
|
||||
render: (optimizations: CostOptimizationType[]) => (
|
||||
<span>
|
||||
{optimizations.map((optimization) => (
|
||||
<Tag key={optimization} color={OPTIMIZATION_TAG_COLOR[optimization]}>
|
||||
{optimization}
|
||||
</Tag>
|
||||
))}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: "Original Cost",
|
||||
dataIndex: "original_cost",
|
||||
key: "original_cost",
|
||||
align: "right",
|
||||
render: (value: number) => <span className="text-gray-500 line-through">{formatUsd(value)}</span>,
|
||||
},
|
||||
{
|
||||
title: "Optimized Cost",
|
||||
dataIndex: "optimized_cost",
|
||||
key: "optimized_cost",
|
||||
align: "right",
|
||||
render: (value: number) => formatUsd(value),
|
||||
},
|
||||
{
|
||||
title: "Savings",
|
||||
dataIndex: "savings",
|
||||
key: "savings",
|
||||
align: "right",
|
||||
render: (value: number) => <span className="text-green-600 font-medium">{formatUsd(value)}</span>,
|
||||
},
|
||||
];
|
||||
|
||||
interface SavingsKpiGridProps {
|
||||
totals: CostSavingsMetrics | undefined;
|
||||
}
|
||||
|
||||
function SavingsKpiGrid({ totals }: SavingsKpiGridProps) {
|
||||
return (
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4 mb-6">
|
||||
<MetricCard label="Total Savings" value={formatUsd(totals?.total_savings ?? 0)} valueColor="text-green-600" />
|
||||
<MetricCard
|
||||
label="Caching Savings"
|
||||
value={formatUsd(totals?.cache_savings ?? 0)}
|
||||
valueColor="text-emerald-600"
|
||||
subtitle={`${(totals?.cache_read_input_tokens ?? 0).toLocaleString()} cached tokens read`}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Compression Savings"
|
||||
value={formatUsd(totals?.compression_savings ?? 0)}
|
||||
valueColor="text-blue-600"
|
||||
subtitle={`${(totals?.compression_saved_tokens ?? 0).toLocaleString()} tokens compressed away`}
|
||||
/>
|
||||
<MetricCard label="Total Spend" value={formatUsd(totals?.spend ?? 0)} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function CostSavingsView() {
|
||||
const { accessToken } = useAuthorized();
|
||||
const [dateValue, setDateValue] = useState<DateRangeValue>(defaultDateRange);
|
||||
|
||||
const startTime = dateValue.from;
|
||||
const endTime = dateValue.to;
|
||||
const rangeReady = Boolean(accessToken && startTime && endTime);
|
||||
|
||||
const activityQuery = useQuery({
|
||||
queryKey: ["costSavingsActivity", accessToken, startTime?.toDateString(), endTime?.toDateString()],
|
||||
queryFn: () => costSavingsActivityCall(accessToken!, startTime!, endTime!),
|
||||
enabled: rangeReady,
|
||||
});
|
||||
|
||||
const recentQuery = useQuery({
|
||||
queryKey: ["costSavingsRecentRequests", accessToken, startTime?.toDateString(), endTime?.toDateString()],
|
||||
queryFn: () => costSavingsRecentRequestsCall(accessToken!, startTime!, endTime!),
|
||||
enabled: rangeReady,
|
||||
});
|
||||
|
||||
const totals = activityQuery.data?.totals;
|
||||
const unpricedModels = activityQuery.data?.unpriced_models ?? [];
|
||||
const chartData =
|
||||
activityQuery.data?.results.map((day) => ({
|
||||
date: day.date,
|
||||
Caching: day.metrics.cache_savings,
|
||||
Compression: day.metrics.compression_savings,
|
||||
})) ?? [];
|
||||
const donutData = totals
|
||||
? [
|
||||
{ name: "Caching", value: totals.cache_savings },
|
||||
{ name: "Compression", value: totals.compression_savings },
|
||||
]
|
||||
: [];
|
||||
const recentRequests = recentQuery.data?.requests ?? [];
|
||||
|
||||
return (
|
||||
<div className="w-full p-8">
|
||||
<div className="flex items-end justify-between gap-6 mb-6">
|
||||
<div>
|
||||
<Typography.Title level={3} className="mb-0!">
|
||||
Cost Savings
|
||||
</Typography.Title>
|
||||
<Typography.Text type="secondary">Savings from prompt caching and prompt compression</Typography.Text>
|
||||
</div>
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={setDateValue} label="" showTimeRange={false} />
|
||||
</div>
|
||||
|
||||
{unpricedModels.length > 0 && (
|
||||
<Alert
|
||||
className="mb-6"
|
||||
type="warning"
|
||||
showIcon
|
||||
message="Some models are missing prices"
|
||||
description={`Savings could not be computed for: ${unpricedModels.join(", ")}. Their savings are shown as $0.`}
|
||||
/>
|
||||
)}
|
||||
|
||||
<SavingsKpiGrid totals={totals} />
|
||||
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-4 mb-6">
|
||||
<Card className="lg:col-span-2 border border-gray-200 rounded-lg">
|
||||
<Typography.Title level={5} className="mb-0!">
|
||||
Savings Over Time
|
||||
</Typography.Title>
|
||||
<Typography.Text type="secondary">Daily savings by optimization type</Typography.Text>
|
||||
{activityQuery.isLoading ? (
|
||||
<ChartLoader />
|
||||
) : (
|
||||
<AreaChart
|
||||
className="mt-4"
|
||||
data={chartData}
|
||||
index="date"
|
||||
categories={SERIES_CATEGORIES}
|
||||
colors={SERIES_COLORS}
|
||||
valueFormatter={formatUsd}
|
||||
yAxisWidth={80}
|
||||
/>
|
||||
)}
|
||||
</Card>
|
||||
<Card className="border border-gray-200 rounded-lg">
|
||||
<Typography.Title level={5} className="mb-0!">
|
||||
Savings Distribution
|
||||
</Typography.Title>
|
||||
<Typography.Text type="secondary">By optimization type</Typography.Text>
|
||||
{activityQuery.isLoading ? (
|
||||
<ChartLoader />
|
||||
) : (
|
||||
<>
|
||||
<DonutChart
|
||||
className="mt-4 h-60"
|
||||
data={donutData}
|
||||
index="name"
|
||||
category="value"
|
||||
colors={SERIES_COLORS}
|
||||
valueFormatter={formatUsd}
|
||||
showLabel
|
||||
/>
|
||||
<CustomLegend categories={SERIES_CATEGORIES} colors={SERIES_COLORS} />
|
||||
</>
|
||||
)}
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<Card className="border border-gray-200 rounded-lg" styles={{ body: { padding: 0 } }}>
|
||||
<div className="p-6 pb-4">
|
||||
<Typography.Title level={5} className="mb-0!">
|
||||
Recent Optimized Requests
|
||||
</Typography.Title>
|
||||
<Typography.Text type="secondary">
|
||||
Latest requests that benefited from caching or compression
|
||||
{recentQuery.data ? ` (scanned last ${recentQuery.data.scanned_requests} requests in range)` : ""}
|
||||
</Typography.Text>
|
||||
</div>
|
||||
{recentQuery.isLoading ? (
|
||||
<div className="p-6">
|
||||
<ChartLoader />
|
||||
</div>
|
||||
) : (
|
||||
<Table
|
||||
columns={RECENT_REQUEST_COLUMNS}
|
||||
dataSource={recentRequests}
|
||||
rowKey="request_id"
|
||||
pagination={false}
|
||||
size="middle"
|
||||
locale={{
|
||||
emptyText:
|
||||
"No optimized requests in this window. Savings appear here once prompt caching or prompt compression kicks in.",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
"use client";
|
||||
|
||||
import CostSavingsView from "./CostSavingsView";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
|
||||
export default function CostSavingsPage() {
|
||||
useAuthorized();
|
||||
return <CostSavingsView />;
|
||||
}
|
||||
|
|
@ -44,6 +44,7 @@ import {
|
|||
Palette,
|
||||
PanelLeftClose,
|
||||
PanelLeftOpen,
|
||||
PiggyBank,
|
||||
PlayCircle,
|
||||
Route,
|
||||
ScrollText,
|
||||
|
|
@ -131,6 +132,13 @@ const menuGroups: MenuGroup[] = [
|
|||
icon: <Network {...ICON} />,
|
||||
roles: rolesAllowedToViewWriteScopedPages,
|
||||
},
|
||||
{
|
||||
key: "cost-savings",
|
||||
page: "cost-savings",
|
||||
label: "Cost Savings",
|
||||
icon: <PiggyBank {...ICON} />,
|
||||
roles: [...all_admin_roles, ...internalUserRoles],
|
||||
},
|
||||
{
|
||||
key: "agentic",
|
||||
page: "agentic",
|
||||
|
|
|
|||
|
|
@ -1448,6 +1448,91 @@ export const teamDailyActivityCall = async (
|
|||
});
|
||||
};
|
||||
|
||||
export interface CostSavingsMetrics {
|
||||
cache_savings: number;
|
||||
compression_savings: number;
|
||||
total_savings: number;
|
||||
spend: number;
|
||||
cache_read_input_tokens: number;
|
||||
cache_creation_input_tokens: number;
|
||||
compression_saved_tokens: number;
|
||||
}
|
||||
|
||||
export interface DailyCostSavings {
|
||||
date: string;
|
||||
metrics: CostSavingsMetrics;
|
||||
}
|
||||
|
||||
export interface CostSavingsActivityResponse {
|
||||
results: DailyCostSavings[];
|
||||
totals: CostSavingsMetrics;
|
||||
unpriced_models: string[];
|
||||
}
|
||||
|
||||
export type CostOptimizationType = "caching" | "compression";
|
||||
|
||||
export interface OptimizedRequestSummary {
|
||||
request_id: string;
|
||||
start_time: string;
|
||||
model: string;
|
||||
total_tokens: number;
|
||||
optimizations: CostOptimizationType[];
|
||||
original_cost: number;
|
||||
optimized_cost: number;
|
||||
savings: number;
|
||||
}
|
||||
|
||||
export interface RecentOptimizedRequestsResponse {
|
||||
requests: OptimizedRequestSummary[];
|
||||
scanned_requests: number;
|
||||
}
|
||||
|
||||
export const costSavingsActivityCall = async (
|
||||
accessToken: string,
|
||||
startTime: Date,
|
||||
endTime: Date,
|
||||
): Promise<CostSavingsActivityResponse> => {
|
||||
/**
|
||||
* Get daily cost savings from prompt caching and prompt compression
|
||||
*/
|
||||
try {
|
||||
return await apiClient.get<CostSavingsActivityResponse>(`/cost_savings/activity`, {
|
||||
accessToken,
|
||||
query: {
|
||||
start_date: formatDate(startTime),
|
||||
end_date: formatDate(endTime),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch cost savings activity:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const costSavingsRecentRequestsCall = async (
|
||||
accessToken: string,
|
||||
startTime: Date,
|
||||
endTime: Date,
|
||||
limit: number = 20,
|
||||
): Promise<RecentOptimizedRequestsResponse> => {
|
||||
/**
|
||||
* Get recent requests that benefited from prompt caching or prompt compression
|
||||
*/
|
||||
try {
|
||||
return await apiClient.get<RecentOptimizedRequestsResponse>(`/cost_savings/recent_requests`, {
|
||||
accessToken,
|
||||
query: {
|
||||
start_date: formatDate(startTime),
|
||||
end_date: formatDate(endTime),
|
||||
limit,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch recent optimized requests:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const organizationDailyActivityCall = async (
|
||||
accessToken: string,
|
||||
startTime: Date,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ export const pageDescriptions: Record<string, string> = {
|
|||
"model-hub-table": "Explore available AI models and providers",
|
||||
"learning-resources": "Access tutorials and documentation",
|
||||
caching: "Configure response caching and coordination Redis settings",
|
||||
"cost-savings": "Track cost savings from prompt caching and prompt compression",
|
||||
"transform-request": "Set up request transformation rules",
|
||||
"cost-tracking": "Track and analyze API costs",
|
||||
"ui-theme": "Customize dashboard appearance",
|
||||
|
|
|
|||
34
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
34
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -2474,6 +2474,40 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/cost_savings": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** cost_savings */
|
||||
get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description OK */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/credentials": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ export const MIGRATED_PAGES: Record<string, string> = {
|
|||
// Legacy alias: the old switch matched ?page=claude-code-plugins for the same panel.
|
||||
"claude-code-plugins": "skills",
|
||||
caching: "caching",
|
||||
"cost-savings": "cost-savings",
|
||||
"cost-tracking": "cost-tracking",
|
||||
"transform-request": "transform-request",
|
||||
"ui-theme": "ui-theme",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue