diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 1dda1f29fb9..402fbb21801 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -228,6 +228,11 @@ LAZY_FEATURES: Tuple[LazyFeature, ...] = ( module_path="litellm.proxy.spend_tracking.vantage_endpoints", path_prefixes=("/vantage",), ), + LazyFeature( + name="cost_savings", + module_path="litellm.proxy.spend_tracking.cost_savings_endpoints", + path_prefixes=("/cost_savings",), + ), LazyFeature( name="usage_ai", module_path="litellm.proxy.management_endpoints.usage_endpoints", diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 35c00ebe284..d28fb366259 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -59,6 +59,10 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, ) from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING +from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, + extract_cache_read_tokens, +) from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, ) @@ -71,31 +75,6 @@ else: ProxyLogging = Any -def _extract_cache_read_tokens(usage_obj: dict) -> int: - """ - Anthropic: top-level cache_read_input_tokens field. - OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens. - """ - explicit = usage_obj.get("cache_read_input_tokens", 0) or 0 - if explicit: - return int(explicit) - details = usage_obj.get("prompt_tokens_details") or {} - return int(details.get("cached_tokens", 0) or 0) - - -def _extract_cache_creation_tokens(usage_obj: dict) -> int: - """ - Anthropic: top-level cache_creation_input_tokens field. - OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens - or prompt_tokens_details.cache_creation_tokens. - """ - explicit = usage_obj.get("cache_creation_input_tokens", 0) or 0 - if explicit: - return int(explicit) - details = usage_obj.get("prompt_tokens_details") or {} - return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0) - - class DBSpendUpdateWriter: """ Module responsible for @@ -1851,8 +1830,8 @@ class DBSpendUpdateWriter: api_requests=1, successful_requests=1 if request_status == "success" else 0, failed_requests=1 if request_status != "success" else 0, - cache_read_input_tokens=_extract_cache_read_tokens(usage_obj), - cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj), + cache_read_input_tokens=extract_cache_read_tokens(usage_obj), + cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj), compression_saved_tokens=extract_compression_saved_tokens(_metadata), ) return daily_transaction diff --git a/litellm/proxy/spend_tracking/cache_savings.py b/litellm/proxy/spend_tracking/cache_savings.py new file mode 100644 index 00000000000..ad17c38493d --- /dev/null +++ b/litellm/proxy/spend_tracking/cache_savings.py @@ -0,0 +1,30 @@ +""" +Single chokepoint for reading prompt-cache token counts out of a SpendLog +``usage_object`` dict. Imported by the daily-spend DB writer and by +cost-savings read endpoints so both resolve cache tokens by the same rule. +""" + + +def extract_cache_read_tokens(usage_obj: dict) -> int: + """ + Anthropic: top-level cache_read_input_tokens field. + OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens. + """ + explicit = usage_obj.get("cache_read_input_tokens", 0) or 0 + if explicit: + return int(explicit) + details = usage_obj.get("prompt_tokens_details") or {} + return int(details.get("cached_tokens", 0) or 0) + + +def extract_cache_creation_tokens(usage_obj: dict) -> int: + """ + Anthropic: top-level cache_creation_input_tokens field. + OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens + or prompt_tokens_details.cache_creation_tokens. + """ + explicit = usage_obj.get("cache_creation_input_tokens", 0) or 0 + if explicit: + return int(explicit) + details = usage_obj.get("prompt_tokens_details") or {} + return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0) diff --git a/litellm/proxy/spend_tracking/cost_savings_endpoints.py b/litellm/proxy/spend_tracking/cost_savings_endpoints.py new file mode 100644 index 00000000000..6df2705c1be --- /dev/null +++ b/litellm/proxy/spend_tracking/cost_savings_endpoints.py @@ -0,0 +1,383 @@ +""" +Cost savings analytics for prompt caching and prompt compression. + +Dollarizes optimization token counts recorded in the daily spend aggregates +(cache_read_input_tokens, cache_creation_input_tokens, compression_saved_tokens) +using the model cost map: + +- caching savings (net) = cache_read * (input_price - cache_read_price) + - cache_creation * (cache_creation_price - input_price) +- compression savings = compression_saved * input_price +""" + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from itertools import groupby +from typing import Annotated + +import fastapi +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +import litellm +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_endpoints.common_utils import ( + _user_has_admin_view, + require_caller_user_id_for_non_admin, +) +from litellm.proxy.management_helpers.utils import management_endpoint_wrapper +from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, + extract_cache_read_tokens, +) +from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens +from litellm.types.proxy.cost_savings_endpoints import ( + CostSavingsActivityResponse, + CostSavingsMetrics, + DailyCostSavings, + OptimizationType, + OptimizedRequestSummary, + RecentOptimizedRequestsResponse, +) + +router = APIRouter(tags=["Budget & Spend Tracking"]) + +RECENT_REQUESTS_SCAN_WINDOW = 500 + +_ACTIVITY_SQL = ( + "SELECT date, COALESCE(model, '') AS model, COALESCE(custom_llm_provider, '') AS custom_llm_provider, " + "SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens, " + "SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens, " + "SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, " + "SUM(spend)::float AS spend " + 'FROM "LiteLLM_DailyUserSpend" WHERE date >= $1 AND date <= $2{user_filter} ' + "GROUP BY date, COALESCE(model, ''), COALESCE(custom_llm_provider, '') " + "ORDER BY date" +) + + +@dataclass(frozen=True, slots=True) +class ModelPricing: + input_cost_per_token: float + cache_read_cost_per_token: float | None + cache_creation_cost_per_token: float | None + + +@dataclass(frozen=True, slots=True) +class SavingsAmounts: + cache_savings: float + compression_savings: float + + +class _CostMapEntry(BaseModel): + input_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + + +class _DailySavingsRow(BaseModel): + date: str + model: str + custom_llm_provider: str + cache_read_input_tokens: int + cache_creation_input_tokens: int + compression_saved_tokens: int + spend: float + + +class _SpendLogRow(BaseModel): + model_config = ConfigDict(from_attributes=True) + + request_id: str + startTime: datetime + model: str + custom_llm_provider: str | None = None + total_tokens: int | None = None + spend: float | None = None + metadata: object = None + + +_DAILY_ROWS_ADAPTER = TypeAdapter(list[_DailySavingsRow]) + + +def _pricing_candidates(model: str, custom_llm_provider: str) -> tuple[str, ...]: + if not custom_llm_provider: + return (model,) + return tuple( + dict.fromkeys( + ( + f"{custom_llm_provider}/{model}", + model, + model.removeprefix(f"{custom_llm_provider}/"), + ) + ) + ) + + +def resolve_model_pricing(model: str, custom_llm_provider: str, cost_map: Mapping[str, object]) -> ModelPricing | None: + for candidate in _pricing_candidates(model, custom_llm_provider): + raw = cost_map.get(candidate) + if not isinstance(raw, Mapping): + continue + try: + entry = _CostMapEntry.model_validate(dict(raw)) + except ValidationError: + continue + if entry.input_cost_per_token: + return ModelPricing( + input_cost_per_token=entry.input_cost_per_token, + cache_read_cost_per_token=entry.cache_read_input_token_cost, + cache_creation_cost_per_token=entry.cache_creation_input_token_cost, + ) + return None + + +def compute_savings_amounts( + cache_read_tokens: int, + cache_creation_tokens: int, + compression_saved_tokens: int, + pricing: ModelPricing | None, +) -> SavingsAmounts: + if pricing is None: + return SavingsAmounts(cache_savings=0.0, compression_savings=0.0) + read_savings = ( + cache_read_tokens * (pricing.input_cost_per_token - pricing.cache_read_cost_per_token) + if pricing.cache_read_cost_per_token is not None + else 0.0 + ) + write_premium = ( + cache_creation_tokens * (pricing.cache_creation_cost_per_token - pricing.input_cost_per_token) + if pricing.cache_creation_cost_per_token is not None + else 0.0 + ) + return SavingsAmounts( + cache_savings=read_savings - write_premium, + compression_savings=compression_saved_tokens * pricing.input_cost_per_token, + ) + + +def _is_unpriced(cache_read_tokens: int, compression_saved_tokens: int, pricing: ModelPricing | None) -> bool: + if pricing is None: + return cache_read_tokens > 0 or compression_saved_tokens > 0 + return cache_read_tokens > 0 and pricing.cache_read_cost_per_token is None + + +def _metrics_for_rows( + rows: list[_DailySavingsRow], + pricing_by_key: Mapping[tuple[str, str], ModelPricing | None], +) -> CostSavingsMetrics: + amounts = [ + compute_savings_amounts( + cache_read_tokens=row.cache_read_input_tokens, + cache_creation_tokens=row.cache_creation_input_tokens, + compression_saved_tokens=row.compression_saved_tokens, + pricing=pricing_by_key[(row.model, row.custom_llm_provider)], + ) + for row in rows + ] + cache_savings = sum(amount.cache_savings for amount in amounts) + compression_savings = sum(amount.compression_savings for amount in amounts) + return CostSavingsMetrics( + cache_savings=cache_savings, + compression_savings=compression_savings, + total_savings=cache_savings + compression_savings, + spend=sum(row.spend for row in rows), + cache_read_input_tokens=sum(row.cache_read_input_tokens for row in rows), + cache_creation_input_tokens=sum(row.cache_creation_input_tokens for row in rows), + compression_saved_tokens=sum(row.compression_saved_tokens for row in rows), + ) + + +def build_activity_response( + rows: list[_DailySavingsRow], cost_map: Mapping[str, object] +) -> CostSavingsActivityResponse: + pricing_by_key = { + (row.model, row.custom_llm_provider): resolve_model_pricing(row.model, row.custom_llm_provider, cost_map) + for row in rows + } + results = [ + DailyCostSavings(date=date_value, metrics=_metrics_for_rows(list(day_rows), pricing_by_key)) + for date_value, day_rows in groupby(rows, key=lambda row: row.date) + ] + unpriced_models = sorted( + { + row.model or "(unknown)" + for row in rows + if _is_unpriced( + row.cache_read_input_tokens, + row.compression_saved_tokens, + pricing_by_key[(row.model, row.custom_llm_provider)], + ) + } + ) + return CostSavingsActivityResponse( + results=results, + totals=_metrics_for_rows(rows, pricing_by_key), + unpriced_models=unpriced_models, + ) + + +def _parse_request_metadata(raw: object) -> dict[str, object]: + if isinstance(raw, dict): + return raw + if not isinstance(raw, str): + return {} + try: + parsed = json.loads(raw) + except ValueError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def summarize_optimized_request(row: _SpendLogRow, cost_map: Mapping[str, object]) -> OptimizedRequestSummary | None: + metadata = _parse_request_metadata(row.metadata) + usage_object = metadata.get("usage_object") + usage = usage_object if isinstance(usage_object, dict) else {} + cache_read_tokens = extract_cache_read_tokens(usage) + cache_creation_tokens = extract_cache_creation_tokens(usage) + compression_saved_tokens = extract_compression_saved_tokens(metadata) + if cache_read_tokens <= 0 and compression_saved_tokens <= 0: + return None + pricing = resolve_model_pricing(row.model, row.custom_llm_provider or "", cost_map) + amounts = compute_savings_amounts( + cache_read_tokens=cache_read_tokens, + cache_creation_tokens=cache_creation_tokens, + compression_saved_tokens=compression_saved_tokens, + pricing=pricing, + ) + savings = amounts.cache_savings + amounts.compression_savings + optimized_cost = row.spend or 0.0 + optimizations: list[OptimizationType] = [ + *(["caching"] if cache_read_tokens > 0 else []), + *(["compression"] if compression_saved_tokens > 0 else []), + ] + return OptimizedRequestSummary( + request_id=row.request_id, + start_time=row.startTime.isoformat(), + model=row.model, + total_tokens=row.total_tokens or 0, + optimizations=optimizations, + original_cost=optimized_cost + savings, + optimized_cost=optimized_cost, + savings=savings, + ) + + +def _validated_date(value: str, param: str) -> str: + try: + parsed = date.fromisoformat(value) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": f"Invalid {param}: expected YYYY-MM-DD, got {value!r}"}, + ) from e + return parsed.isoformat() + + +def _scoped_user_id(user_api_key_dict: UserAPIKeyAuth) -> str | None: + if _user_has_admin_view(user_api_key_dict): + return None + return require_caller_user_id_for_non_admin(user_api_key_dict) + + +@router.get( + "/cost_savings/activity", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + response_model=CostSavingsActivityResponse, +) +@management_endpoint_wrapper +async def get_cost_savings_activity( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[str | None, fastapi.Query(description="Start date in YYYY-MM-DD format")] = None, + end_date: Annotated[str | None, fastapi.Query(description="End date in YYYY-MM-DD format")] = None, +) -> CostSavingsActivityResponse: + """ + Daily cost savings from prompt caching and prompt compression over a date window. + + Admins see gateway-wide savings; other callers see savings for their own usage. + Savings are computed from the daily spend aggregates joined with the model cost + map at query time; models missing prices are reported in unpriced_models. + """ + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # circular import with proxy_server + + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + if start_date is None or end_date is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "Please provide start_date and end_date"}, + ) + window_start = _validated_date(start_date, "start_date") + window_end = _validated_date(end_date, "end_date") + scoped_user_id = _scoped_user_id(user_api_key_dict) + sql = _ACTIVITY_SQL.format(user_filter=" AND user_id = $3" if scoped_user_id is not None else "") + params = [window_start, window_end, *([scoped_user_id] if scoped_user_id is not None else [])] + raw_rows = await prisma_client.db.query_raw(sql, *params) + rows = _DAILY_ROWS_ADAPTER.validate_python(raw_rows) + return build_activity_response(rows, litellm.model_cost) + + +@router.get( + "/cost_savings/recent_requests", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + response_model=RecentOptimizedRequestsResponse, +) +@management_endpoint_wrapper +async def get_recent_optimized_requests( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[str | None, fastapi.Query(description="Start date in YYYY-MM-DD format")] = None, + end_date: Annotated[str | None, fastapi.Query(description="End date in YYYY-MM-DD format")] = None, + limit: Annotated[int, fastapi.Query(ge=1, le=100)] = 20, +) -> RecentOptimizedRequestsResponse: + """ + Most recent requests in the window that benefited from prompt caching or + prompt compression, with their actual cost, counterfactual unoptimized cost, + and savings. + + Scans up to the scanned_requests most recent spend logs in the window; + admins see all requests, other callers see their own. + """ + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # circular import with proxy_server + + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + if start_date is None or end_date is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "Please provide start_date and end_date"}, + ) + window_start = datetime.combine( + date.fromisoformat(_validated_date(start_date, "start_date")), datetime.min.time(), tzinfo=timezone.utc + ) + window_end_exclusive = datetime.combine( + date.fromisoformat(_validated_date(end_date, "end_date")) + timedelta(days=1), + datetime.min.time(), + tzinfo=timezone.utc, + ) + scoped_user_id = _scoped_user_id(user_api_key_dict) + user_scope = {"user": scoped_user_id} if scoped_user_id is not None else {} + raw_rows = await prisma_client.db.litellm_spendlogs.find_many( + where={"startTime": {"gte": window_start, "lt": window_end_exclusive}, **user_scope}, + order={"startTime": "desc"}, + take=RECENT_REQUESTS_SCAN_WINDOW, + ) + summaries = [ + summary + for summary in ( + summarize_optimized_request(_SpendLogRow.model_validate(raw_row), litellm.model_cost) + for raw_row in raw_rows + ) + if summary is not None + ] + return RecentOptimizedRequestsResponse(requests=summaries[:limit], scanned_requests=len(raw_rows)) diff --git a/litellm/types/proxy/cost_savings_endpoints.py b/litellm/types/proxy/cost_savings_endpoints.py new file mode 100644 index 00000000000..f85849659b0 --- /dev/null +++ b/litellm/types/proxy/cost_savings_endpoints.py @@ -0,0 +1,49 @@ +from typing import Literal + +from pydantic import BaseModel, Field + +OptimizationType = Literal["caching", "compression"] + + +class CostSavingsMetrics(BaseModel): + cache_savings: float = 0.0 + compression_savings: float = 0.0 + total_savings: float = 0.0 + spend: float = 0.0 + cache_read_input_tokens: int = 0 + cache_creation_input_tokens: int = 0 + compression_saved_tokens: int = 0 + + +class DailyCostSavings(BaseModel): + date: str + metrics: CostSavingsMetrics + + +class CostSavingsActivityResponse(BaseModel): + results: list[DailyCostSavings] + totals: CostSavingsMetrics = Field(default_factory=CostSavingsMetrics) + unpriced_models: list[str] = Field( + default_factory=list, + description="Models with optimized tokens in the window but no usable prices in the model cost map; " + "their savings are reported as 0", + ) + + +class OptimizedRequestSummary(BaseModel): + request_id: str + start_time: str + model: str + total_tokens: int + optimizations: list[OptimizationType] + original_cost: float + optimized_cost: float + savings: float + + +class RecentOptimizedRequestsResponse(BaseModel): + requests: list[OptimizedRequestSummary] + scanned_requests: int = Field( + description="Number of most-recent requests in the window scanned for optimizations; " + "optimized requests older than the scan window are not listed" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_cost_savings_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cost_savings_endpoints.py new file mode 100644 index 00000000000..077277d3356 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_cost_savings_endpoints.py @@ -0,0 +1,317 @@ +import os +import sys +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +import litellm.proxy.proxy_server as ps +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.proxy_server import app +from litellm.proxy.spend_tracking.cost_savings_endpoints import ( + _DailySavingsRow, + _SpendLogRow, + build_activity_response, + compute_savings_amounts, + resolve_model_pricing, + summarize_optimized_request, +) + +COST_MAP = { + "anthropic/claude-x": { + "input_cost_per_token": 3e-06, + "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, + }, + "gpt-x": { + "input_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-06, + }, + "no-cache-price-model": {"input_cost_per_token": 5e-06}, + "free-model": {"input_cost_per_token": 0.0}, + "malformed-model": {"input_cost_per_token": "not-a-number"}, +} + + +@pytest.fixture +def client(): + return TestClient(app) + + +class TestResolveModelPricing: + def test_provider_qualified_key_wins(self): + pricing = resolve_model_pricing("claude-x", "anthropic", COST_MAP) + assert pricing is not None + assert pricing.input_cost_per_token == 3e-06 + assert pricing.cache_read_cost_per_token == 3e-07 + assert pricing.cache_creation_cost_per_token == 3.75e-06 + + def test_bare_key_fallback(self): + pricing = resolve_model_pricing("gpt-x", "openai", COST_MAP) + assert pricing is not None + assert pricing.input_cost_per_token == 2e-06 + assert pricing.cache_creation_cost_per_token is None + + def test_provider_prefixed_stored_model_resolves_bare_key(self): + pricing = resolve_model_pricing("openai/gpt-x", "openai", COST_MAP) + assert pricing is not None + assert pricing.input_cost_per_token == 2e-06 + + def test_unknown_model_returns_none(self): + assert resolve_model_pricing("nope", "openai", COST_MAP) is None + + def test_zero_input_price_returns_none(self): + assert resolve_model_pricing("free-model", "", COST_MAP) is None + + def test_malformed_entry_returns_none(self): + assert resolve_model_pricing("malformed-model", "", COST_MAP) is None + + +class TestComputeSavingsAmounts: + def test_net_cache_savings_subtracts_write_premium(self): + pricing = resolve_model_pricing("claude-x", "anthropic", COST_MAP) + amounts = compute_savings_amounts( + cache_read_tokens=1000, cache_creation_tokens=100, compression_saved_tokens=0, pricing=pricing + ) + assert amounts.cache_savings == pytest.approx(1000 * (3e-06 - 3e-07) - 100 * (3.75e-06 - 3e-06)) + + def test_compression_savings_use_input_price(self): + pricing = resolve_model_pricing("gpt-x", "", COST_MAP) + amounts = compute_savings_amounts( + cache_read_tokens=0, cache_creation_tokens=0, compression_saved_tokens=500, pricing=pricing + ) + assert amounts.compression_savings == pytest.approx(500 * 2e-06) + + def test_missing_cache_read_price_yields_zero_cache_savings(self): + pricing = resolve_model_pricing("no-cache-price-model", "", COST_MAP) + amounts = compute_savings_amounts( + cache_read_tokens=1000, cache_creation_tokens=0, compression_saved_tokens=0, pricing=pricing + ) + assert amounts.cache_savings == 0.0 + + def test_none_pricing_yields_zero(self): + amounts = compute_savings_amounts( + cache_read_tokens=1000, cache_creation_tokens=10, compression_saved_tokens=500, pricing=None + ) + assert amounts.cache_savings == 0.0 + assert amounts.compression_savings == 0.0 + + +def _row(**overrides): + defaults = { + "date": "2026-07-15", + "model": "claude-x", + "custom_llm_provider": "anthropic", + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "spend": 0.0, + } + return _DailySavingsRow(**{**defaults, **overrides}) + + +class TestBuildActivityResponse: + def test_days_grouped_and_totals_summed(self): + rows = [ + _row(date="2026-07-15", cache_read_input_tokens=1000, spend=1.0), + _row(date="2026-07-15", model="gpt-x", custom_llm_provider="openai", compression_saved_tokens=500, spend=2.0), + _row(date="2026-07-16", cache_read_input_tokens=2000, cache_creation_input_tokens=100, spend=3.0), + ] + response = build_activity_response(rows, COST_MAP) + assert [daily.date for daily in response.results] == ["2026-07-15", "2026-07-16"] + day_one, day_two = response.results + assert day_one.metrics.cache_savings == pytest.approx(1000 * (3e-06 - 3e-07)) + assert day_one.metrics.compression_savings == pytest.approx(500 * 2e-06) + assert day_two.metrics.cache_savings == pytest.approx(2000 * (3e-06 - 3e-07) - 100 * 7.5e-07) + assert response.totals.spend == pytest.approx(6.0) + assert response.totals.total_savings == pytest.approx( + day_one.metrics.total_savings + day_two.metrics.total_savings + ) + assert response.totals.cache_read_input_tokens == 3000 + assert response.totals.compression_saved_tokens == 500 + assert response.unpriced_models == [] + + def test_unpriced_models_reported(self): + rows = [ + _row(model="mystery-model", custom_llm_provider="", cache_read_input_tokens=100), + _row(model="no-cache-price-model", custom_llm_provider="", cache_read_input_tokens=100), + _row(model="mystery-idle", custom_llm_provider="", spend=1.0), + ] + response = build_activity_response(rows, COST_MAP) + assert response.unpriced_models == ["mystery-model", "no-cache-price-model"] + assert response.totals.cache_savings == 0.0 + + +def _spend_log_row(metadata, **overrides): + defaults = { + "request_id": "req_1", + "startTime": datetime(2026, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + "model": "claude-x", + "custom_llm_provider": "anthropic", + "total_tokens": 1500, + "spend": 0.01, + "metadata": metadata, + } + return _SpendLogRow(**{**defaults, **overrides}) + + +class TestSummarizeOptimizedRequest: + def test_anthropic_style_cache_read(self): + row = _spend_log_row({"usage_object": {"cache_read_input_tokens": 1000}}) + summary = summarize_optimized_request(row, COST_MAP) + assert summary is not None + assert summary.optimizations == ["caching"] + assert summary.savings == pytest.approx(1000 * (3e-06 - 3e-07)) + assert summary.original_cost == pytest.approx(summary.optimized_cost + summary.savings) + + def test_openai_style_cached_tokens(self): + row = _spend_log_row( + {"usage_object": {"prompt_tokens_details": {"cached_tokens": 800}}}, + model="gpt-x", + custom_llm_provider="openai", + ) + summary = summarize_optimized_request(row, COST_MAP) + assert summary is not None + assert summary.optimizations == ["caching"] + assert summary.savings == pytest.approx(800 * (2e-06 - 1e-06)) + + def test_headroom_guardrail_compression(self): + row = _spend_log_row( + { + "usage_object": {}, + "guardrail_information": [ + {"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": 400}} + ], + } + ) + summary = summarize_optimized_request(row, COST_MAP) + assert summary is not None + assert summary.optimizations == ["compression"] + assert summary.savings == pytest.approx(400 * 3e-06) + + def test_compression_and_caching_both(self): + row = _spend_log_row( + { + "usage_object": {"cache_read_input_tokens": 1000}, + "compression_savings": {"tokens_saved": 600}, + } + ) + summary = summarize_optimized_request(row, COST_MAP) + assert summary is not None + assert summary.optimizations == ["caching", "compression"] + assert summary.savings == pytest.approx(1000 * (3e-06 - 3e-07) + 600 * 3e-06) + + def test_metadata_as_json_string(self): + row = _spend_log_row('{"usage_object": {"cache_read_input_tokens": 100}}') + summary = summarize_optimized_request(row, COST_MAP) + assert summary is not None + assert summary.optimizations == ["caching"] + + def test_unoptimized_request_returns_none(self): + assert summarize_optimized_request(_spend_log_row({"usage_object": {}}), COST_MAP) is None + assert summarize_optimized_request(_spend_log_row(None), COST_MAP) is None + + +def _override_auth(role, user_id="user-1"): + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role, user_id=user_id) + + +class TestActivityEndpoint: + def _setup(self, monkeypatch, rows): + mock_prisma = MagicMock() + mock_prisma.db.query_raw = AsyncMock(return_value=rows) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "model_cost", COST_MAP) + return mock_prisma + + def test_admin_gets_global_view(self, client, monkeypatch): + mock_prisma = self._setup( + monkeypatch, + [ + { + "date": "2026-07-15", + "model": "claude-x", + "custom_llm_provider": "anthropic", + "cache_read_input_tokens": 1000, + "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "spend": 1.0, + } + ], + ) + _override_auth(LitellmUserRoles.PROXY_ADMIN) + try: + response = client.get( + "/cost_savings/activity", params={"start_date": "2026-07-09", "end_date": "2026-07-15"} + ) + assert response.status_code == 200 + body = response.json() + assert body["totals"]["cache_savings"] == pytest.approx(1000 * (3e-06 - 3e-07)) + sql, *params = mock_prisma.db.query_raw.await_args.args + assert "user_id" not in sql + assert params == ["2026-07-09", "2026-07-15"] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + def test_non_admin_scoped_to_own_user_id(self, client, monkeypatch): + mock_prisma = self._setup(monkeypatch, []) + _override_auth(LitellmUserRoles.INTERNAL_USER, user_id="user-42") + try: + response = client.get( + "/cost_savings/activity", params={"start_date": "2026-07-09", "end_date": "2026-07-15"} + ) + assert response.status_code == 200 + sql, *params = mock_prisma.db.query_raw.await_args.args + assert "user_id = $3" in sql + assert params == ["2026-07-09", "2026-07-15", "user-42"] + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + def test_invalid_date_rejected(self, client, monkeypatch): + self._setup(monkeypatch, []) + _override_auth(LitellmUserRoles.PROXY_ADMIN) + try: + response = client.get( + "/cost_savings/activity", params={"start_date": "not-a-date", "end_date": "2026-07-15"} + ) + assert response.status_code == 400 + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +class TestRecentRequestsEndpoint: + def test_non_admin_scoped_and_filtered(self, client, monkeypatch): + optimized = { + "request_id": "req_hit", + "startTime": datetime(2026, 7, 15, 12, 0, 0, tzinfo=timezone.utc), + "model": "claude-x", + "custom_llm_provider": "anthropic", + "total_tokens": 1500, + "spend": 0.01, + "metadata": {"usage_object": {"cache_read_input_tokens": 1000}}, + } + plain = {**optimized, "request_id": "req_plain", "metadata": {"usage_object": {}}} + mock_prisma = MagicMock() + mock_prisma.db.litellm_spendlogs.find_many = AsyncMock( + return_value=[MagicMock(**optimized), MagicMock(**plain)] + ) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "model_cost", COST_MAP) + _override_auth(LitellmUserRoles.INTERNAL_USER, user_id="user-42") + try: + response = client.get( + "/cost_savings/recent_requests", + params={"start_date": "2026-07-09", "end_date": "2026-07-15"}, + ) + assert response.status_code == 200 + body = response.json() + assert [request["request_id"] for request in body["requests"]] == ["req_hit"] + assert body["scanned_requests"] == 2 + where = mock_prisma.db.litellm_spendlogs.find_many.await_args.kwargs["where"] + assert where["user"] == "user-42" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 9636db4f4cd..ac3a5c40e4c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3128,7 +3128,7 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens def test_extract_cache_read_tokens_anthropic_top_level(): - from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens + from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens usage_obj = { "prompt_tokens": 100, @@ -3136,34 +3136,34 @@ def test_extract_cache_read_tokens_anthropic_top_level(): "prompt_tokens_details": {"cached_tokens": 80}, } # Anthropic top-level value should win over prompt_tokens_details fallback. - assert _extract_cache_read_tokens(usage_obj) == 80 + assert extract_cache_read_tokens(usage_obj) == 80 def test_extract_cache_read_tokens_openai_compatible_fallback(): - from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens + from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens # Anthropic field absent — fall back to prompt_tokens_details.cached_tokens. usage_obj = { "prompt_tokens": 22583, "prompt_tokens_details": {"cached_tokens": 22016}, } - assert _extract_cache_read_tokens(usage_obj) == 22016 + assert extract_cache_read_tokens(usage_obj) == 22016 def test_extract_cache_read_tokens_zero_when_missing(): - from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens + from litellm.proxy.spend_tracking.cache_savings import extract_cache_read_tokens - assert _extract_cache_read_tokens({}) == 0 - assert _extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0 + assert extract_cache_read_tokens({}) == 0 + assert extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0 assert ( - _extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}}) + extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}}) == 0 ) def test_extract_cache_creation_tokens_anthropic_top_level(): - from litellm.proxy.db.db_spend_update_writer import ( - _extract_cache_creation_tokens, + from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, ) usage_obj = { @@ -3172,12 +3172,12 @@ def test_extract_cache_creation_tokens_anthropic_top_level(): "prompt_tokens_details": {"cache_write_tokens": 50}, } # Anthropic top-level should short-circuit the fallback. - assert _extract_cache_creation_tokens(usage_obj) == 50 + assert extract_cache_creation_tokens(usage_obj) == 50 def test_extract_cache_creation_tokens_openai_cache_write_alias(): - from litellm.proxy.db.db_spend_update_writer import ( - _extract_cache_creation_tokens, + from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, ) # kimi-k2 emits cache_write_tokens. @@ -3185,12 +3185,12 @@ def test_extract_cache_creation_tokens_openai_cache_write_alias(): "prompt_tokens": 1000, "prompt_tokens_details": {"cache_write_tokens": 200}, } - assert _extract_cache_creation_tokens(usage_obj) == 200 + assert extract_cache_creation_tokens(usage_obj) == 200 def test_extract_cache_creation_tokens_openai_cache_creation_alias(): - from litellm.proxy.db.db_spend_update_writer import ( - _extract_cache_creation_tokens, + from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, ) # Other OpenAI-compatible providers emit cache_creation_tokens. @@ -3198,18 +3198,18 @@ def test_extract_cache_creation_tokens_openai_cache_creation_alias(): "prompt_tokens": 1000, "prompt_tokens_details": {"cache_creation_tokens": 300}, } - assert _extract_cache_creation_tokens(usage_obj) == 300 + assert extract_cache_creation_tokens(usage_obj) == 300 def test_extract_cache_creation_tokens_zero_when_missing(): - from litellm.proxy.db.db_spend_update_writer import ( - _extract_cache_creation_tokens, + from litellm.proxy.spend_tracking.cache_savings import ( + extract_cache_creation_tokens, ) - assert _extract_cache_creation_tokens({}) == 0 - assert _extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0 + assert extract_cache_creation_tokens({}) == 0 + assert extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0 assert ( - _extract_cache_creation_tokens( + extract_cache_creation_tokens( {"prompt_tokens_details": {"cache_write_tokens": None}} ) == 0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.test.tsx new file mode 100644 index 00000000000..f12118160ad --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.test.tsx @@ -0,0 +1,156 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import * as networking from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import CostSavingsView, { formatUsd } from "./CostSavingsView"; + +function renderView() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + , + ); +} + +beforeAll(() => { + if (typeof window !== "undefined" && !window.ResizeObserver) { + window.ResizeObserver = class ResizeObserver { + observe() {} + unobserve() {} + disconnect() {} + } as any; + } +}); + +vi.mock("@/components/networking", () => ({ + costSavingsActivityCall: vi.fn(), + costSavingsRecentRequestsCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + __esModule: true, + default: vi.fn(), +})); + +vi.mock("@/components/shared/advanced_date_picker", async () => { + const React = await import("react"); + const AdvancedDatePicker = () => React.createElement("div", { "data-testid": "advanced-date-picker" }, "Date Picker"); + AdvancedDatePicker.displayName = "AdvancedDatePicker"; + return { default: AdvancedDatePicker }; +}); + +const ACTIVITY_RESPONSE: networking.CostSavingsActivityResponse = { + results: [ + { + date: "2026-07-15", + metrics: { + cache_savings: 1.5, + compression_savings: 0.5, + total_savings: 2.0, + spend: 10.0, + cache_read_input_tokens: 1000, + cache_creation_input_tokens: 0, + compression_saved_tokens: 250, + }, + }, + ], + totals: { + cache_savings: 1.5, + compression_savings: 0.5, + total_savings: 2.0, + spend: 10.0, + cache_read_input_tokens: 1000, + cache_creation_input_tokens: 0, + compression_saved_tokens: 250, + }, + unpriced_models: [], +}; + +const RECENT_RESPONSE: networking.RecentOptimizedRequestsResponse = { + requests: [ + { + request_id: "req_abc123", + start_time: "2026-07-15T12:00:00+00:00", + model: "claude-x", + total_tokens: 1500, + optimizations: ["caching", "compression"], + original_cost: 0.05, + optimized_cost: 0.02, + savings: 0.03, + }, + ], + scanned_requests: 42, +}; + +describe("formatUsd", () => { + it("formats zero, cents, and sub-cent values", () => { + expect(formatUsd(0)).toBe("$0"); + expect(formatUsd(12.345)).toBe("$12.35"); + expect(formatUsd(0.002625)).toBe("$0.002625"); + expect(formatUsd(1234.5)).toBe("$1,234.50"); + }); +}); + +describe("CostSavingsView", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(useAuthorized).mockReturnValue({ + accessToken: "sk-test", + token: "token", + userId: "user-1", + userRole: "Admin", + userEmail: null, + premiumUser: false, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + } as any); + vi.mocked(networking.costSavingsActivityCall).mockResolvedValue(ACTIVITY_RESPONSE); + vi.mocked(networking.costSavingsRecentRequestsCall).mockResolvedValue(RECENT_RESPONSE); + }); + + it("renders KPI totals from the activity response", async () => { + renderView(); + await waitFor(() => { + expect(screen.getAllByText("$2.00").length).toBeGreaterThan(0); + }); + expect(screen.getByText("Total Savings")).toBeInTheDocument(); + expect(screen.getAllByText("$1.50").length).toBeGreaterThan(0); + expect(screen.getAllByText("$0.50").length).toBeGreaterThan(0); + expect(screen.getAllByText("$10.00").length).toBeGreaterThan(0); + }); + + it("renders recent optimized requests with type badges and savings", async () => { + renderView(); + await waitFor(() => { + expect(screen.getByText("req_abc123")).toBeInTheDocument(); + }); + expect(screen.getByText("caching")).toBeInTheDocument(); + expect(screen.getByText("compression")).toBeInTheDocument(); + expect(screen.getByText("$0.03")).toBeInTheDocument(); + expect(screen.getByText(/scanned last 42 requests/)).toBeInTheDocument(); + }); + + it("shows a warning when models are missing prices", async () => { + vi.mocked(networking.costSavingsActivityCall).mockResolvedValue({ + ...ACTIVITY_RESPONSE, + unpriced_models: ["mystery-model"], + }); + renderView(); + await waitFor(() => { + expect(screen.getByText("Some models are missing prices")).toBeInTheDocument(); + }); + expect(screen.getByText(/mystery-model/)).toBeInTheDocument(); + }); + + it("shows the empty state when no requests were optimized", async () => { + vi.mocked(networking.costSavingsRecentRequestsCall).mockResolvedValue({ requests: [], scanned_requests: 0 }); + renderView(); + await waitFor(() => { + expect(screen.getByText(/No optimized requests in this window/)).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.tsx new file mode 100644 index 00000000000..8ae1dfec965 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/CostSavingsView.tsx @@ -0,0 +1,256 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { Alert, Card, Table, Tag, Typography } from "antd"; +import type { ColumnsType } from "antd/es/table"; +import { ComponentProps, useState } from "react"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import ChartLoader from "@/components/shared/chart_loader"; +import { AreaChart, CustomLegend, DonutChart, type ChartColor } from "@/components/shared/charts"; +import { + CostOptimizationType, + CostSavingsMetrics, + costSavingsActivityCall, + costSavingsRecentRequestsCall, + OptimizedRequestSummary, +} from "@/components/networking"; + +type DateRangeValue = ComponentProps["value"]; + +const SERIES_CATEGORIES = ["Caching", "Compression"] as const; +const SERIES_COLORS: readonly ChartColor[] = ["emerald", "blue"]; + +const OPTIMIZATION_TAG_COLOR: Record = { + caching: "green", + compression: "blue", +}; + +export function formatUsd(value: number): string { + if (value === 0) return "$0"; + const abs = Math.abs(value); + if (abs >= 0.01) { + return `$${value.toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; + } + return `$${value.toFixed(6).replace(/0+$/, "").replace(/\.$/, "")}`; +} + +function defaultDateRange(): DateRangeValue { + return { + from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + to: new Date(), + }; +} + +const RECENT_REQUEST_COLUMNS: ColumnsType = [ + { + title: "Request ID", + dataIndex: "request_id", + key: "request_id", + render: (value: string) => {value}, + }, + { title: "Model", dataIndex: "model", key: "model" }, + { + title: "Tokens", + dataIndex: "total_tokens", + key: "total_tokens", + render: (value: number) => value.toLocaleString(), + }, + { + title: "Type", + dataIndex: "optimizations", + key: "optimizations", + render: (optimizations: CostOptimizationType[]) => ( + + {optimizations.map((optimization) => ( + + {optimization} + + ))} + + ), + }, + { + title: "Original Cost", + dataIndex: "original_cost", + key: "original_cost", + align: "right", + render: (value: number) => {formatUsd(value)}, + }, + { + title: "Optimized Cost", + dataIndex: "optimized_cost", + key: "optimized_cost", + align: "right", + render: (value: number) => formatUsd(value), + }, + { + title: "Savings", + dataIndex: "savings", + key: "savings", + align: "right", + render: (value: number) => {formatUsd(value)}, + }, +]; + +interface SavingsKpiGridProps { + totals: CostSavingsMetrics | undefined; +} + +function SavingsKpiGrid({ totals }: SavingsKpiGridProps) { + return ( +
+ + + + +
+ ); +} + +export default function CostSavingsView() { + const { accessToken } = useAuthorized(); + const [dateValue, setDateValue] = useState(defaultDateRange); + + const startTime = dateValue.from; + const endTime = dateValue.to; + const rangeReady = Boolean(accessToken && startTime && endTime); + + const activityQuery = useQuery({ + queryKey: ["costSavingsActivity", accessToken, startTime?.toDateString(), endTime?.toDateString()], + queryFn: () => costSavingsActivityCall(accessToken!, startTime!, endTime!), + enabled: rangeReady, + }); + + const recentQuery = useQuery({ + queryKey: ["costSavingsRecentRequests", accessToken, startTime?.toDateString(), endTime?.toDateString()], + queryFn: () => costSavingsRecentRequestsCall(accessToken!, startTime!, endTime!), + enabled: rangeReady, + }); + + const totals = activityQuery.data?.totals; + const unpricedModels = activityQuery.data?.unpriced_models ?? []; + const chartData = + activityQuery.data?.results.map((day) => ({ + date: day.date, + Caching: day.metrics.cache_savings, + Compression: day.metrics.compression_savings, + })) ?? []; + const donutData = totals + ? [ + { name: "Caching", value: totals.cache_savings }, + { name: "Compression", value: totals.compression_savings }, + ] + : []; + const recentRequests = recentQuery.data?.requests ?? []; + + return ( +
+
+
+ + Cost Savings + + Savings from prompt caching and prompt compression +
+ +
+ + {unpricedModels.length > 0 && ( + + )} + + + +
+ + + Savings Over Time + + Daily savings by optimization type + {activityQuery.isLoading ? ( + + ) : ( + + )} + + + + Savings Distribution + + By optimization type + {activityQuery.isLoading ? ( + + ) : ( + <> + + + + )} + +
+ + +
+ + Recent Optimized Requests + + + Latest requests that benefited from caching or compression + {recentQuery.data ? ` (scanned last ${recentQuery.data.scanned_requests} requests in range)` : ""} + +
+ {recentQuery.isLoading ? ( +
+ +
+ ) : ( + + )} + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/page.tsx new file mode 100644 index 00000000000..2868c98936a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-savings/page.tsx @@ -0,0 +1,9 @@ +"use client"; + +import CostSavingsView from "./CostSavingsView"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export default function CostSavingsPage() { + useAuthorized(); + return ; +} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 76bfe174d5a..323e2653b0a 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -44,6 +44,7 @@ import { Palette, PanelLeftClose, PanelLeftOpen, + PiggyBank, PlayCircle, Route, ScrollText, @@ -131,6 +132,13 @@ const menuGroups: MenuGroup[] = [ icon: , roles: rolesAllowedToViewWriteScopedPages, }, + { + key: "cost-savings", + page: "cost-savings", + label: "Cost Savings", + icon: , + roles: [...all_admin_roles, ...internalUserRoles], + }, { key: "agentic", page: "agentic", diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d44a491b840..ad73cf5e82f 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1448,6 +1448,91 @@ export const teamDailyActivityCall = async ( }); }; +export interface CostSavingsMetrics { + cache_savings: number; + compression_savings: number; + total_savings: number; + spend: number; + cache_read_input_tokens: number; + cache_creation_input_tokens: number; + compression_saved_tokens: number; +} + +export interface DailyCostSavings { + date: string; + metrics: CostSavingsMetrics; +} + +export interface CostSavingsActivityResponse { + results: DailyCostSavings[]; + totals: CostSavingsMetrics; + unpriced_models: string[]; +} + +export type CostOptimizationType = "caching" | "compression"; + +export interface OptimizedRequestSummary { + request_id: string; + start_time: string; + model: string; + total_tokens: number; + optimizations: CostOptimizationType[]; + original_cost: number; + optimized_cost: number; + savings: number; +} + +export interface RecentOptimizedRequestsResponse { + requests: OptimizedRequestSummary[]; + scanned_requests: number; +} + +export const costSavingsActivityCall = async ( + accessToken: string, + startTime: Date, + endTime: Date, +): Promise => { + /** + * Get daily cost savings from prompt caching and prompt compression + */ + try { + return await apiClient.get(`/cost_savings/activity`, { + accessToken, + query: { + start_date: formatDate(startTime), + end_date: formatDate(endTime), + }, + }); + } catch (error) { + console.error("Failed to fetch cost savings activity:", error); + throw error; + } +}; + +export const costSavingsRecentRequestsCall = async ( + accessToken: string, + startTime: Date, + endTime: Date, + limit: number = 20, +): Promise => { + /** + * Get recent requests that benefited from prompt caching or prompt compression + */ + try { + return await apiClient.get(`/cost_savings/recent_requests`, { + accessToken, + query: { + start_date: formatDate(startTime), + end_date: formatDate(endTime), + limit, + }, + }); + } catch (error) { + console.error("Failed to fetch recent optimized requests:", error); + throw error; + } +}; + export const organizationDailyActivityCall = async ( accessToken: string, startTime: Date, diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 845e868b917..af93587eb7e 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -31,6 +31,7 @@ export const pageDescriptions: Record = { "model-hub-table": "Explore available AI models and providers", "learning-resources": "Access tutorials and documentation", caching: "Configure response caching and coordination Redis settings", + "cost-savings": "Track cost savings from prompt caching and prompt compression", "transform-request": "Set up request transformation rules", "cost-tracking": "Track and analyze API costs", "ui-theme": "Customize dashboard appearance", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5727eb5d2b8..096d2f5c1cb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2474,6 +2474,40 @@ export interface paths { patch?: never; trace?: never; }; + "/cost_savings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** cost_savings */ + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/credentials": { parameters: { query?: never; diff --git a/ui/litellm-dashboard/src/utils/migratedPages.ts b/ui/litellm-dashboard/src/utils/migratedPages.ts index b0dfb46e312..89485350ad8 100644 --- a/ui/litellm-dashboard/src/utils/migratedPages.ts +++ b/ui/litellm-dashboard/src/utils/migratedPages.ts @@ -34,6 +34,7 @@ export const MIGRATED_PAGES: Record = { // Legacy alias: the old switch matched ?page=claude-code-plugins for the same panel. "claude-code-plugins": "skills", caching: "caching", + "cost-savings": "cost-savings", "cost-tracking": "cost-tracking", "transform-request": "transform-request", "ui-theme": "ui-theme",