scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

feat(spend): aggregate compression and prompt caching dollar savings in daily rollups

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-07-18 22:21:58 +00:00
parent be609cff3a
commit 710d1d8d7d
15 changed files with 310 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,23 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

12
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

4
litellm/proxy/_types.py
View file

@ -4465,6 +4465,10 @@ class BaseDailySpendTransaction(TypedDict):
cache_creation_input_tokens: int
compression_saved_tokens: int
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
# request level metrics
spend: float
api_requests: int

32
litellm/proxy/db/db_spend_update_writer.py
View file

@ -62,6 +62,7 @@ from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
from litellm.proxy.spend_tracking.savings import compute_savings_spend
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
if TYPE_CHECKING:
@ -1561,6 +1562,14 @@ class DBSpendUpdateWriter:
common_data["compression_saved_tokens"] = transaction.get(
"compression_saved_tokens", 0
)
if "compression_savings_spend" in transaction:
common_data["compression_savings_spend"] = transaction.get(
"compression_savings_spend", 0
)
if "prompt_caching_savings_spend" in transaction:
common_data["prompt_caching_savings_spend"] = transaction.get(
"prompt_caching_savings_spend", 0
)
if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")
@ -1588,6 +1597,14 @@ class DBSpendUpdateWriter:
update_data["compression_saved_tokens"] = {
"increment": transaction.get("compression_saved_tokens", 0)
}
if "compression_savings_spend" in transaction:
update_data["compression_savings_spend"] = {
"increment": transaction.get("compression_savings_spend", 0)
}
if "prompt_caching_savings_spend" in transaction:
update_data["prompt_caching_savings_spend"] = {
"increment": transaction.get("prompt_caching_savings_spend", 0)
}
if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")
@ -1837,6 +1854,15 @@ class DBSpendUpdateWriter:
if call_type:
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
cache_read_input_tokens = _extract_cache_read_tokens(usage_obj)
compression_saved_tokens = extract_compression_saved_tokens(_metadata)
savings_spend = compute_savings_spend(
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
cache_read_input_tokens=cache_read_input_tokens,
)
daily_transaction = BaseDailySpendTransaction(
date=date,
api_key=payload["api_key"],
@ -1851,9 +1877,11 @@ class DBSpendUpdateWriter:
api_requests=1,
successful_requests=1 if request_status == "success" else 0,
failed_requests=1 if request_status != "success" else 0,
cache_read_input_tokens=_extract_cache_read_tokens(usage_obj),
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
compression_saved_tokens=extract_compression_saved_tokens(_metadata),
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
)
return daily_transaction
except Exception as e:

8
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -126,6 +126,14 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("compression_saved_tokens", 0) or 0
) + daily_transaction.get("compression_saved_tokens", 0)
daily_transaction["compression_savings_spend"] = (
payload.get("compression_savings_spend", 0) or 0
) + daily_transaction.get("compression_savings_spend", 0)
daily_transaction["prompt_caching_savings_spend"] = (
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions

10
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -50,6 +50,8 @@ def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics:
existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0
existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens or 0
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
existing_metrics.failed_requests += record.failed_requests or 0
@ -475,6 +477,8 @@ def _build_aggregated_sql_query(
SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens,
SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens,
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests
@ -615,6 +619,8 @@ def _record_to_spend_metrics(record: Any) -> SpendMetrics:
cache_read_input_tokens=record.cache_read_input_tokens or 0,
cache_creation_input_tokens=record.cache_creation_input_tokens or 0,
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
failed_requests=record.failed_requests or 0,
@ -866,6 +872,8 @@ async def get_daily_activity(
total_cache_read_input_tokens=metadata_metrics.cache_read_input_tokens,
total_cache_creation_input_tokens=metadata_metrics.cache_creation_input_tokens,
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count,
@ -953,6 +961,8 @@ async def get_daily_activity_aggregated(
total_cache_read_input_tokens=aggregated["totals"].cache_read_input_tokens,
total_cache_creation_input_tokens=aggregated["totals"].cache_creation_input_tokens,
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
page=1,
total_pages=1,
has_more=False,

12
litellm/proxy/schema.prisma
View file

@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

63
litellm/proxy/spend_tracking/savings.py Normal file
View file

@ -0,0 +1,63 @@
"""
Per-request cost-savings computation for the Cost Optimization dashboard.
Turns the token-level savings recorded on a request into dollar amounts using
the model's own pricing. Daily rollup rows are keyed by date and entity, not by
model, so the dollars have to be computed here (where the model and its prices
are known) and summed into the daily tables; tokens cannot be priced after they
have been aggregated across models.
"""
from typing import NamedTuple
import litellm
from litellm._logging import verbose_proxy_logger
class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]:
"""
Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model.
Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to
zero rather than raising inside the spend writer. When a model has no
separate cache-read price the cache-read cost mirrors the input cost, which
yields zero caching savings.
"""
if not model:
return 0.0, 0.0
try:
info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
verbose_proxy_logger.debug(
"savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e
)
return 0.0, 0.0
input_cost = float(info.get("input_cost_per_token") or 0.0)
cache_read_cost = info.get("cache_read_input_token_cost")
if cache_read_cost is None:
return input_cost, input_cost
return input_cost, float(cache_read_cost)
def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
compression_saved_tokens: int,
cache_read_input_tokens: int,
) -> SavingsSpend:
"""
Dollar savings for one request, split by optimization driver.
Compression savings price the tokens compression removed at the model's
input rate. Prompt-caching savings price the cache-read tokens at the
difference between the input rate and the discounted cache-read rate.
"""
input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
compression = max(compression_saved_tokens, 0) * input_cost
prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
return SavingsSpend(compression=compression, prompt_caching=prompt_caching)

6
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -23,6 +23,8 @@ class SpendMetrics(BaseModel):
cache_read_input_tokens: int = Field(default=0)
cache_creation_input_tokens: int = Field(default=0)
compression_saved_tokens: int = Field(default=0)
compression_savings_spend: float = Field(default=0.0)
prompt_caching_savings_spend: float = Field(default=0.0)
total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
failed_requests: int = Field(default=0)
@ -81,6 +83,8 @@ class DailySpendMetadata(BaseModel):
total_cache_read_input_tokens: int = Field(default=0)
total_cache_creation_input_tokens: int = Field(default=0)
total_compression_saved_tokens: int = Field(default=0)
total_compression_savings_spend: float = Field(default=0.0)
total_prompt_caching_savings_spend: float = Field(default=0.0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
has_more: bool = Field(default=False)
@ -105,6 +109,8 @@ class LiteLLM_DailyUserSpend(BaseModel):
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
compression_saved_tokens: int = 0
compression_savings_spend: float = 0.0
prompt_caching_savings_spend: float = 0.0
spend: float = 0.0
api_requests: int = 0
successful_requests: int = 0

12
schema.prisma
View file

@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend {
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

6
tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
View file

@ -494,6 +494,8 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
"cache_read_input_tokens": 7,
"cache_creation_input_tokens": 3,
"compression_saved_tokens": 7000,
"compression_savings_spend": 0.007,
"prompt_caching_savings_spend": 0.0063,
}
transaction2 = {
"spend": 2.0,
@ -505,6 +507,8 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
"cache_read_input_tokens": 5,
"cache_creation_input_tokens": 4,
"compression_saved_tokens": 600,
"compression_savings_spend": 0.0006,
"prompt_caching_savings_spend": 0.0045,
}
await daily_spend_update_queue.add_update({test_key: transaction1})
@ -517,3 +521,5 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
assert agg["compression_saved_tokens"] == 7600
assert agg["cache_read_input_tokens"] == 12
assert agg["cache_creation_input_tokens"] == 7
assert agg["compression_savings_spend"] == pytest.approx(0.0076)
assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108)

12
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -1959,6 +1959,16 @@ async def test_daily_transaction_carries_compression_saved_tokens():
assert transaction["cache_read_input_tokens"] == 40
assert transaction["cache_creation_input_tokens"] == 15
model_info = litellm.get_model_info(model="claude-sonnet-5", custom_llm_provider="anthropic")
input_cost = model_info["input_cost_per_token"] or 0.0
cache_read_cost = model_info.get("cache_read_input_token_cost") or input_cost
assert transaction["compression_savings_spend"] == pytest.approx(7600 * input_cost)
assert transaction["prompt_caching_savings_spend"] == pytest.approx(
40 * max(input_cost - cache_read_cost, 0.0)
)
assert transaction["compression_savings_spend"] > 0
assert transaction["prompt_caching_savings_spend"] > 0
@pytest.mark.asyncio
async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
@ -1990,3 +2000,5 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
assert transaction is not None
assert transaction["compression_saved_tokens"] == 0
assert transaction["compression_savings_spend"] == 0
assert transaction["prompt_caching_savings_spend"] == 0

14
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -151,6 +151,8 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"failed_requests": 0,
}
mock_rows = [
@ -494,6 +496,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.cache_read_input_tokens = 0
mock_record_1.cache_creation_input_tokens = 0
mock_record_1.compression_saved_tokens = 0
mock_record_1.compression_savings_spend = 0.0
mock_record_1.prompt_caching_savings_spend = 0.0
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
mock_record_1.failed_requests = 1
@ -514,6 +518,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.cache_read_input_tokens = 0
mock_record_2.cache_creation_input_tokens = 0
mock_record_2.compression_saved_tokens = 0
mock_record_2.compression_savings_spend = 0.0
mock_record_2.prompt_caching_savings_spend = 0.0
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
mock_record_2.failed_requests = 0
@ -574,6 +580,8 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"failed_requests": 0,
}
mock_rows = [
@ -659,6 +667,8 @@ def _daily_user_spend_record(*, user_id, api_key, spend):
cache_read_input_tokens=0,
cache_creation_input_tokens=0,
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
api_requests=1,
successful_requests=1,
failed_requests=0,
@ -871,6 +881,8 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"cache_read_input_tokens": None,
"cache_creation_input_tokens": None,
"compression_saved_tokens": None,
"compression_savings_spend": None,
"prompt_caching_savings_spend": None,
"api_requests": None,
"successful_requests": None,
"failed_requests": None,
@ -912,6 +924,8 @@ def _no_spend_record():
cache_read_input_tokens=None,
cache_creation_input_tokens=None,
compression_saved_tokens=None,
compression_savings_spend=None,
prompt_caching_savings_spend=None,
api_requests=None,
successful_requests=None,
failed_requests=None,

78
tests/test_litellm/proxy/spend_tracking/test_savings.py Normal file
View file

@ -0,0 +1,78 @@
import os
import sys
sys.path.insert(0, os.path.abspath("../../../.."))
import pytest
import litellm
from litellm.proxy.spend_tracking.savings import compute_savings_spend
def _anthropic_costs(model: str) -> tuple[float, float]:
info = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
input_cost = info["input_cost_per_token"] or 0.0
cache_read_cost = info.get("cache_read_input_token_cost") or input_cost
return input_cost, cache_read_cost
def test_compression_savings_priced_at_input_rate():
input_cost, _ = _anthropic_costs("claude-sonnet-5")
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=4389,
cache_read_input_tokens=0,
)
assert result.compression == pytest.approx(4389 * input_cost)
assert result.compression > 0
assert result.prompt_caching == 0.0
def test_prompt_caching_savings_priced_at_input_minus_cache_read():
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
# A model that supports prompt caching must charge less to read from cache;
# otherwise this test is asserting nothing.
assert cache_read_cost < input_cost
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
cache_read_input_tokens=8200,
)
assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost))
assert result.prompt_caching > 0
assert result.compression == 0.0
def test_unknown_model_fails_open_to_zero():
result = compute_savings_spend(
model="totally-made-up-model-xyz",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
cache_read_input_tokens=1000,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0
def test_missing_model_fails_open_to_zero():
result = compute_savings_spend(
model=None,
custom_llm_provider=None,
compression_saved_tokens=1000,
cache_read_input_tokens=1000,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0
def test_negative_token_counts_clamp_to_zero():
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=-500,
cache_read_input_tokens=-500,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0

20
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -23244,6 +23244,11 @@ export interface components {
* @default 0
*/
total_compression_saved_tokens: number;
/**
* Total Compression Savings Spend
* @default 0
*/
total_compression_savings_spend: number;
/**
* Total Failed Requests
* @default 0
@ -23254,6 +23259,11 @@ export interface components {
* @default 1
*/
total_pages: number;
/**
* Total Prompt Caching Savings Spend
* @default 0
*/
total_prompt_caching_savings_spend: number;
/**
* Total Prompt Tokens
* @default 0
@ -30874,11 +30884,21 @@ export interface components {
* @default 0
*/
compression_saved_tokens: number;
/**
* Compression Savings Spend
* @default 0
*/
compression_savings_spend: number;
/**
* Failed Requests
* @default 0
*/
failed_requests: number;
/**
* Prompt Caching Savings Spend
* @default 0
*/
prompt_caching_savings_spend: number;
/**
* Prompt Tokens
* @default 0