scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

feat(spend): track prompt compression saved tokens in daily spend aggregates (#33810)

Browse source 
* feat(spend): track prompt compression saved tokens in daily spend aggregates

Native compression interception now records tokens_before/after/saved into the
request litellm_metadata so savings land in the SpendLog metadata JSON under a
typed compression_savings key. A single normalizer
(extract_compression_saved_tokens) sums that key with Headroom guardrail
tokens_saved; the two writers are disjoint and run at different stages, so
summing never double-counts. The spend-log redactor now preserves purely
numeric compression stats inside guardrail_response so Headroom savings
survive the store_prompts_in_spend_logs=false default. compression_saved_tokens
is threaded through BaseDailySpendTransaction, queue aggregation, the daily
upsert blocks, a new BigInt column on all six daily spend tables, and the
daily activity read path (SpendMetrics, DailySpendMetadata, raw-SQL rollups)

* fix(spend): normalize legacy guardrail shapes and float token stats in compression savings reader

* feat(spend): aggregate compression and prompt caching dollar savings in daily rollups

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(spend): update daily spend aggregation fixtures for savings columns

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* feat(ui): add Cost Optimization dashboard page

New left-nav Cost Optimization page under Observability that surfaces money saved by prompt compression and prompt caching. It reads the daily activity rollup (userDailyActivityCall / get_daily_activity) and never scans SpendLogs, so it stays fast at 1M+ rows.

Renders a Total saved card, per-driver Compression and Prompt caching cards, a savings-over-time area chart, and a savings-by-driver donut, all aggregated in memory from the per-day metrics.compression_savings_spend and metrics.prompt_caching_savings_spend fields.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Krrish Dholakia <krrishdholakia@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
tin-berri 2026-07-18 17:47:54 -07:00 • committed by GitHub
parent 366ec6f487
commit 3f3295b33f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 1357 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql Normal file
View file

@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,23 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

18
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

42
litellm/integrations/compression_interception/handler.py
View file

@ -14,6 +14,7 @@ from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
CompressionSavingsMetadata,
)
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
@ -25,6 +26,41 @@ LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
_CACHE_TTL_SECONDS = 15 * 60
def _compression_savings_from_counts(
original_tokens: object, compressed_tokens: object
) -> CompressionSavingsMetadata | None:
if isinstance(original_tokens, bool) or not isinstance(original_tokens, int):
return None
if isinstance(compressed_tokens, bool) or not isinstance(compressed_tokens, int):
return None
if compressed_tokens < 0 or original_tokens < compressed_tokens:
return None
return CompressionSavingsMetadata(
tokens_before=original_tokens,
tokens_after=compressed_tokens,
tokens_saved=original_tokens - compressed_tokens,
source="compression_interception",
)
def _record_compression_savings(kwargs: dict[str, object], savings: CompressionSavingsMetadata) -> None:
"""
Attach savings to the request's litellm metadata so they land in the
SpendLog row's metadata JSON under ``compression_savings``.
``/v1/messages`` requests carry proxy metadata under ``litellm_metadata``
(the ``metadata`` key is Anthropic's own API field). The existing dict is
updated in place because the proxy and the logging object hold references
to the same object; replacing it would orphan writes made through those
references.
"""
existing = kwargs.get("litellm_metadata")
if isinstance(existing, dict):
existing["compression_savings"] = savings
return
kwargs["litellm_metadata"] = {"compression_savings": savings}
class CompressionInterceptionLogger(CustomLogger):
"""
CustomLogger that implements transparent prompt compression + retrieval loops.
@ -130,6 +166,12 @@ class CompressionInterceptionLogger(CustomLogger):
call_id = str(uuid.uuid4())
kwargs["litellm_call_id"] = call_id
self._compression_cache_by_call_id[call_id] = (cache, time.time())
savings = _compression_savings_from_counts(
original_tokens=compressed.get("original_tokens"),
compressed_tokens=compressed.get("compressed_tokens"),
)
if savings is not None:
_record_compression_savings(kwargs=kwargs, savings=savings)
verbose_logger.debug(
"CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
call_id,

9
litellm/proxy/_types.py
View file

@ -20,6 +20,9 @@ from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_no_callback_env_reference,
)
from litellm.types.integrations.compression_interception import (
CompressionSavingsMetadata,
)
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import (
AllMessageValues,
@ -3242,6 +3245,7 @@ class SpendLogsMetadata(TypedDict):
attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded)
max_retries: Optional[int] # Max retries configured for this request
cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
class SpendLogsPayload(TypedDict):
@ -4461,6 +4465,11 @@ class BaseDailySpendTransaction(TypedDict):
completion_tokens: int
cache_read_input_tokens: int
cache_creation_input_tokens: int
compression_saved_tokens: int
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
# request level metrics
spend: float

1
litellm/proxy/agent_endpoints/endpoints.py
View file

@ -1082,6 +1082,7 @@ async def get_agent_daily_activity(
total_failed_requests=0,
total_cache_read_input_tokens=0,
total_cache_creation_input_tokens=0,
total_compression_saved_tokens=0,
page=page,
total_pages=0,
has_more=False,

42
litellm/proxy/db/db_spend_update_writer.py
View file

@ -59,6 +59,10 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
ToolDiscoveryQueue,
)
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
from litellm.proxy.spend_tracking.savings import compute_savings_spend
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
if TYPE_CHECKING:
@ -1554,6 +1558,18 @@ class DBSpendUpdateWriter:
common_data["cache_creation_input_tokens"] = transaction.get(
"cache_creation_input_tokens", 0
)
if "compression_saved_tokens" in transaction:
common_data["compression_saved_tokens"] = transaction.get(
"compression_saved_tokens", 0
)
if "compression_savings_spend" in transaction:
common_data["compression_savings_spend"] = transaction.get(
"compression_savings_spend", 0
)
if "prompt_caching_savings_spend" in transaction:
common_data["prompt_caching_savings_spend"] = transaction.get(
"prompt_caching_savings_spend", 0
)
if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")
@ -1577,6 +1593,18 @@ class DBSpendUpdateWriter:
update_data["cache_creation_input_tokens"] = {
"increment": transaction.get("cache_creation_input_tokens", 0)
}
if "compression_saved_tokens" in transaction:
update_data["compression_saved_tokens"] = {
"increment": transaction.get("compression_saved_tokens", 0)
}
if "compression_savings_spend" in transaction:
update_data["compression_savings_spend"] = {
"increment": transaction.get("compression_savings_spend", 0)
}
if "prompt_caching_savings_spend" in transaction:
update_data["prompt_caching_savings_spend"] = {
"increment": transaction.get("prompt_caching_savings_spend", 0)
}
if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")
@ -1826,6 +1854,15 @@ class DBSpendUpdateWriter:
if call_type:
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
cache_read_input_tokens = _extract_cache_read_tokens(usage_obj)
compression_saved_tokens = extract_compression_saved_tokens(_metadata)
savings_spend = compute_savings_spend(
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
cache_read_input_tokens=cache_read_input_tokens,
)
daily_transaction = BaseDailySpendTransaction(
date=date,
api_key=payload["api_key"],
@ -1840,8 +1877,11 @@ class DBSpendUpdateWriter:
api_requests=1,
successful_requests=1 if request_status == "success" else 0,
failed_requests=1 if request_status != "success" else 0,
cache_read_input_tokens=_extract_cache_read_tokens(usage_obj),
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
)
return daily_transaction
except Exception as e:

12
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -122,6 +122,18 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("cache_creation_input_tokens", 0) or 0
) + daily_transaction.get("cache_creation_input_tokens", 0)
daily_transaction["compression_saved_tokens"] = (
payload.get("compression_saved_tokens", 0) or 0
) + daily_transaction.get("compression_saved_tokens", 0)
daily_transaction["compression_savings_spend"] = (
payload.get("compression_savings_spend", 0) or 0
) + daily_transaction.get("compression_savings_spend", 0)
daily_transaction["prompt_caching_savings_spend"] = (
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions

3
litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
View file

@ -11,6 +11,7 @@ from fastapi import HTTPException
import litellm
from httpx import Response as HttpxResponse
from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
from typing_extensions import TypeGuard
from litellm._logging import verbose_proxy_logger
@ -487,7 +488,7 @@ class HeadroomGuardrail(CustomGuardrail):
guardrail_json_response=stats,
request_data=request_data,
guardrail_status="success",
guardrail_provider="headroom",
guardrail_provider=HEADROOM_GUARDRAIL_PROVIDER,
start_time=start_time,
end_time=end_time,
duration=end_time - start_time,

15
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -49,6 +49,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics:
existing_metrics.total_tokens += prompt_tokens + completion_tokens
existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0
existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens or 0
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
existing_metrics.failed_requests += record.failed_requests or 0
@ -473,6 +476,9 @@ def _build_aggregated_sql_query(
SUM(completion_tokens)::bigint AS completion_tokens,
SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens,
SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens,
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests
@ -612,6 +618,9 @@ def _record_to_spend_metrics(record: Any) -> SpendMetrics:
total_tokens=prompt_tokens + completion_tokens,
cache_read_input_tokens=record.cache_read_input_tokens or 0,
cache_creation_input_tokens=record.cache_creation_input_tokens or 0,
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
failed_requests=record.failed_requests or 0,
@ -862,6 +871,9 @@ async def get_daily_activity(
total_failed_requests=metadata_metrics.failed_requests,
total_cache_read_input_tokens=metadata_metrics.cache_read_input_tokens,
total_cache_creation_input_tokens=metadata_metrics.cache_creation_input_tokens,
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count,
@ -948,6 +960,9 @@ async def get_daily_activity_aggregated(
total_failed_requests=aggregated["totals"].failed_requests,
total_cache_read_input_tokens=aggregated["totals"].cache_read_input_tokens,
total_cache_creation_input_tokens=aggregated["totals"].cache_creation_input_tokens,
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
page=1,
total_pages=1,
has_more=False,

18
litellm/proxy/schema.prisma
View file

@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

61
litellm/proxy/spend_tracking/compression_savings.py Normal file
View file

@ -0,0 +1,61 @@
"""
Single chokepoint for reading prompt-compression token savings out of a parsed
SpendLog ``metadata`` JSON dict. Imported by the daily-spend DB writer and by
cost-savings read endpoints.
"""
from collections.abc import Mapping
HEADROOM_GUARDRAIL_PROVIDER = "headroom"
def _saved_tokens_or_zero(value: object) -> int:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return 0
if value < 0:
return 0
return int(value)
def _tokens_saved_from_stats(stats: object) -> int:
if not isinstance(stats, Mapping):
return 0
return _saved_tokens_or_zero(stats.get("tokens_saved"))
def _headroom_entry_saved_tokens(entry: object) -> int:
if not isinstance(entry, Mapping):
return 0
if entry.get("guardrail_provider") != HEADROOM_GUARDRAIL_PROVIDER:
return 0
return _tokens_saved_from_stats(entry.get("guardrail_response"))
def _headroom_saved_tokens(guardrail_information: object) -> int:
entries = [guardrail_information] if isinstance(guardrail_information, Mapping) else guardrail_information
if not isinstance(entries, list):
return 0
return sum(_headroom_entry_saved_tokens(entry) for entry in entries)
def extract_compression_saved_tokens(metadata: Mapping[str, object]) -> int:
"""
Return the total prompt tokens saved by compression for one request.
Sums two disjoint sources:
- the native ``compression_savings`` key, written only by
``CompressionInterceptionLogger`` in its pre-call deployment hook
- ``guardrail_information`` entries with ``guardrail_provider ==
"headroom"``, written only by the Headroom guardrail
Each writer records only its own transform pass and the two run at
different stages (guardrail pre-call vs deployment pre-call), so when both
fire on one request their measured savings are independent and additive;
summing them never double-counts. Malformed or missing values contribute 0.
A bare dict ``guardrail_information`` is treated as a single entry, matching
the spend-log redactor's normalization of that legacy shape.
"""
return _tokens_saved_from_stats(metadata.get("compression_savings")) + _headroom_saved_tokens(
metadata.get("guardrail_information")
)

63
litellm/proxy/spend_tracking/savings.py Normal file
View file

@ -0,0 +1,63 @@
"""
Per-request cost-savings computation for the Cost Optimization dashboard.
Turns the token-level savings recorded on a request into dollar amounts using
the model's own pricing. Daily rollup rows are keyed by date and entity, not by
model, so the dollars have to be computed here (where the model and its prices
are known) and summed into the daily tables; tokens cannot be priced after they
have been aggregated across models.
"""
from typing import NamedTuple
import litellm
from litellm._logging import verbose_proxy_logger
class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]:
"""
Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model.
Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to
zero rather than raising inside the spend writer. When a model has no
separate cache-read price the cache-read cost mirrors the input cost, which
yields zero caching savings.
"""
if not model:
return 0.0, 0.0
try:
info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
verbose_proxy_logger.debug(
"savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e
)
return 0.0, 0.0
input_cost = float(info.get("input_cost_per_token") or 0.0)
cache_read_cost = info.get("cache_read_input_token_cost")
if cache_read_cost is None:
return input_cost, input_cost
return input_cost, float(cache_read_cost)
def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
compression_saved_tokens: int,
cache_read_input_tokens: int,
) -> SavingsSpend:
"""
Dollar savings for one request, split by optimization driver.
Compression savings price the tokens compression removed at the model's
input rate. Prompt-caching savings price the cache-read tokens at the
difference between the input rate and the discounted cache-read rate.
"""
input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
compression = max(compression_saved_tokens, 0) * input_cost
prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
return SavingsSpend(compression=compression, prompt_caching=prompt_caching)

34
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -115,6 +115,7 @@ def _get_spend_logs_metadata(
attempted_retries=None,
max_retries=None,
cost_breakdown=None,
compression_savings=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(
@ -909,14 +910,45 @@ _PROMPT_CARRYING_GUARDRAIL_FIELDS = (
"classification",
)
_NUMERIC_COMPRESSION_STAT_KEYS = (
"tokens_before",
"tokens_after",
"tokens_saved",
"compression_ratio",
)
def _numeric_compression_stats_from_guardrail_response(
guardrail_response: object,
) -> dict[str, int | float] | None:
if not isinstance(guardrail_response, dict):
return None
stats = {
key: value
for key, value in guardrail_response.items()
if key in _NUMERIC_COMPRESSION_STAT_KEYS and isinstance(value, (int, float)) and not isinstance(value, bool)
}
return stats or None
def _redact_prompt_fields_in_guardrail_entry(
entry: StandardLoggingGuardrailInformation,
) -> StandardLoggingGuardrailInformation:
return {
"""
Replace prompt-carrying fields with the redaction marker. Purely numeric
compression stats inside ``guardrail_response`` (e.g. Headroom's
``tokens_saved``) cannot carry prompt content, so they are preserved as a
stats-only dict; spend aggregation reads them via
``extract_compression_saved_tokens``.
"""
preserved_stats = _numeric_compression_stats_from_guardrail_response(entry.get("guardrail_response"))
redacted: StandardLoggingGuardrailInformation = {
**entry,
**{key: REDACTED_BY_LITELM_STRING for key in _PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry},
}
if preserved_stats is None:
return redacted
return {**redacted, "guardrail_response": preserved_stats}
def _sanitize_error_information_for_spend_logs(

14
litellm/types/integrations/compression_interception.py
View file

@ -2,7 +2,7 @@
Type definitions for Compression Interception integration.
"""
from typing import Any, Dict, Optional, TypedDict
from typing import Any, Dict, Literal, Optional, TypedDict
class CompressionInterceptionConfig(TypedDict, total=False):
@ -25,3 +25,15 @@ class CompressionInterceptionConfig(TypedDict, total=False):
compression_target: Optional[int]
embedding_model: Optional[str]
embedding_model_params: Optional[Dict[str, Any]]
class CompressionSavingsMetadata(TypedDict):
"""
Per-request prompt-compression savings recorded into the spend-log metadata
JSON so daily spend aggregates can track tokens saved by compression.
"""
tokens_before: int
tokens_after: int
tokens_saved: int
source: Literal["compression_interception"]

9
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -22,6 +22,9 @@ class SpendMetrics(BaseModel):
completion_tokens: int = Field(default=0)
cache_read_input_tokens: int = Field(default=0)
cache_creation_input_tokens: int = Field(default=0)
compression_saved_tokens: int = Field(default=0)
compression_savings_spend: float = Field(default=0.0)
prompt_caching_savings_spend: float = Field(default=0.0)
total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
failed_requests: int = Field(default=0)
@ -79,6 +82,9 @@ class DailySpendMetadata(BaseModel):
total_failed_requests: int = Field(default=0)
total_cache_read_input_tokens: int = Field(default=0)
total_cache_creation_input_tokens: int = Field(default=0)
total_compression_saved_tokens: int = Field(default=0)
total_compression_savings_spend: float = Field(default=0.0)
total_prompt_caching_savings_spend: float = Field(default=0.0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
has_more: bool = Field(default=False)
@ -102,6 +108,9 @@ class LiteLLM_DailyUserSpend(BaseModel):
completion_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
compression_saved_tokens: int = 0
compression_savings_spend: float = 0.0
prompt_caching_savings_spend: float = 0.0
spend: float = 0.0
api_requests: int = 0
successful_requests: int = 0

18
schema.prisma
View file

@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

127
tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py
View file

@ -396,3 +396,130 @@ async def test_build_agentic_loop_plan_missing_key_fallback():
plan.request_patch.messages[-1]["content"][0]["content"]
== "[compressed content key 'not_found.py' not found]"
)
def _stub_compress_result(original_tokens, compressed_tokens, cache):
return {
"messages": [{"role": "user", "content": "stubbed"}],
"original_tokens": original_tokens,
"compressed_tokens": compressed_tokens,
"compression_ratio": 0.5,
"cache": cache,
"tools": [],
}
@pytest.mark.asyncio
async def test_pre_call_hook_records_compression_savings_in_litellm_metadata(monkeypatch):
"""
When compression fires, the hook must record tokens_before/after/saved into
the request's litellm_metadata IN PLACE (the proxy and logging object hold
references to the same dict), so the savings land in the SpendLog row's
metadata JSON under ``compression_savings``.
"""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(12000, 5000, {"auth.py": "content"}),
)
litellm_metadata = {"user_api_key": "hashed-key", "user_api_key_user_id": "u1"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "very large context"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"] is litellm_metadata
assert litellm_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
assert litellm_metadata["user_api_key"] == "hashed-key"
@pytest.mark.asyncio
async def test_pre_call_hook_creates_litellm_metadata_when_absent(monkeypatch):
"""SDK-direct calls have no litellm_metadata dict yet; the hook creates it."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(300, 100, {"k": "v"}),
)
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"]["compression_savings"]["tokens_saved"] == 200
@pytest.mark.asyncio
@pytest.mark.parametrize(
"original_tokens,compressed_tokens",
[
(None, 5000),
(12000, None),
("12000", 5000),
(12000, "5000"),
(True, 5000),
(5000, 12000),
(12000, -1),
],
)
async def test_pre_call_hook_invalid_token_counts_fail_open(monkeypatch, original_tokens, compressed_tokens):
"""Invalid token counts must never crash the request; savings simply are not recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(original_tokens, compressed_tokens, {"k": "v"}),
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert "compression_savings" not in litellm_metadata
@pytest.mark.asyncio
async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
"""When compression is a no-op (empty cache) nothing is recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [],
"original_tokens": 100,
"compressed_tokens": 100,
"cache": {},
"tools": [],
"compression_skipped_reason": "below_trigger",
},
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "small"}],
"litellm_metadata": litellm_metadata,
}
await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert "compression_savings" not in litellm_metadata

51
tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
View file

@ -206,6 +206,9 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"failed_requests": 0, # 0 + 0
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
}
updates = [{test_key: test_transaction1}, {test_key: test_transaction2}]
@ -253,6 +256,9 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"failed_requests": 0, # 0 + 0
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
}
# Add updates to queue
@ -476,3 +482,48 @@ async def test_queue_size_reduction_with_large_volume(
assert result[user2_key]["api_requests"] == 100
assert result[user2_key]["successful_requests"] == 100
assert result[user2_key]["failed_requests"] == 0
@pytest.mark.asyncio
async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
"""compression_saved_tokens must accumulate across payloads for the same key."""
test_key = "user1_2023-01-01_key123_claude-sonnet-5_anthropic"
transaction1 = {
"spend": 1.0,
"prompt_tokens": 10,
"completion_tokens": 5,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
"cache_read_input_tokens": 7,
"cache_creation_input_tokens": 3,
"compression_saved_tokens": 7000,
"compression_savings_spend": 0.007,
"prompt_caching_savings_spend": 0.0063,
}
transaction2 = {
"spend": 2.0,
"prompt_tokens": 20,
"completion_tokens": 10,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
"cache_read_input_tokens": 5,
"cache_creation_input_tokens": 4,
"compression_saved_tokens": 600,
"compression_savings_spend": 0.0006,
"prompt_caching_savings_spend": 0.0045,
}
await daily_spend_update_queue.add_update({test_key: transaction1})
await daily_spend_update_queue.add_update({test_key: transaction2})
await daily_spend_update_queue.aggregate_queue_updates()
updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue()
assert len(updates) == 1
agg = updates[0][test_key]
assert agg["compression_saved_tokens"] == 7600
assert agg["cache_read_input_tokens"] == 12
assert agg["cache_creation_input_tokens"] == 7
assert agg["compression_savings_spend"] == pytest.approx(0.0076)
assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108)

100
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -1902,3 +1902,103 @@ async def test_update_user_db_enqueues_user_spend_without_cache_dependency():
by_type = {u["entity_type"]: u["entity_id"] for u in queued}
assert by_type[Litellm_EntityType.USER] == "user-123"
assert by_type[Litellm_EntityType.END_USER] == "end-user-9"
@pytest.mark.asyncio
async def test_daily_transaction_carries_compression_saved_tokens():
"""
The daily transaction built from a SpendLog payload must carry
compression_saved_tokens summed from both the native compression_savings
metadata key and Headroom guardrail entries, alongside the cache token
fields extracted from usage_object.
"""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
metadata = {
"usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15},
"compression_savings": {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
},
"guardrail_information": [
{
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600},
}
],
}
payload = {
"request_id": "req-compression-1",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
"spend": 0.05,
"metadata": json.dumps(metadata),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["compression_saved_tokens"] == 7600
assert transaction["cache_read_input_tokens"] == 40
assert transaction["cache_creation_input_tokens"] == 15
model_info = litellm.get_model_info(model="claude-sonnet-5", custom_llm_provider="anthropic")
input_cost = model_info["input_cost_per_token"] or 0.0
cache_read_cost = model_info.get("cache_read_input_token_cost") or input_cost
assert transaction["compression_savings_spend"] == pytest.approx(7600 * input_cost)
assert transaction["prompt_caching_savings_spend"] == pytest.approx(
40 * max(input_cost - cache_read_cost, 0.0)
)
assert transaction["compression_savings_spend"] > 0
assert transaction["prompt_caching_savings_spend"] > 0
@pytest.mark.asyncio
async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
"""Requests without any compression metadata produce a zero count."""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
payload = {
"request_id": "req-no-compression",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"call_type": "anthropic_messages",
"prompt_tokens": 100,
"completion_tokens": 10,
"spend": 0.01,
"metadata": json.dumps({"usage_object": {}}),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["compression_saved_tokens"] == 0
assert transaction["compression_savings_spend"] == 0
assert transaction["prompt_caching_savings_spend"] == 0

24
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -150,6 +150,9 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"mcp_namespaced_tool_name": None,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"failed_requests": 0,
}
mock_rows = [
@ -492,6 +495,9 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.completion_tokens = 200
mock_record_1.cache_read_input_tokens = 0
mock_record_1.cache_creation_input_tokens = 0
mock_record_1.compression_saved_tokens = 0
mock_record_1.compression_savings_spend = 0.0
mock_record_1.prompt_caching_savings_spend = 0.0
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
mock_record_1.failed_requests = 1
@ -511,6 +517,9 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.completion_tokens = 100
mock_record_2.cache_read_input_tokens = 0
mock_record_2.cache_creation_input_tokens = 0
mock_record_2.compression_saved_tokens = 0
mock_record_2.compression_savings_spend = 0.0
mock_record_2.prompt_caching_savings_spend = 0.0
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
mock_record_2.failed_requests = 0
@ -570,6 +579,9 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"mcp_namespaced_tool_name": None,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"failed_requests": 0,
}
mock_rows = [
@ -654,6 +666,9 @@ def _daily_user_spend_record(*, user_id, api_key, spend):
completion_tokens=5,
cache_read_input_tokens=0,
cache_creation_input_tokens=0,
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
api_requests=1,
successful_requests=1,
failed_requests=0,
@ -865,6 +880,9 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"completion_tokens": None,
"cache_read_input_tokens": None,
"cache_creation_input_tokens": None,
"compression_saved_tokens": None,
"compression_savings_spend": None,
"prompt_caching_savings_spend": None,
"api_requests": None,
"successful_requests": None,
"failed_requests": None,
@ -894,6 +912,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
assert result.metadata.total_failed_requests == 0
assert result.metadata.total_cache_read_input_tokens == 0
assert result.metadata.total_cache_creation_input_tokens == 0
assert result.metadata.total_compression_saved_tokens == 0
def _no_spend_record():
@ -904,6 +923,9 @@ def _no_spend_record():
completion_tokens=None,
cache_read_input_tokens=None,
cache_creation_input_tokens=None,
compression_saved_tokens=None,
compression_savings_spend=None,
prompt_caching_savings_spend=None,
api_requests=None,
successful_requests=None,
failed_requests=None,
@ -922,6 +944,7 @@ def test_record_to_spend_metrics_handles_none_values():
assert metrics.failed_requests == 0
assert metrics.cache_read_input_tokens == 0
assert metrics.cache_creation_input_tokens == 0
assert metrics.compression_saved_tokens == 0
def test_update_metrics_handles_none_values():
@ -936,3 +959,4 @@ def test_update_metrics_handles_none_values():
assert metrics.failed_requests == 0
assert metrics.cache_read_input_tokens == 0
assert metrics.cache_creation_input_tokens == 0
assert metrics.compression_saved_tokens == 0

131
tests/test_litellm/proxy/spend_tracking/test_compression_savings.py Normal file
View file

@ -0,0 +1,131 @@
"""
Unit tests for the compression-savings spend-log metadata normalizer.
"""
import pytest
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
NATIVE_SAVINGS = {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
HEADROOM_ENTRY = {
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600},
}
def test_native_key_only():
assert extract_compression_saved_tokens({"compression_savings": NATIVE_SAVINGS}) == 7000
def test_headroom_only():
assert extract_compression_saved_tokens({"guardrail_information": [HEADROOM_ENTRY]}) == 600
def test_native_and_headroom_sum():
metadata = {
"compression_savings": NATIVE_SAVINGS,
"guardrail_information": [HEADROOM_ENTRY],
}
assert extract_compression_saved_tokens(metadata) == 7600
def test_multiple_headroom_entries_sum():
metadata = {"guardrail_information": [HEADROOM_ENTRY, HEADROOM_ENTRY]}
assert extract_compression_saved_tokens(metadata) == 1200
def test_neither_source_present():
assert extract_compression_saved_tokens({"user_api_key": "abc", "usage_object": {}}) == 0
def test_non_headroom_guardrail_entries_ignored():
metadata = {
"guardrail_information": [
{
"guardrail_name": "pii-guard",
"guardrail_provider": "presidio",
"guardrail_response": {"tokens_saved": 999},
}
]
}
assert extract_compression_saved_tokens(metadata) == 0
@pytest.mark.parametrize(
"compression_savings",
[
None,
"not-a-dict",
{},
{"tokens_saved": None},
{"tokens_saved": "7000"},
{"tokens_saved": True},
{"tokens_saved": -5},
{"tokens_before": 100, "tokens_after": 50},
],
)
def test_malformed_native_key_contributes_zero(compression_savings):
assert extract_compression_saved_tokens({"compression_savings": compression_savings}) == 0
@pytest.mark.parametrize(
"guardrail_information",
[
None,
"not-a-list",
{"guardrail_provider": "headroom"},
[],
[None],
["not-a-dict"],
[{"guardrail_provider": "headroom"}],
[{"guardrail_provider": "headroom", "guardrail_response": "REDACTED"}],
[{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": "600"}}],
[{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": -600}}],
[{"guardrail_response": {"tokens_saved": 600}}],
],
)
def test_malformed_headroom_information_contributes_zero(guardrail_information):
assert extract_compression_saved_tokens({"guardrail_information": guardrail_information}) == 0
def test_valid_headroom_entry_survives_alongside_malformed_ones():
metadata = {
"guardrail_information": [
None,
{"guardrail_provider": "headroom", "guardrail_response": "REDACTED"},
HEADROOM_ENTRY,
]
}
assert extract_compression_saved_tokens(metadata) == 600
def test_float_tokens_saved_counts_as_int():
entry = {"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": 600.0}}
assert extract_compression_saved_tokens({"guardrail_information": [entry]}) == 600
assert extract_compression_saved_tokens({"compression_savings": {"tokens_saved": 7000.0}}) == 7000
assert extract_compression_saved_tokens({"compression_savings": {"tokens_saved": 12.5}}) == 12
def test_bare_dict_guardrail_information_counts_as_single_entry():
assert extract_compression_saved_tokens({"guardrail_information": HEADROOM_ENTRY}) == 600
def test_headroom_writer_and_reader_share_provider_slug():
from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
HEADROOM_GUARDRAIL_PROVIDER as writer_slug,
)
from litellm.proxy.spend_tracking.compression_savings import (
HEADROOM_GUARDRAIL_PROVIDER as reader_slug,
)
assert writer_slug == reader_slug == "headroom"

78
tests/test_litellm/proxy/spend_tracking/test_savings.py Normal file
View file

@ -0,0 +1,78 @@
import os
import sys
sys.path.insert(0, os.path.abspath("../../../.."))
import pytest
import litellm
from litellm.proxy.spend_tracking.savings import compute_savings_spend
def _anthropic_costs(model: str) -> tuple[float, float]:
info = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
input_cost = info["input_cost_per_token"] or 0.0
cache_read_cost = info.get("cache_read_input_token_cost") or input_cost
return input_cost, cache_read_cost
def test_compression_savings_priced_at_input_rate():
input_cost, _ = _anthropic_costs("claude-sonnet-5")
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=4389,
cache_read_input_tokens=0,
)
assert result.compression == pytest.approx(4389 * input_cost)
assert result.compression > 0
assert result.prompt_caching == 0.0
def test_prompt_caching_savings_priced_at_input_minus_cache_read():
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
# A model that supports prompt caching must charge less to read from cache;
# otherwise this test is asserting nothing.
assert cache_read_cost < input_cost
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
cache_read_input_tokens=8200,
)
assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost))
assert result.prompt_caching > 0
assert result.compression == 0.0
def test_unknown_model_fails_open_to_zero():
result = compute_savings_spend(
model="totally-made-up-model-xyz",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
cache_read_input_tokens=1000,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0
def test_missing_model_fails_open_to_zero():
result = compute_savings_spend(
model=None,
custom_llm_provider=None,
compression_saved_tokens=1000,
cache_read_input_tokens=1000,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0
def test_negative_token_counts_clamp_to_zero():
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=-500,
cache_read_input_tokens=-500,
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -1998,7 +1998,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -2094,7 +2094,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -2188,7 +2188,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

131
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -2634,3 +2634,134 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key():
assert not metadata_dict["user_api_key"].startswith("sk-"), (
f"metadata user_api_key contains unhashed key: {metadata_dict['user_api_key']}"
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats(
mock_should_store,
):
"""
Headroom's compression stats live in guardrail_response, which is redacted
by default. Purely numeric token stats carry no prompt content and must
survive so daily spend aggregation can count compression_saved_tokens;
everything else in guardrail_response stays redacted.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
"transforms_applied": ["dedupe_messages"],
},
},
{
"guardrail_name": "echo-guard",
"guardrail_status": "success",
"guardrail_response": {
"evaluated_input": "secret prompt",
"tokens_saved": "prompt text hiding in a stat key",
},
},
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
assert result[0]["guardrail_response"] == {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
}
assert result[1]["guardrail_response"] == REDACTED_BY_LITELM_STRING
@pytest.mark.asyncio
async def test_compression_savings_survive_to_spend_log_payload_metadata(monkeypatch):
"""
End-to-end seam test for the native compression write path on
/v1/messages: the pre-call deployment hook records savings into the call
kwargs' litellm_metadata, the logging object captures it via
update_from_kwargs (exactly as llm_http_handler does for
anthropic_messages), and get_logging_payload lands it in
SpendLogsPayload.metadata JSON under ``compression_savings``.
"""
from litellm.integrations.compression_interception.handler import (
CompressionInterceptionLogger,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CallTypes
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [{"role": "user", "content": "compressed"}],
"original_tokens": 12000,
"compressed_tokens": 5000,
"cache": {"auth.py": "content"},
"tools": [],
},
)
logger = CompressionInterceptionLogger()
call_kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "big context"}],
"max_tokens": 512,
"litellm_call_id": "test-compression-call-id",
"litellm_metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key_user_id": "u1",
"user_api_key_team_id": "t1",
},
}
hooked = await logger.async_pre_call_deployment_hook(kwargs=call_kwargs, call_type=CallTypes.anthropic_messages)
assert hooked is not None
start_time = datetime.datetime.now(timezone.utc)
logging_obj = Logging(
model="claude-sonnet-5",
messages=hooked["messages"],
stream=False,
call_type="anthropic_messages",
start_time=start_time,
litellm_call_id="test-compression-call-id",
function_id="test",
)
logging_obj.update_from_kwargs(
kwargs=hooked,
model="claude-sonnet-5",
optional_params={"max_tokens": 512},
litellm_params={
"preset_cache_key": None,
"stream_response": {},
"model_info": hooked.get("model_info"),
},
custom_llm_provider="anthropic",
)
end_time = datetime.datetime.now(timezone.utc)
logging_obj.model_call_details["completion_start_time"] = end_time
payload = get_logging_payload(
kwargs=logging_obj.model_call_details,
response_obj={
"id": "msg_test",
"usage": {"prompt_tokens": 5000, "completion_tokens": 10, "total_tokens": 5010},
},
start_time=start_time,
end_time=end_time,
)
payload_metadata = json.loads(payload["metadata"])
assert payload_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}

109
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx Normal file
View file

@ -0,0 +1,109 @@
import { render } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
const mockUsePaginatedDailyActivity = vi.fn();
vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({
usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args),
}));
vi.mock("@/components/networking", () => ({
userDailyActivityCall: vi.fn(),
}));
vi.mock("@/components/shared/advanced_date_picker", () => ({
__esModule: true,
default: () => <div data-testid="date-picker" />,
}));
vi.mock("@/components/shared/charts", () => ({
AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => (
<div data-testid="area-chart" data-categories={categories.join(",")} data-series={JSON.stringify(data)} />
),
DonutChart: ({ data, label }: { data: unknown; label: string }) => (
<div data-testid="donut-chart" data-label={label} data-slices={JSON.stringify(data)} />
),
}));
import CostOptimizationView from "./CostOptimizationView";
const baseMetrics = (overrides: Partial<SpendMetrics>): SpendMetrics => ({
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
...overrides,
});
const day = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
date,
metrics: baseMetrics(metrics),
breakdown: {
models: {},
model_groups: {},
mcp_servers: {},
providers: {},
api_keys: {},
entities: {},
},
});
const renderWith = (results: DailyData[]) => {
mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false });
return render(<CostOptimizationView accessToken="test-token" userId="u1" userRole="proxy_admin" />);
};
describe("CostOptimizationView", () => {
it("sums compression and caching dollars across days into the summary cards", () => {
const { getByText } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
compression_saved_tokens: 40000,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
compression_saved_tokens: 100000,
}),
]);
// compression 0.14 + caching 0.016 = 0.156
expect(getByText("$0.1560")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
expect(getByText("$0.0160")).toBeInTheDocument();
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
});
it("builds a per-day time series and per-driver donut from the daily rows", () => {
const { getByTestId } = renderWith([
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
]);
const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]");
expect(series).toHaveLength(2);
expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 });
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 });
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
]);
});
it("omits a driver slice when that driver has no savings", () => {
const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]);
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
});
});

157
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx Normal file
View file

@ -0,0 +1,157 @@
"use client";
import React, { useMemo, useState } from "react";
import { PiggyBank } from "lucide-react";
import { AreaChart, DonutChart } from "@/components/shared/charts";
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { userDailyActivityCall } from "@/components/networking";
import { DailyData, SpendMetrics } from "@/components/UsagePage/types";
import { formatNumberWithCommas } from "@/utils/dataUtils";
import { all_admin_roles } from "@/utils/roles";
import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity";
interface CostOptimizationViewProps {
accessToken: string | null;
userId: string | null;
userRole: string;
}
type DateRange = { from?: Date; to?: Date };
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
const usd = (value: number): string => {
const decimals = value > 0 && value < 1 ? 4 : 2;
return `$${formatNumberWithCommas(value, decimals)}`;
};
const shortDate = (iso: string): string =>
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => (
<Card>
<CardHeader>
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
</CardHeader>
<CardContent>
<p className="text-2xl font-semibold text-foreground">{value}</p>
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
</CardContent>
</Card>
);
const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken, userId, userRole }) => {
const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []);
const initialTo = useMemo(() => new Date(), []);
const [dateValue, setDateValue] = useState<DateRange>({ from: initialFrom, to: initialTo });
const startTime = dateValue.from ?? null;
const endTime = dateValue.to ?? null;
const isAdmin = all_admin_roles.includes(userRole);
const effectiveUserId = isAdmin ? null : userId;
const { data, loading, isFetchingMore } = usePaginatedDailyActivity({
fetchFn: userDailyActivityCall,
args: [accessToken, startTime, endTime, effectiveUserId],
enabled: !!accessToken && !!startTime && !!endTime,
});
const results = data.results as DailyData[];
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
const totalSaved = compressionTotal + cachingTotal;
const overTime = useMemo(
() =>
results.map((d) => ({
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
})),
[results],
);
const byDriver = useMemo(
() =>
[
{ driver: "Compression", usd: compressionTotal },
{ driver: "Prompt caching", usd: cachingTotal },
].filter((d) => d.usd > 0),
[compressionTotal, cachingTotal],
);
return (
<div className="w-full space-y-6 p-6">
<div className="flex flex-wrap items-start justify-between gap-4">
<div>
<div className="flex items-center gap-2">
<PiggyBank className="size-6 text-emerald-600" strokeWidth={1.75} />
<h1 className="text-xl font-semibold text-foreground">Cost Optimization</h1>
</div>
<p className="mt-1 text-sm text-muted-foreground">
Money saved by prompt compression and prompt caching across your requests
</p>
</div>
<AdvancedDatePicker value={dateValue} onValueChange={(v) => setDateValue(v)} />
</div>
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<SummaryCard
label="Total saved"
value={usd(totalSaved)}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
/>
<SummaryCard
label="Compression savings"
value={usd(compressionTotal)}
hint={`${formatNumberWithCommas(savedTokensTotal)} tokens compressed`}
/>
<SummaryCard label="Prompt caching savings" value={usd(cachingTotal)} hint="Cache read discount" />
</div>
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
<Card className="lg:col-span-2">
<CardHeader>
<CardTitle>Savings over time</CardTitle>
</CardHeader>
<CardContent>
<AreaChart
data={overTime}
index="date"
categories={["Compression", "Prompt caching"]}
colors={["emerald", "blue"]}
valueFormatter={usd}
/>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Savings by driver</CardTitle>
</CardHeader>
<CardContent>
<DonutChart
className="h-80"
data={byDriver}
index="driver"
category="usd"
colors={["emerald", "blue"]}
valueFormatter={usd}
showLabel
label={usd(totalSaved)}
/>
</CardContent>
</Card>
</div>
</div>
);
};
export default CostOptimizationView;

9
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/page.tsx Normal file
View file

@ -0,0 +1,9 @@
"use client";
import CostOptimizationView from "./_components/CostOptimizationView";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
export default function CostOptimizationPage() {
const { accessToken, userId, userRole } = useAuthorized();
return <CostOptimizationView accessToken={accessToken} userId={userId} userRole={userRole} />;
}

3
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -8,6 +8,9 @@ export interface SpendMetrics {
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
compression_saved_tokens?: number;
compression_savings_spend?: number;
prompt_caching_savings_spend?: number;
}
export type DailyData = {

8
ui/litellm-dashboard/src/components/leftnav.tsx
View file

@ -44,6 +44,7 @@ import {
Palette,
PanelLeftClose,
PanelLeftOpen,
PiggyBank,
PlayCircle,
Route,
ScrollText,
@ -181,6 +182,13 @@ const menuGroups: MenuGroup[] = [
roles: [...all_admin_roles, ...internalUserRoles],
label: "Usage",
},
{
key: "cost-optimization",
page: "cost-optimization",
icon: <PiggyBank {...ICON} />,
roles: [...all_admin_roles, ...internalUserRoles],
label: "Cost Optimization",
},
{ key: "logs", page: "logs", label: "Logs", icon: <Activity {...ICON} /> },
{
key: "guardrails-monitor",

30
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -23243,6 +23243,16 @@ export interface components {
* @default 0
*/
total_completion_tokens: number;
/**
* Total Compression Saved Tokens
* @default 0
*/
total_compression_saved_tokens: number;
/**
* Total Compression Savings Spend
* @default 0
*/
total_compression_savings_spend: number;
/**
* Total Failed Requests
* @default 0
@ -23253,6 +23263,11 @@ export interface components {
* @default 1
*/
total_pages: number;
/**
* Total Prompt Caching Savings Spend
* @default 0
*/
total_prompt_caching_savings_spend: number;
/**
* Total Prompt Tokens
* @default 0
@ -30868,11 +30883,26 @@ export interface components {
* @default 0
*/
completion_tokens: number;
/**
* Compression Saved Tokens
* @default 0
*/
compression_saved_tokens: number;
/**
* Compression Savings Spend
* @default 0
*/
compression_savings_spend: number;
/**
* Failed Requests
* @default 0
*/
failed_requests: number;
/**
* Prompt Caching Savings Spend
* @default 0
*/
prompt_caching_savings_spend: number;
/**
* Prompt Tokens
* @default 0

1
ui/litellm-dashboard/src/utils/migratedPages.ts
View file

@ -44,6 +44,7 @@ export const MIGRATED_PAGES: Record<string, string> = {
// The modern usage dashboard; the legacy ?page=usage report routes to /old-usage.
new_usage: "usage",
usage: "old-usage",
"cost-optimization": "cost-optimization",
agents: "agents",
"router-settings": "router-settings",
users: "users",