From 3f3295b33ff77a986ca729be26c279d7f25d5ed5 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Sat, 18 Jul 2026 17:47:54 -0700 Subject: [PATCH] feat(spend): track prompt compression saved tokens in daily spend aggregates (#33810) * feat(spend): track prompt compression saved tokens in daily spend aggregates Native compression interception now records tokens_before/after/saved into the request litellm_metadata so savings land in the SpendLog metadata JSON under a typed compression_savings key. A single normalizer (extract_compression_saved_tokens) sums that key with Headroom guardrail tokens_saved; the two writers are disjoint and run at different stages, so summing never double-counts. The spend-log redactor now preserves purely numeric compression stats inside guardrail_response so Headroom savings survive the store_prompts_in_spend_logs=false default. compression_saved_tokens is threaded through BaseDailySpendTransaction, queue aggregation, the daily upsert blocks, a new BigInt column on all six daily spend tables, and the daily activity read path (SpendMetrics, DailySpendMetadata, raw-SQL rollups) * fix(spend): normalize legacy guardrail shapes and float token stats in compression savings reader * feat(spend): aggregate compression and prompt caching dollar savings in daily rollups Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): update daily spend aggregation fixtures for savings columns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): add Cost Optimization dashboard page New left-nav Cost Optimization page under Observability that surfaces money saved by prompt compression and prompt caching. It reads the daily activity rollup (userDailyActivityCall / get_daily_activity) and never scans SpendLogs, so it stays fast at 1M+ rows. Renders a Total saved card, per-driver Compression and Prompt caching cards, a savings-over-time area chart, and a savings-by-driver donut, all aggregated in memory from the per-day metrics.compression_savings_spend and metrics.prompt_caching_savings_spend fields. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Krrish Dholakia Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../migration.sql | 17 ++ .../migration.sql | 23 +++ .../litellm_proxy_extras/schema.prisma | 18 ++ .../compression_interception/handler.py | 42 +++++ litellm/proxy/_types.py | 9 + litellm/proxy/agent_endpoints/endpoints.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 42 ++++- .../daily_spend_update_queue.py | 12 ++ .../guardrail_hooks/headroom/headroom.py | 3 +- .../common_daily_activity.py | 15 ++ litellm/proxy/schema.prisma | 18 ++ .../spend_tracking/compression_savings.py | 61 +++++++ litellm/proxy/spend_tracking/savings.py | 63 +++++++ .../spend_tracking/spend_tracking_utils.py | 34 +++- .../integrations/compression_interception.py | 14 +- .../common_daily_activity.py | 9 + schema.prisma | 18 ++ .../test_compression_interception_handler.py | 127 ++++++++++++++ .../test_daily_spend_update_queue.py | 51 ++++++ .../proxy/db/test_db_spend_update_writer.py | 100 +++++++++++ .../test_common_daily_activity.py | 24 +++ .../test_compression_savings.py | 131 +++++++++++++++ .../proxy/spend_tracking/test_savings.py | 78 +++++++++ .../test_spend_management_endpoints.py | 6 +- .../test_spend_tracking_utils.py | 131 +++++++++++++++ .../_components/CostOptimizationView.test.tsx | 109 ++++++++++++ .../_components/CostOptimizationView.tsx | 157 ++++++++++++++++++ .../(dashboard)/cost-optimization/page.tsx | 9 + .../src/components/UsagePage/types.ts | 3 + .../src/components/leftnav.tsx | 8 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 30 ++++ .../src/utils/migratedPages.ts | 1 + 32 files changed, 1357 insertions(+), 7 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql create mode 100644 litellm/proxy/spend_tracking/compression_savings.py create mode 100644 litellm/proxy/spend_tracking/savings.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_compression_savings.py create mode 100644 tests/test_litellm/proxy/spend_tracking/test_savings.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/page.tsx diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql new file mode 100644 index 00000000000..dff889bc7fb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql new file mode 100644 index 00000000000..f4cca662850 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql @@ -0,0 +1,23 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index a99cec49417..b27ddea010b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/integrations/compression_interception/handler.py b/litellm/integrations/compression_interception/handler.py index c82f9ff477f..f0a696aa1e1 100644 --- a/litellm/integrations/compression_interception/handler.py +++ b/litellm/integrations/compression_interception/handler.py @@ -14,6 +14,7 @@ from litellm.compression import compress from litellm.integrations.custom_logger import CustomLogger from litellm.types.integrations.compression_interception import ( CompressionInterceptionConfig, + CompressionSavingsMetadata, ) from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, @@ -25,6 +26,41 @@ LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve" _CACHE_TTL_SECONDS = 15 * 60 +def _compression_savings_from_counts( + original_tokens: object, compressed_tokens: object +) -> CompressionSavingsMetadata | None: + if isinstance(original_tokens, bool) or not isinstance(original_tokens, int): + return None + if isinstance(compressed_tokens, bool) or not isinstance(compressed_tokens, int): + return None + if compressed_tokens < 0 or original_tokens < compressed_tokens: + return None + return CompressionSavingsMetadata( + tokens_before=original_tokens, + tokens_after=compressed_tokens, + tokens_saved=original_tokens - compressed_tokens, + source="compression_interception", + ) + + +def _record_compression_savings(kwargs: dict[str, object], savings: CompressionSavingsMetadata) -> None: + """ + Attach savings to the request's litellm metadata so they land in the + SpendLog row's metadata JSON under ``compression_savings``. + + ``/v1/messages`` requests carry proxy metadata under ``litellm_metadata`` + (the ``metadata`` key is Anthropic's own API field). The existing dict is + updated in place because the proxy and the logging object hold references + to the same object; replacing it would orphan writes made through those + references. + """ + existing = kwargs.get("litellm_metadata") + if isinstance(existing, dict): + existing["compression_savings"] = savings + return + kwargs["litellm_metadata"] = {"compression_savings": savings} + + class CompressionInterceptionLogger(CustomLogger): """ CustomLogger that implements transparent prompt compression + retrieval loops. @@ -130,6 +166,12 @@ class CompressionInterceptionLogger(CustomLogger): call_id = str(uuid.uuid4()) kwargs["litellm_call_id"] = call_id self._compression_cache_by_call_id[call_id] = (cache, time.time()) + savings = _compression_savings_from_counts( + original_tokens=compressed.get("original_tokens"), + compressed_tokens=compressed.get("compressed_tokens"), + ) + if savings is not None: + _record_compression_savings(kwargs=kwargs, savings=savings) verbose_logger.debug( "CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]", call_id, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b47b43411c5..8dd9dbc0afa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -20,6 +20,9 @@ from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_no_callback_env_reference, ) +from litellm.types.integrations.compression_interception import ( + CompressionSavingsMetadata, +) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.llms.openai import ( AllMessageValues, @@ -3242,6 +3245,7 @@ class SpendLogsMetadata(TypedDict): attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded) max_retries: Optional[int] # Max retries configured for this request cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.) + compression_savings: CompressionSavingsMetadata | None class SpendLogsPayload(TypedDict): @@ -4461,6 +4465,11 @@ class BaseDailySpendTransaction(TypedDict): completion_tokens: int cache_read_input_tokens: int cache_creation_input_tokens: int + compression_saved_tokens: int + + # cost-savings metrics (dollars, priced per request before aggregation) + compression_savings_spend: float + prompt_caching_savings_spend: float # request level metrics spend: float diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 51e451efbb9..a7ceffed97b 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -1082,6 +1082,7 @@ async def get_agent_daily_activity( total_failed_requests=0, total_cache_read_input_tokens=0, total_cache_creation_input_tokens=0, + total_compression_saved_tokens=0, page=page, total_pages=0, has_more=False, diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 54a4c2dad91..2262141f426 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -59,6 +59,10 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, ) from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING +from litellm.proxy.spend_tracking.compression_savings import ( + extract_compression_saved_tokens, +) +from litellm.proxy.spend_tracking.savings import compute_savings_spend from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error if TYPE_CHECKING: @@ -1554,6 +1558,18 @@ class DBSpendUpdateWriter: common_data["cache_creation_input_tokens"] = transaction.get( "cache_creation_input_tokens", 0 ) + if "compression_saved_tokens" in transaction: + common_data["compression_saved_tokens"] = transaction.get( + "compression_saved_tokens", 0 + ) + if "compression_savings_spend" in transaction: + common_data["compression_savings_spend"] = transaction.get( + "compression_savings_spend", 0 + ) + if "prompt_caching_savings_spend" in transaction: + common_data["prompt_caching_savings_spend"] = transaction.get( + "prompt_caching_savings_spend", 0 + ) if entity_type == "tag" and "request_id" in transaction: common_data["request_id"] = transaction.get("request_id") @@ -1577,6 +1593,18 @@ class DBSpendUpdateWriter: update_data["cache_creation_input_tokens"] = { "increment": transaction.get("cache_creation_input_tokens", 0) } + if "compression_saved_tokens" in transaction: + update_data["compression_saved_tokens"] = { + "increment": transaction.get("compression_saved_tokens", 0) + } + if "compression_savings_spend" in transaction: + update_data["compression_savings_spend"] = { + "increment": transaction.get("compression_savings_spend", 0) + } + if "prompt_caching_savings_spend" in transaction: + update_data["prompt_caching_savings_spend"] = { + "increment": transaction.get("prompt_caching_savings_spend", 0) + } if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") @@ -1826,6 +1854,15 @@ class DBSpendUpdateWriter: if call_type: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) + cache_read_input_tokens = _extract_cache_read_tokens(usage_obj) + compression_saved_tokens = extract_compression_saved_tokens(_metadata) + savings_spend = compute_savings_spend( + model=payload.get("model", None), + custom_llm_provider=payload.get("custom_llm_provider", None), + compression_saved_tokens=compression_saved_tokens, + cache_read_input_tokens=cache_read_input_tokens, + ) + daily_transaction = BaseDailySpendTransaction( date=date, api_key=payload["api_key"], @@ -1840,8 +1877,11 @@ class DBSpendUpdateWriter: api_requests=1, successful_requests=1 if request_status == "success" else 0, failed_requests=1 if request_status != "success" else 0, - cache_read_input_tokens=_extract_cache_read_tokens(usage_obj), + cache_read_input_tokens=cache_read_input_tokens, cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj), + compression_saved_tokens=compression_saved_tokens, + compression_savings_spend=savings_spend.compression, + prompt_caching_savings_spend=savings_spend.prompt_caching, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 19f8f4a94ad..b6462636393 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -122,6 +122,18 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("cache_creation_input_tokens", 0) or 0 ) + daily_transaction.get("cache_creation_input_tokens", 0) + daily_transaction["compression_saved_tokens"] = ( + payload.get("compression_saved_tokens", 0) or 0 + ) + daily_transaction.get("compression_saved_tokens", 0) + + daily_transaction["compression_savings_spend"] = ( + payload.get("compression_savings_spend", 0) or 0 + ) + daily_transaction.get("compression_savings_spend", 0) + + daily_transaction["prompt_caching_savings_spend"] = ( + payload.get("prompt_caching_savings_spend", 0) or 0 + ) + daily_transaction.get("prompt_caching_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index e6f76b67c3c..2d67c22f0aa 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -11,6 +11,7 @@ from fastapi import HTTPException import litellm from httpx import Response as HttpxResponse +from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER from typing_extensions import TypeGuard from litellm._logging import verbose_proxy_logger @@ -487,7 +488,7 @@ class HeadroomGuardrail(CustomGuardrail): guardrail_json_response=stats, request_data=request_data, guardrail_status="success", - guardrail_provider="headroom", + guardrail_provider=HEADROOM_GUARDRAIL_PROVIDER, start_time=start_time, end_time=end_time, duration=end_time - start_time, diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 60cb3ccd30d..a5ecf4e7f93 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -49,6 +49,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics: existing_metrics.total_tokens += prompt_tokens + completion_tokens existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0 existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens or 0 + existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 + existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 + existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -473,6 +476,9 @@ def _build_aggregated_sql_query( SUM(completion_tokens)::bigint AS completion_tokens, SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens, SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens, + SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, + SUM(compression_savings_spend)::float AS compression_savings_spend, + SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -612,6 +618,9 @@ def _record_to_spend_metrics(record: Any) -> SpendMetrics: total_tokens=prompt_tokens + completion_tokens, cache_read_input_tokens=record.cache_read_input_tokens or 0, cache_creation_input_tokens=record.cache_creation_input_tokens or 0, + compression_saved_tokens=record.compression_saved_tokens or 0, + compression_savings_spend=record.compression_savings_spend or 0, + prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -862,6 +871,9 @@ async def get_daily_activity( total_failed_requests=metadata_metrics.failed_requests, total_cache_read_input_tokens=metadata_metrics.cache_read_input_tokens, total_cache_creation_input_tokens=metadata_metrics.cache_creation_input_tokens, + total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, + total_compression_savings_spend=metadata_metrics.compression_savings_spend, + total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -948,6 +960,9 @@ async def get_daily_activity_aggregated( total_failed_requests=aggregated["totals"].failed_requests, total_cache_read_input_tokens=aggregated["totals"].cache_read_input_tokens, total_cache_creation_input_tokens=aggregated["totals"].cache_creation_input_tokens, + total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, + total_compression_savings_spend=aggregated["totals"].compression_savings_spend, + total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index a99cec49417..b27ddea010b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/compression_savings.py b/litellm/proxy/spend_tracking/compression_savings.py new file mode 100644 index 00000000000..34241715756 --- /dev/null +++ b/litellm/proxy/spend_tracking/compression_savings.py @@ -0,0 +1,61 @@ +""" +Single chokepoint for reading prompt-compression token savings out of a parsed +SpendLog ``metadata`` JSON dict. Imported by the daily-spend DB writer and by +cost-savings read endpoints. +""" + +from collections.abc import Mapping + +HEADROOM_GUARDRAIL_PROVIDER = "headroom" + + +def _saved_tokens_or_zero(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0 + if value < 0: + return 0 + return int(value) + + +def _tokens_saved_from_stats(stats: object) -> int: + if not isinstance(stats, Mapping): + return 0 + return _saved_tokens_or_zero(stats.get("tokens_saved")) + + +def _headroom_entry_saved_tokens(entry: object) -> int: + if not isinstance(entry, Mapping): + return 0 + if entry.get("guardrail_provider") != HEADROOM_GUARDRAIL_PROVIDER: + return 0 + return _tokens_saved_from_stats(entry.get("guardrail_response")) + + +def _headroom_saved_tokens(guardrail_information: object) -> int: + entries = [guardrail_information] if isinstance(guardrail_information, Mapping) else guardrail_information + if not isinstance(entries, list): + return 0 + return sum(_headroom_entry_saved_tokens(entry) for entry in entries) + + +def extract_compression_saved_tokens(metadata: Mapping[str, object]) -> int: + """ + Return the total prompt tokens saved by compression for one request. + + Sums two disjoint sources: + + - the native ``compression_savings`` key, written only by + ``CompressionInterceptionLogger`` in its pre-call deployment hook + - ``guardrail_information`` entries with ``guardrail_provider == + "headroom"``, written only by the Headroom guardrail + + Each writer records only its own transform pass and the two run at + different stages (guardrail pre-call vs deployment pre-call), so when both + fire on one request their measured savings are independent and additive; + summing them never double-counts. Malformed or missing values contribute 0. + A bare dict ``guardrail_information`` is treated as a single entry, matching + the spend-log redactor's normalization of that legacy shape. + """ + return _tokens_saved_from_stats(metadata.get("compression_savings")) + _headroom_saved_tokens( + metadata.get("guardrail_information") + ) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py new file mode 100644 index 00000000000..ad9d02052bb --- /dev/null +++ b/litellm/proxy/spend_tracking/savings.py @@ -0,0 +1,63 @@ +""" +Per-request cost-savings computation for the Cost Optimization dashboard. + +Turns the token-level savings recorded on a request into dollar amounts using +the model's own pricing. Daily rollup rows are keyed by date and entity, not by +model, so the dollars have to be computed here (where the model and its prices +are known) and summed into the daily tables; tokens cannot be priced after they +have been aggregated across models. +""" + +from typing import NamedTuple + +import litellm +from litellm._logging import verbose_proxy_logger + + +class SavingsSpend(NamedTuple): + compression: float + prompt_caching: float + + +def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: + """ + Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model. + + Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to + zero rather than raising inside the spend writer. When a model has no + separate cache-read price the cache-read cost mirrors the input cost, which + yields zero caching savings. + """ + if not model: + return 0.0, 0.0 + try: + info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e + ) + return 0.0, 0.0 + input_cost = float(info.get("input_cost_per_token") or 0.0) + cache_read_cost = info.get("cache_read_input_token_cost") + if cache_read_cost is None: + return input_cost, input_cost + return input_cost, float(cache_read_cost) + + +def compute_savings_spend( + model: str | None, + custom_llm_provider: str | None, + compression_saved_tokens: int, + cache_read_input_tokens: int, +) -> SavingsSpend: + """ + Dollar savings for one request, split by optimization driver. + + Compression savings price the tokens compression removed at the model's + input rate. Prompt-caching savings price the cache-read tokens at the + difference between the input rate and the discounted cache-read rate. + """ + input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) + compression = max(compression_saved_tokens, 0) * input_cost + prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 23e7711b223..50f6f791bc2 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -115,6 +115,7 @@ def _get_spend_logs_metadata( attempted_retries=None, max_retries=None, cost_breakdown=None, + compression_savings=None, litellm_call_id=litellm_call_id, ) verbose_proxy_logger.debug( @@ -909,14 +910,45 @@ _PROMPT_CARRYING_GUARDRAIL_FIELDS = ( "classification", ) +_NUMERIC_COMPRESSION_STAT_KEYS = ( + "tokens_before", + "tokens_after", + "tokens_saved", + "compression_ratio", +) + + +def _numeric_compression_stats_from_guardrail_response( + guardrail_response: object, +) -> dict[str, int | float] | None: + if not isinstance(guardrail_response, dict): + return None + stats = { + key: value + for key, value in guardrail_response.items() + if key in _NUMERIC_COMPRESSION_STAT_KEYS and isinstance(value, (int, float)) and not isinstance(value, bool) + } + return stats or None + def _redact_prompt_fields_in_guardrail_entry( entry: StandardLoggingGuardrailInformation, ) -> StandardLoggingGuardrailInformation: - return { + """ + Replace prompt-carrying fields with the redaction marker. Purely numeric + compression stats inside ``guardrail_response`` (e.g. Headroom's + ``tokens_saved``) cannot carry prompt content, so they are preserved as a + stats-only dict; spend aggregation reads them via + ``extract_compression_saved_tokens``. + """ + preserved_stats = _numeric_compression_stats_from_guardrail_response(entry.get("guardrail_response")) + redacted: StandardLoggingGuardrailInformation = { **entry, **{key: REDACTED_BY_LITELM_STRING for key in _PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry}, } + if preserved_stats is None: + return redacted + return {**redacted, "guardrail_response": preserved_stats} def _sanitize_error_information_for_spend_logs( diff --git a/litellm/types/integrations/compression_interception.py b/litellm/types/integrations/compression_interception.py index fe52d2ad0d5..1466e9b693c 100644 --- a/litellm/types/integrations/compression_interception.py +++ b/litellm/types/integrations/compression_interception.py @@ -2,7 +2,7 @@ Type definitions for Compression Interception integration. """ -from typing import Any, Dict, Optional, TypedDict +from typing import Any, Dict, Literal, Optional, TypedDict class CompressionInterceptionConfig(TypedDict, total=False): @@ -25,3 +25,15 @@ class CompressionInterceptionConfig(TypedDict, total=False): compression_target: Optional[int] embedding_model: Optional[str] embedding_model_params: Optional[Dict[str, Any]] + + +class CompressionSavingsMetadata(TypedDict): + """ + Per-request prompt-compression savings recorded into the spend-log metadata + JSON so daily spend aggregates can track tokens saved by compression. + """ + + tokens_before: int + tokens_after: int + tokens_saved: int + source: Literal["compression_interception"] diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index b57d39f4c1a..00f67d0f39c 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -22,6 +22,9 @@ class SpendMetrics(BaseModel): completion_tokens: int = Field(default=0) cache_read_input_tokens: int = Field(default=0) cache_creation_input_tokens: int = Field(default=0) + compression_saved_tokens: int = Field(default=0) + compression_savings_spend: float = Field(default=0.0) + prompt_caching_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -79,6 +82,9 @@ class DailySpendMetadata(BaseModel): total_failed_requests: int = Field(default=0) total_cache_read_input_tokens: int = Field(default=0) total_cache_creation_input_tokens: int = Field(default=0) + total_compression_saved_tokens: int = Field(default=0) + total_compression_savings_spend: float = Field(default=0.0) + total_prompt_caching_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -102,6 +108,9 @@ class LiteLLM_DailyUserSpend(BaseModel): completion_tokens: int = 0 cache_read_input_tokens: int = 0 cache_creation_input_tokens: int = 0 + compression_saved_tokens: int = 0 + compression_savings_spend: float = 0.0 + prompt_caching_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/schema.prisma b/schema.prisma index a99cec49417..b27ddea010b 100644 --- a/schema.prisma +++ b/schema.prisma @@ -736,6 +736,9 @@ model LiteLLM_DailyUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -767,6 +770,9 @@ model LiteLLM_DailyOrganizationSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -798,6 +804,9 @@ model LiteLLM_DailyEndUserSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -828,6 +837,9 @@ model LiteLLM_DailyAgentSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -858,6 +870,9 @@ model LiteLLM_DailyTeamSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -890,6 +905,9 @@ model LiteLLM_DailyTagSpend { completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) + compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py b/tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py index ffa81abf86c..bc4dccc7d70 100644 --- a/tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py +++ b/tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py @@ -396,3 +396,130 @@ async def test_build_agentic_loop_plan_missing_key_fallback(): plan.request_patch.messages[-1]["content"][0]["content"] == "[compressed content key 'not_found.py' not found]" ) + + +def _stub_compress_result(original_tokens, compressed_tokens, cache): + return { + "messages": [{"role": "user", "content": "stubbed"}], + "original_tokens": original_tokens, + "compressed_tokens": compressed_tokens, + "compression_ratio": 0.5, + "cache": cache, + "tools": [], + } + + +@pytest.mark.asyncio +async def test_pre_call_hook_records_compression_savings_in_litellm_metadata(monkeypatch): + """ + When compression fires, the hook must record tokens_before/after/saved into + the request's litellm_metadata IN PLACE (the proxy and logging object hold + references to the same dict), so the savings land in the SpendLog row's + metadata JSON under ``compression_savings``. + """ + logger = CompressionInterceptionLogger() + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **kwargs: _stub_compress_result(12000, 5000, {"auth.py": "content"}), + ) + + litellm_metadata = {"user_api_key": "hashed-key", "user_api_key_user_id": "u1"} + kwargs = { + "model": "claude-sonnet-5", + "messages": [{"role": "user", "content": "very large context"}], + "litellm_metadata": litellm_metadata, + } + + result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages) + + assert result is not None + assert result["litellm_metadata"] is litellm_metadata + assert litellm_metadata["compression_savings"] == { + "tokens_before": 12000, + "tokens_after": 5000, + "tokens_saved": 7000, + "source": "compression_interception", + } + assert litellm_metadata["user_api_key"] == "hashed-key" + + +@pytest.mark.asyncio +async def test_pre_call_hook_creates_litellm_metadata_when_absent(monkeypatch): + """SDK-direct calls have no litellm_metadata dict yet; the hook creates it.""" + logger = CompressionInterceptionLogger() + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **kwargs: _stub_compress_result(300, 100, {"k": "v"}), + ) + + kwargs = { + "model": "claude-sonnet-5", + "messages": [{"role": "user", "content": "ctx"}], + } + + result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages) + + assert result is not None + assert result["litellm_metadata"]["compression_savings"]["tokens_saved"] == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "original_tokens,compressed_tokens", + [ + (None, 5000), + (12000, None), + ("12000", 5000), + (12000, "5000"), + (True, 5000), + (5000, 12000), + (12000, -1), + ], +) +async def test_pre_call_hook_invalid_token_counts_fail_open(monkeypatch, original_tokens, compressed_tokens): + """Invalid token counts must never crash the request; savings simply are not recorded.""" + logger = CompressionInterceptionLogger() + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **kwargs: _stub_compress_result(original_tokens, compressed_tokens, {"k": "v"}), + ) + + litellm_metadata = {"user_api_key": "hashed-key"} + kwargs = { + "model": "claude-sonnet-5", + "messages": [{"role": "user", "content": "ctx"}], + "litellm_metadata": litellm_metadata, + } + + result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages) + + assert result is not None + assert "compression_savings" not in litellm_metadata + + +@pytest.mark.asyncio +async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch): + """When compression is a no-op (empty cache) nothing is recorded.""" + logger = CompressionInterceptionLogger() + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **kwargs: { + "messages": [], + "original_tokens": 100, + "compressed_tokens": 100, + "cache": {}, + "tools": [], + "compression_skipped_reason": "below_trigger", + }, + ) + + litellm_metadata = {"user_api_key": "hashed-key"} + kwargs = { + "model": "claude-sonnet-5", + "messages": [{"role": "user", "content": "small"}], + "litellm_metadata": litellm_metadata, + } + + await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages) + + assert "compression_savings" not in litellm_metadata diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index 86bd9b71d04..c7e7ef5d469 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -206,6 +206,9 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "failed_requests": 0, # 0 + 0 "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0, + "prompt_caching_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -253,6 +256,9 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "failed_requests": 0, # 0 + 0 "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0, + "prompt_caching_savings_spend": 0, } # Add updates to queue @@ -476,3 +482,48 @@ async def test_queue_size_reduction_with_large_volume( assert result[user2_key]["api_requests"] == 100 assert result[user2_key]["successful_requests"] == 100 assert result[user2_key]["failed_requests"] == 0 + + +@pytest.mark.asyncio +async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): + """compression_saved_tokens must accumulate across payloads for the same key.""" + test_key = "user1_2023-01-01_key123_claude-sonnet-5_anthropic" + transaction1 = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "cache_read_input_tokens": 7, + "cache_creation_input_tokens": 3, + "compression_saved_tokens": 7000, + "compression_savings_spend": 0.007, + "prompt_caching_savings_spend": 0.0063, + } + transaction2 = { + "spend": 2.0, + "prompt_tokens": 20, + "completion_tokens": 10, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 4, + "compression_saved_tokens": 600, + "compression_savings_spend": 0.0006, + "prompt_caching_savings_spend": 0.0045, + } + + await daily_spend_update_queue.add_update({test_key: transaction1}) + await daily_spend_update_queue.add_update({test_key: transaction2}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert len(updates) == 1 + agg = updates[0][test_key] + assert agg["compression_saved_tokens"] == 7600 + assert agg["cache_read_input_tokens"] == 12 + assert agg["cache_creation_input_tokens"] == 7 + assert agg["compression_savings_spend"] == pytest.approx(0.0076) + assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 4c17c5d3482..8149cf90e70 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1902,3 +1902,103 @@ async def test_update_user_db_enqueues_user_spend_without_cache_dependency(): by_type = {u["entity_type"]: u["entity_id"] for u in queued} assert by_type[Litellm_EntityType.USER] == "user-123" assert by_type[Litellm_EntityType.END_USER] == "end-user-9" + + +@pytest.mark.asyncio +async def test_daily_transaction_carries_compression_saved_tokens(): + """ + The daily transaction built from a SpendLog payload must carry + compression_saved_tokens summed from both the native compression_savings + metadata key and Headroom guardrail entries, alongside the cache token + fields extracted from usage_object. + """ + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + metadata = { + "usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15}, + "compression_savings": { + "tokens_before": 12000, + "tokens_after": 5000, + "tokens_saved": 7000, + "source": "compression_interception", + }, + "guardrail_information": [ + { + "guardrail_name": "headroom-compressor", + "guardrail_provider": "headroom", + "guardrail_status": "success", + "guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600}, + } + ], + } + payload = { + "request_id": "req-compression-1", + "user": "test-user", + "startTime": "2026-07-17T00:00:00", + "api_key": "test-key", + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "model_group": "claude-sonnet-5", + "call_type": "anthropic_messages", + "prompt_tokens": 5000, + "completion_tokens": 10, + "spend": 0.05, + "metadata": json.dumps(metadata), + } + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["compression_saved_tokens"] == 7600 + assert transaction["cache_read_input_tokens"] == 40 + assert transaction["cache_creation_input_tokens"] == 15 + + model_info = litellm.get_model_info(model="claude-sonnet-5", custom_llm_provider="anthropic") + input_cost = model_info["input_cost_per_token"] or 0.0 + cache_read_cost = model_info.get("cache_read_input_token_cost") or input_cost + assert transaction["compression_savings_spend"] == pytest.approx(7600 * input_cost) + assert transaction["prompt_caching_savings_spend"] == pytest.approx( + 40 * max(input_cost - cache_read_cost, 0.0) + ) + assert transaction["compression_savings_spend"] > 0 + assert transaction["prompt_caching_savings_spend"] > 0 + + +@pytest.mark.asyncio +async def test_daily_transaction_compression_saved_tokens_zero_when_absent(): + """Requests without any compression metadata produce a zero count.""" + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-no-compression", + "user": "test-user", + "startTime": "2026-07-17T00:00:00", + "api_key": "test-key", + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "model_group": "claude-sonnet-5", + "call_type": "anthropic_messages", + "prompt_tokens": 100, + "completion_tokens": 10, + "spend": 0.01, + "metadata": json.dumps({"usage_object": {}}), + } + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["compression_saved_tokens"] == 0 + assert transaction["compression_savings_spend"] == 0 + assert transaction["prompt_caching_savings_spend"] == 0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index b882090e8f1..0769568d6cd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -150,6 +150,9 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "mcp_namespaced_tool_name": None, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -492,6 +495,9 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.completion_tokens = 200 mock_record_1.cache_read_input_tokens = 0 mock_record_1.cache_creation_input_tokens = 0 + mock_record_1.compression_saved_tokens = 0 + mock_record_1.compression_savings_spend = 0.0 + mock_record_1.prompt_caching_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -511,6 +517,9 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.completion_tokens = 100 mock_record_2.cache_read_input_tokens = 0 mock_record_2.cache_creation_input_tokens = 0 + mock_record_2.compression_saved_tokens = 0 + mock_record_2.compression_savings_spend = 0.0 + mock_record_2.prompt_caching_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -570,6 +579,9 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "mcp_namespaced_tool_name": None, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, + "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -654,6 +666,9 @@ def _daily_user_spend_record(*, user_id, api_key, spend): completion_tokens=5, cache_read_input_tokens=0, cache_creation_input_tokens=0, + compression_saved_tokens=0, + compression_savings_spend=0.0, + prompt_caching_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -865,6 +880,9 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "completion_tokens": None, "cache_read_input_tokens": None, "cache_creation_input_tokens": None, + "compression_saved_tokens": None, + "compression_savings_spend": None, + "prompt_caching_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -894,6 +912,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): assert result.metadata.total_failed_requests == 0 assert result.metadata.total_cache_read_input_tokens == 0 assert result.metadata.total_cache_creation_input_tokens == 0 + assert result.metadata.total_compression_saved_tokens == 0 def _no_spend_record(): @@ -904,6 +923,9 @@ def _no_spend_record(): completion_tokens=None, cache_read_input_tokens=None, cache_creation_input_tokens=None, + compression_saved_tokens=None, + compression_savings_spend=None, + prompt_caching_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -922,6 +944,7 @@ def test_record_to_spend_metrics_handles_none_values(): assert metrics.failed_requests == 0 assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 + assert metrics.compression_saved_tokens == 0 def test_update_metrics_handles_none_values(): @@ -936,3 +959,4 @@ def test_update_metrics_handles_none_values(): assert metrics.failed_requests == 0 assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 + assert metrics.compression_saved_tokens == 0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_compression_savings.py b/tests/test_litellm/proxy/spend_tracking/test_compression_savings.py new file mode 100644 index 00000000000..77e7b4b55c3 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_compression_savings.py @@ -0,0 +1,131 @@ +""" +Unit tests for the compression-savings spend-log metadata normalizer. +""" + +import pytest + +from litellm.proxy.spend_tracking.compression_savings import ( + extract_compression_saved_tokens, +) + +NATIVE_SAVINGS = { + "tokens_before": 12000, + "tokens_after": 5000, + "tokens_saved": 7000, + "source": "compression_interception", +} + +HEADROOM_ENTRY = { + "guardrail_name": "headroom-compressor", + "guardrail_provider": "headroom", + "guardrail_status": "success", + "guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600}, +} + + +def test_native_key_only(): + assert extract_compression_saved_tokens({"compression_savings": NATIVE_SAVINGS}) == 7000 + + +def test_headroom_only(): + assert extract_compression_saved_tokens({"guardrail_information": [HEADROOM_ENTRY]}) == 600 + + +def test_native_and_headroom_sum(): + metadata = { + "compression_savings": NATIVE_SAVINGS, + "guardrail_information": [HEADROOM_ENTRY], + } + assert extract_compression_saved_tokens(metadata) == 7600 + + +def test_multiple_headroom_entries_sum(): + metadata = {"guardrail_information": [HEADROOM_ENTRY, HEADROOM_ENTRY]} + assert extract_compression_saved_tokens(metadata) == 1200 + + +def test_neither_source_present(): + assert extract_compression_saved_tokens({"user_api_key": "abc", "usage_object": {}}) == 0 + + +def test_non_headroom_guardrail_entries_ignored(): + metadata = { + "guardrail_information": [ + { + "guardrail_name": "pii-guard", + "guardrail_provider": "presidio", + "guardrail_response": {"tokens_saved": 999}, + } + ] + } + assert extract_compression_saved_tokens(metadata) == 0 + + +@pytest.mark.parametrize( + "compression_savings", + [ + None, + "not-a-dict", + {}, + {"tokens_saved": None}, + {"tokens_saved": "7000"}, + {"tokens_saved": True}, + {"tokens_saved": -5}, + {"tokens_before": 100, "tokens_after": 50}, + ], +) +def test_malformed_native_key_contributes_zero(compression_savings): + assert extract_compression_saved_tokens({"compression_savings": compression_savings}) == 0 + + +@pytest.mark.parametrize( + "guardrail_information", + [ + None, + "not-a-list", + {"guardrail_provider": "headroom"}, + [], + [None], + ["not-a-dict"], + [{"guardrail_provider": "headroom"}], + [{"guardrail_provider": "headroom", "guardrail_response": "REDACTED"}], + [{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": "600"}}], + [{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": -600}}], + [{"guardrail_response": {"tokens_saved": 600}}], + ], +) +def test_malformed_headroom_information_contributes_zero(guardrail_information): + assert extract_compression_saved_tokens({"guardrail_information": guardrail_information}) == 0 + + +def test_valid_headroom_entry_survives_alongside_malformed_ones(): + metadata = { + "guardrail_information": [ + None, + {"guardrail_provider": "headroom", "guardrail_response": "REDACTED"}, + HEADROOM_ENTRY, + ] + } + assert extract_compression_saved_tokens(metadata) == 600 + + +def test_float_tokens_saved_counts_as_int(): + entry = {"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": 600.0}} + assert extract_compression_saved_tokens({"guardrail_information": [entry]}) == 600 + assert extract_compression_saved_tokens({"compression_savings": {"tokens_saved": 7000.0}}) == 7000 + assert extract_compression_saved_tokens({"compression_savings": {"tokens_saved": 12.5}}) == 12 + + +def test_bare_dict_guardrail_information_counts_as_single_entry(): + assert extract_compression_saved_tokens({"guardrail_information": HEADROOM_ENTRY}) == 600 + + +def test_headroom_writer_and_reader_share_provider_slug(): + from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( + HEADROOM_GUARDRAIL_PROVIDER as writer_slug, + ) + from litellm.proxy.spend_tracking.compression_savings import ( + HEADROOM_GUARDRAIL_PROVIDER as reader_slug, + ) + + assert writer_slug == reader_slug == "headroom" diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py new file mode 100644 index 00000000000..704cf7a63dd --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -0,0 +1,78 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest + +import litellm +from litellm.proxy.spend_tracking.savings import compute_savings_spend + + +def _anthropic_costs(model: str) -> tuple[float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + cache_read_cost = info.get("cache_read_input_token_cost") or input_cost + return input_cost, cache_read_cost + + +def test_compression_savings_priced_at_input_rate(): + input_cost, _ = _anthropic_costs("claude-sonnet-5") + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + cache_read_input_tokens=0, + ) + assert result.compression == pytest.approx(4389 * input_cost) + assert result.compression > 0 + assert result.prompt_caching == 0.0 + + +def test_prompt_caching_savings_priced_at_input_minus_cache_read(): + input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") + # A model that supports prompt caching must charge less to read from cache; + # otherwise this test is asserting nothing. + assert cache_read_cost < input_cost + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=8200, + ) + assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost)) + assert result.prompt_caching > 0 + assert result.compression == 0.0 + + +def test_unknown_model_fails_open_to_zero(): + result = compute_savings_spend( + model="totally-made-up-model-xyz", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=1000, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 + + +def test_missing_model_fails_open_to_zero(): + result = compute_savings_spend( + model=None, + custom_llm_provider=None, + compression_saved_tokens=1000, + cache_read_input_tokens=1000, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 + + +def test_negative_token_counts_clamp_to_zero(): + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=-500, + cache_read_input_tokens=-500, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index ffca5e5a368..579f46c8c77 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1998,7 +1998,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2094,7 +2094,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -2188,7 +2188,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 51d72aa2ab2..5d10bb33751 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2634,3 +2634,134 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key(): assert not metadata_dict["user_api_key"].startswith("sk-"), ( f"metadata user_api_key contains unhashed key: {metadata_dict['user_api_key']}" ) + + +@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs") +def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats( + mock_should_store, +): + """ + Headroom's compression stats live in guardrail_response, which is redacted + by default. Purely numeric token stats carry no prompt content and must + survive so daily spend aggregation can count compression_saved_tokens; + everything else in guardrail_response stays redacted. + """ + mock_should_store.return_value = False + guardrail_info = [ + { + "guardrail_name": "headroom-compressor", + "guardrail_provider": "headroom", + "guardrail_status": "success", + "guardrail_response": { + "tokens_before": 1000, + "tokens_after": 400, + "tokens_saved": 600, + "compression_ratio": 0.4, + "transforms_applied": ["dedupe_messages"], + }, + }, + { + "guardrail_name": "echo-guard", + "guardrail_status": "success", + "guardrail_response": { + "evaluated_input": "secret prompt", + "tokens_saved": "prompt text hiding in a stat key", + }, + }, + ] + + result = _sanitize_guardrail_information_for_spend_logs(guardrail_info) + + assert result is not None + assert result[0]["guardrail_response"] == { + "tokens_before": 1000, + "tokens_after": 400, + "tokens_saved": 600, + "compression_ratio": 0.4, + } + assert result[1]["guardrail_response"] == REDACTED_BY_LITELM_STRING + + +@pytest.mark.asyncio +async def test_compression_savings_survive_to_spend_log_payload_metadata(monkeypatch): + """ + End-to-end seam test for the native compression write path on + /v1/messages: the pre-call deployment hook records savings into the call + kwargs' litellm_metadata, the logging object captures it via + update_from_kwargs (exactly as llm_http_handler does for + anthropic_messages), and get_logging_payload lands it in + SpendLogsPayload.metadata JSON under ``compression_savings``. + """ + from litellm.integrations.compression_interception.handler import ( + CompressionInterceptionLogger, + ) + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CallTypes + + monkeypatch.setattr( + "litellm.integrations.compression_interception.handler.compress", + lambda **kwargs: { + "messages": [{"role": "user", "content": "compressed"}], + "original_tokens": 12000, + "compressed_tokens": 5000, + "cache": {"auth.py": "content"}, + "tools": [], + }, + ) + logger = CompressionInterceptionLogger() + + call_kwargs = { + "model": "claude-sonnet-5", + "messages": [{"role": "user", "content": "big context"}], + "max_tokens": 512, + "litellm_call_id": "test-compression-call-id", + "litellm_metadata": { + "user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b", + "user_api_key_user_id": "u1", + "user_api_key_team_id": "t1", + }, + } + hooked = await logger.async_pre_call_deployment_hook(kwargs=call_kwargs, call_type=CallTypes.anthropic_messages) + assert hooked is not None + + start_time = datetime.datetime.now(timezone.utc) + logging_obj = Logging( + model="claude-sonnet-5", + messages=hooked["messages"], + stream=False, + call_type="anthropic_messages", + start_time=start_time, + litellm_call_id="test-compression-call-id", + function_id="test", + ) + logging_obj.update_from_kwargs( + kwargs=hooked, + model="claude-sonnet-5", + optional_params={"max_tokens": 512}, + litellm_params={ + "preset_cache_key": None, + "stream_response": {}, + "model_info": hooked.get("model_info"), + }, + custom_llm_provider="anthropic", + ) + + end_time = datetime.datetime.now(timezone.utc) + logging_obj.model_call_details["completion_start_time"] = end_time + payload = get_logging_payload( + kwargs=logging_obj.model_call_details, + response_obj={ + "id": "msg_test", + "usage": {"prompt_tokens": 5000, "completion_tokens": 10, "total_tokens": 5010}, + }, + start_time=start_time, + end_time=end_time, + ) + + payload_metadata = json.loads(payload["metadata"]) + assert payload_metadata["compression_savings"] == { + "tokens_before": 12000, + "tokens_after": 5000, + "tokens_saved": 7000, + "source": "compression_interception", + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx new file mode 100644 index 00000000000..92426d3ce04 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.test.tsx @@ -0,0 +1,109 @@ +import { render } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; + +const mockUsePaginatedDailyActivity = vi.fn(); + +vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", () => ({ + usePaginatedDailyActivity: (args: unknown) => mockUsePaginatedDailyActivity(args), +})); + +vi.mock("@/components/networking", () => ({ + userDailyActivityCall: vi.fn(), +})); + +vi.mock("@/components/shared/advanced_date_picker", () => ({ + __esModule: true, + default: () =>
, +})); + +vi.mock("@/components/shared/charts", () => ({ + AreaChart: ({ data, categories }: { data: unknown; categories: string[] }) => ( +
+ ), + DonutChart: ({ data, label }: { data: unknown; label: string }) => ( +
+ ), +})); + +import CostOptimizationView from "./CostOptimizationView"; + +const baseMetrics = (overrides: Partial): SpendMetrics => ({ + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + ...overrides, +}); + +const day = (date: string, metrics: Partial): DailyData => ({ + date, + metrics: baseMetrics(metrics), + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, +}); + +const renderWith = (results: DailyData[]) => { + mockUsePaginatedDailyActivity.mockReturnValue({ data: { results }, loading: false, isFetchingMore: false }); + return render(); +}; + +describe("CostOptimizationView", () => { + it("sums compression and caching dollars across days into the summary cards", () => { + const { getByText } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + compression_saved_tokens: 40000, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + compression_saved_tokens: 100000, + }), + ]); + + // compression 0.14 + caching 0.016 = 0.156 + expect(getByText("$0.1560")).toBeInTheDocument(); + expect(getByText("$0.1400")).toBeInTheDocument(); + expect(getByText("$0.0160")).toBeInTheDocument(); + expect(getByText("140,000 tokens compressed")).toBeInTheDocument(); + }); + + it("builds a per-day time series and per-driver donut from the daily rows", () => { + const { getByTestId } = renderWith([ + day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }), + day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }), + ]); + + const series = JSON.parse(getByTestId("area-chart").getAttribute("data-series") ?? "[]"); + expect(series).toHaveLength(2); + expect(series[0]).toMatchObject({ Compression: 0.04, "Prompt caching": 0.006 }); + expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.01 }); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + ]); + }); + + it("omits a driver slice when that driver has no savings", () => { + const { getByTestId } = renderWith([day("2026-07-12", { compression_savings_spend: 0.04 })]); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx new file mode 100644 index 00000000000..e45e3e23f1d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -0,0 +1,157 @@ +"use client"; + +import React, { useMemo, useState } from "react"; +import { PiggyBank } from "lucide-react"; + +import { AreaChart, DonutChart } from "@/components/shared/charts"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { userDailyActivityCall } from "@/components/networking"; +import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { all_admin_roles } from "@/utils/roles"; +import { usePaginatedDailyActivity } from "@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity"; + +interface CostOptimizationViewProps { + accessToken: string | null; + userId: string | null; + userRole: string; +} + +type DateRange = { from?: Date; to?: Date }; + +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +const usd = (value: number): string => { + const decimals = value > 0 && value < 1 ? 4 : 2; + return `$${formatNumberWithCommas(value, decimals)}`; +}; + +const shortDate = (iso: string): string => + new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); + +const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; +const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; + +const SummaryCard = ({ label, value, hint }: { label: string; value: string; hint?: string }) => ( + + + {label} + + +

{value}

+ {hint &&

{hint}

} + + +); + +const CostOptimizationView: React.FC = ({ accessToken, userId, userRole }) => { + const initialFrom = useMemo(() => new Date(new Date().getTime() - THIRTY_DAYS_MS), []); + const initialTo = useMemo(() => new Date(), []); + const [dateValue, setDateValue] = useState({ from: initialFrom, to: initialTo }); + + const startTime = dateValue.from ?? null; + const endTime = dateValue.to ?? null; + const isAdmin = all_admin_roles.includes(userRole); + const effectiveUserId = isAdmin ? null : userId; + + const { data, loading, isFetchingMore } = usePaginatedDailyActivity({ + fetchFn: userDailyActivityCall, + args: [accessToken, startTime, endTime, effectiveUserId], + enabled: !!accessToken && !!startTime && !!endTime, + }); + + const results = data.results as DailyData[]; + + const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); + const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); + const totalSaved = compressionTotal + cachingTotal; + + const overTime = useMemo( + () => + results.map((d) => ({ + date: shortDate(d.date), + Compression: compressionOf(d.metrics), + "Prompt caching": cachingOf(d.metrics), + })), + [results], + ); + + const byDriver = useMemo( + () => + [ + { driver: "Compression", usd: compressionTotal }, + { driver: "Prompt caching", usd: cachingTotal }, + ].filter((d) => d.usd > 0), + [compressionTotal, cachingTotal], + ); + + return ( +
+
+
+
+ +

Cost Optimization

+
+

+ Money saved by prompt compression and prompt caching across your requests +

+
+ setDateValue(v)} /> +
+ +
+ + + +
+ +
+ + + Savings over time + + + + + + + + Savings by driver + + + + + +
+
+ ); +}; + +export default CostOptimizationView; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/page.tsx new file mode 100644 index 00000000000..e82cc633ae2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/page.tsx @@ -0,0 +1,9 @@ +"use client"; + +import CostOptimizationView from "./_components/CostOptimizationView"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export default function CostOptimizationPage() { + const { accessToken, userId, userRole } = useAuthorized(); + return ; +} diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index 6c1297613fa..bf33fa37111 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -8,6 +8,9 @@ export interface SpendMetrics { failed_requests: number; cache_read_input_tokens: number; cache_creation_input_tokens: number; + compression_saved_tokens?: number; + compression_savings_spend?: number; + prompt_caching_savings_spend?: number; } export type DailyData = { diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 76bfe174d5a..b0d534e9b7c 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -44,6 +44,7 @@ import { Palette, PanelLeftClose, PanelLeftOpen, + PiggyBank, PlayCircle, Route, ScrollText, @@ -181,6 +182,13 @@ const menuGroups: MenuGroup[] = [ roles: [...all_admin_roles, ...internalUserRoles], label: "Usage", }, + { + key: "cost-optimization", + page: "cost-optimization", + icon: , + roles: [...all_admin_roles, ...internalUserRoles], + label: "Cost Optimization", + }, { key: "logs", page: "logs", label: "Logs", icon: }, { key: "guardrails-monitor", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index daba5639a4f..2ceb75a06a9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23243,6 +23243,16 @@ export interface components { * @default 0 */ total_completion_tokens: number; + /** + * Total Compression Saved Tokens + * @default 0 + */ + total_compression_saved_tokens: number; + /** + * Total Compression Savings Spend + * @default 0 + */ + total_compression_savings_spend: number; /** * Total Failed Requests * @default 0 @@ -23253,6 +23263,11 @@ export interface components { * @default 1 */ total_pages: number; + /** + * Total Prompt Caching Savings Spend + * @default 0 + */ + total_prompt_caching_savings_spend: number; /** * Total Prompt Tokens * @default 0 @@ -30868,11 +30883,26 @@ export interface components { * @default 0 */ completion_tokens: number; + /** + * Compression Saved Tokens + * @default 0 + */ + compression_saved_tokens: number; + /** + * Compression Savings Spend + * @default 0 + */ + compression_savings_spend: number; /** * Failed Requests * @default 0 */ failed_requests: number; + /** + * Prompt Caching Savings Spend + * @default 0 + */ + prompt_caching_savings_spend: number; /** * Prompt Tokens * @default 0 diff --git a/ui/litellm-dashboard/src/utils/migratedPages.ts b/ui/litellm-dashboard/src/utils/migratedPages.ts index b0dfb46e312..73ab71ce4ac 100644 --- a/ui/litellm-dashboard/src/utils/migratedPages.ts +++ b/ui/litellm-dashboard/src/utils/migratedPages.ts @@ -44,6 +44,7 @@ export const MIGRATED_PAGES: Record = { // The modern usage dashboard; the legacy ?page=usage report routes to /old-usage. new_usage: "usage", usage: "old-usage", + "cost-optimization": "cost-optimization", agents: "agents", "router-settings": "router-settings", users: "users",