diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql new file mode 100644 index 00000000000..f4cca662850 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260718000000_add_savings_spend/migration.sql @@ -0,0 +1,23 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "prompt_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c31d4bc5c33..230ca09959b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 95c6f16f108..795b2d1da47 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4465,6 +4465,10 @@ class BaseDailySpendTransaction(TypedDict): cache_creation_input_tokens: int compression_saved_tokens: int + # cost-savings metrics (dollars, priced per request before aggregation) + compression_savings_spend: float + prompt_caching_savings_spend: float + # request level metrics spend: float api_requests: int diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 35c00ebe284..2262141f426 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -62,6 +62,7 @@ from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, ) +from litellm.proxy.spend_tracking.savings import compute_savings_spend from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error if TYPE_CHECKING: @@ -1561,6 +1562,14 @@ class DBSpendUpdateWriter: common_data["compression_saved_tokens"] = transaction.get( "compression_saved_tokens", 0 ) + if "compression_savings_spend" in transaction: + common_data["compression_savings_spend"] = transaction.get( + "compression_savings_spend", 0 + ) + if "prompt_caching_savings_spend" in transaction: + common_data["prompt_caching_savings_spend"] = transaction.get( + "prompt_caching_savings_spend", 0 + ) if entity_type == "tag" and "request_id" in transaction: common_data["request_id"] = transaction.get("request_id") @@ -1588,6 +1597,14 @@ class DBSpendUpdateWriter: update_data["compression_saved_tokens"] = { "increment": transaction.get("compression_saved_tokens", 0) } + if "compression_savings_spend" in transaction: + update_data["compression_savings_spend"] = { + "increment": transaction.get("compression_savings_spend", 0) + } + if "prompt_caching_savings_spend" in transaction: + update_data["prompt_caching_savings_spend"] = { + "increment": transaction.get("prompt_caching_savings_spend", 0) + } if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") @@ -1837,6 +1854,15 @@ class DBSpendUpdateWriter: if call_type: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) + cache_read_input_tokens = _extract_cache_read_tokens(usage_obj) + compression_saved_tokens = extract_compression_saved_tokens(_metadata) + savings_spend = compute_savings_spend( + model=payload.get("model", None), + custom_llm_provider=payload.get("custom_llm_provider", None), + compression_saved_tokens=compression_saved_tokens, + cache_read_input_tokens=cache_read_input_tokens, + ) + daily_transaction = BaseDailySpendTransaction( date=date, api_key=payload["api_key"], @@ -1851,9 +1877,11 @@ class DBSpendUpdateWriter: api_requests=1, successful_requests=1 if request_status == "success" else 0, failed_requests=1 if request_status != "success" else 0, - cache_read_input_tokens=_extract_cache_read_tokens(usage_obj), + cache_read_input_tokens=cache_read_input_tokens, cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj), - compression_saved_tokens=extract_compression_saved_tokens(_metadata), + compression_saved_tokens=compression_saved_tokens, + compression_savings_spend=savings_spend.compression, + prompt_caching_savings_spend=savings_spend.prompt_caching, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 5c38aecf26d..b6462636393 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -126,6 +126,14 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("compression_saved_tokens", 0) or 0 ) + daily_transaction.get("compression_saved_tokens", 0) + daily_transaction["compression_savings_spend"] = ( + payload.get("compression_savings_spend", 0) or 0 + ) + daily_transaction.get("compression_savings_spend", 0) + + daily_transaction["prompt_caching_savings_spend"] = ( + payload.get("prompt_caching_savings_spend", 0) or 0 + ) + daily_transaction.get("prompt_caching_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 4c5f3d1ce2d..a5ecf4e7f93 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -50,6 +50,8 @@ def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics: existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0 existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens or 0 existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 + existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 + existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -475,6 +477,8 @@ def _build_aggregated_sql_query( SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens, SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens, SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, + SUM(compression_savings_spend)::float AS compression_savings_spend, + SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -615,6 +619,8 @@ def _record_to_spend_metrics(record: Any) -> SpendMetrics: cache_read_input_tokens=record.cache_read_input_tokens or 0, cache_creation_input_tokens=record.cache_creation_input_tokens or 0, compression_saved_tokens=record.compression_saved_tokens or 0, + compression_savings_spend=record.compression_savings_spend or 0, + prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -866,6 +872,8 @@ async def get_daily_activity( total_cache_read_input_tokens=metadata_metrics.cache_read_input_tokens, total_cache_creation_input_tokens=metadata_metrics.cache_creation_input_tokens, total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, + total_compression_savings_spend=metadata_metrics.compression_savings_spend, + total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -953,6 +961,8 @@ async def get_daily_activity_aggregated( total_cache_read_input_tokens=aggregated["totals"].cache_read_input_tokens, total_cache_creation_input_tokens=aggregated["totals"].cache_creation_input_tokens, total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, + total_compression_savings_spend=aggregated["totals"].compression_savings_spend, + total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index c31d4bc5c33..230ca09959b 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py new file mode 100644 index 00000000000..ad9d02052bb --- /dev/null +++ b/litellm/proxy/spend_tracking/savings.py @@ -0,0 +1,63 @@ +""" +Per-request cost-savings computation for the Cost Optimization dashboard. + +Turns the token-level savings recorded on a request into dollar amounts using +the model's own pricing. Daily rollup rows are keyed by date and entity, not by +model, so the dollars have to be computed here (where the model and its prices +are known) and summed into the daily tables; tokens cannot be priced after they +have been aggregated across models. +""" + +from typing import NamedTuple + +import litellm +from litellm._logging import verbose_proxy_logger + + +class SavingsSpend(NamedTuple): + compression: float + prompt_caching: float + + +def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: + """ + Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model. + + Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to + zero rather than raising inside the spend writer. When a model has no + separate cache-read price the cache-read cost mirrors the input cost, which + yields zero caching savings. + """ + if not model: + return 0.0, 0.0 + try: + info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e + ) + return 0.0, 0.0 + input_cost = float(info.get("input_cost_per_token") or 0.0) + cache_read_cost = info.get("cache_read_input_token_cost") + if cache_read_cost is None: + return input_cost, input_cost + return input_cost, float(cache_read_cost) + + +def compute_savings_spend( + model: str | None, + custom_llm_provider: str | None, + compression_saved_tokens: int, + cache_read_input_tokens: int, +) -> SavingsSpend: + """ + Dollar savings for one request, split by optimization driver. + + Compression savings price the tokens compression removed at the model's + input rate. Prompt-caching savings price the cache-read tokens at the + difference between the input rate and the discounted cache-read rate. + """ + input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) + compression = max(compression_saved_tokens, 0) * input_cost + prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index c6f2a8d1970..00f67d0f39c 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -23,6 +23,8 @@ class SpendMetrics(BaseModel): cache_read_input_tokens: int = Field(default=0) cache_creation_input_tokens: int = Field(default=0) compression_saved_tokens: int = Field(default=0) + compression_savings_spend: float = Field(default=0.0) + prompt_caching_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -81,6 +83,8 @@ class DailySpendMetadata(BaseModel): total_cache_read_input_tokens: int = Field(default=0) total_cache_creation_input_tokens: int = Field(default=0) total_compression_saved_tokens: int = Field(default=0) + total_compression_savings_spend: float = Field(default=0.0) + total_prompt_caching_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -105,6 +109,8 @@ class LiteLLM_DailyUserSpend(BaseModel): cache_read_input_tokens: int = 0 cache_creation_input_tokens: int = 0 compression_saved_tokens: int = 0 + compression_savings_spend: float = 0.0 + prompt_caching_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/schema.prisma b/schema.prisma index c31d4bc5c33..230ca09959b 100644 --- a/schema.prisma +++ b/schema.prisma @@ -730,6 +730,8 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -762,6 +764,8 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -794,6 +798,8 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -825,6 +831,8 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -856,6 +864,8 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -889,6 +899,8 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) + compression_savings_spend Float @default(0.0) + prompt_caching_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index be5d38a71a9..d8a937d30fa 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -494,6 +494,8 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): "cache_read_input_tokens": 7, "cache_creation_input_tokens": 3, "compression_saved_tokens": 7000, + "compression_savings_spend": 0.007, + "prompt_caching_savings_spend": 0.0063, } transaction2 = { "spend": 2.0, @@ -505,6 +507,8 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): "cache_read_input_tokens": 5, "cache_creation_input_tokens": 4, "compression_saved_tokens": 600, + "compression_savings_spend": 0.0006, + "prompt_caching_savings_spend": 0.0045, } await daily_spend_update_queue.add_update({test_key: transaction1}) @@ -517,3 +521,5 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["compression_saved_tokens"] == 7600 assert agg["cache_read_input_tokens"] == 12 assert agg["cache_creation_input_tokens"] == 7 + assert agg["compression_savings_spend"] == pytest.approx(0.0076) + assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index abc74f2477d..8149cf90e70 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -1959,6 +1959,16 @@ async def test_daily_transaction_carries_compression_saved_tokens(): assert transaction["cache_read_input_tokens"] == 40 assert transaction["cache_creation_input_tokens"] == 15 + model_info = litellm.get_model_info(model="claude-sonnet-5", custom_llm_provider="anthropic") + input_cost = model_info["input_cost_per_token"] or 0.0 + cache_read_cost = model_info.get("cache_read_input_token_cost") or input_cost + assert transaction["compression_savings_spend"] == pytest.approx(7600 * input_cost) + assert transaction["prompt_caching_savings_spend"] == pytest.approx( + 40 * max(input_cost - cache_read_cost, 0.0) + ) + assert transaction["compression_savings_spend"] > 0 + assert transaction["prompt_caching_savings_spend"] > 0 + @pytest.mark.asyncio async def test_daily_transaction_compression_saved_tokens_zero_when_absent(): @@ -1990,3 +2000,5 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent(): assert transaction is not None assert transaction["compression_saved_tokens"] == 0 + assert transaction["compression_savings_spend"] == 0 + assert transaction["prompt_caching_savings_spend"] == 0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 1f9cf71d1dc..0769568d6cd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -151,6 +151,8 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -494,6 +496,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.cache_read_input_tokens = 0 mock_record_1.cache_creation_input_tokens = 0 mock_record_1.compression_saved_tokens = 0 + mock_record_1.compression_savings_spend = 0.0 + mock_record_1.prompt_caching_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -514,6 +518,8 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.cache_read_input_tokens = 0 mock_record_2.cache_creation_input_tokens = 0 mock_record_2.compression_saved_tokens = 0 + mock_record_2.compression_savings_spend = 0.0 + mock_record_2.prompt_caching_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -574,6 +580,8 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, "compression_saved_tokens": 0, + "compression_savings_spend": 0.0, + "prompt_caching_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -659,6 +667,8 @@ def _daily_user_spend_record(*, user_id, api_key, spend): cache_read_input_tokens=0, cache_creation_input_tokens=0, compression_saved_tokens=0, + compression_savings_spend=0.0, + prompt_caching_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -871,6 +881,8 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "cache_read_input_tokens": None, "cache_creation_input_tokens": None, "compression_saved_tokens": None, + "compression_savings_spend": None, + "prompt_caching_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -912,6 +924,8 @@ def _no_spend_record(): cache_read_input_tokens=None, cache_creation_input_tokens=None, compression_saved_tokens=None, + compression_savings_spend=None, + prompt_caching_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py new file mode 100644 index 00000000000..704cf7a63dd --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -0,0 +1,78 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest + +import litellm +from litellm.proxy.spend_tracking.savings import compute_savings_spend + + +def _anthropic_costs(model: str) -> tuple[float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + cache_read_cost = info.get("cache_read_input_token_cost") or input_cost + return input_cost, cache_read_cost + + +def test_compression_savings_priced_at_input_rate(): + input_cost, _ = _anthropic_costs("claude-sonnet-5") + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + cache_read_input_tokens=0, + ) + assert result.compression == pytest.approx(4389 * input_cost) + assert result.compression > 0 + assert result.prompt_caching == 0.0 + + +def test_prompt_caching_savings_priced_at_input_minus_cache_read(): + input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") + # A model that supports prompt caching must charge less to read from cache; + # otherwise this test is asserting nothing. + assert cache_read_cost < input_cost + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=8200, + ) + assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost)) + assert result.prompt_caching > 0 + assert result.compression == 0.0 + + +def test_unknown_model_fails_open_to_zero(): + result = compute_savings_spend( + model="totally-made-up-model-xyz", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=1000, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 + + +def test_missing_model_fails_open_to_zero(): + result = compute_savings_spend( + model=None, + custom_llm_provider=None, + compression_saved_tokens=1000, + cache_read_input_tokens=1000, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 + + +def test_negative_token_counts_clamp_to_zero(): + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=-500, + cache_read_input_tokens=-500, + ) + assert result.compression == 0.0 + assert result.prompt_caching == 0.0 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5727eb5d2b8..bff9d6b19ed 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23244,6 +23244,11 @@ export interface components { * @default 0 */ total_compression_saved_tokens: number; + /** + * Total Compression Savings Spend + * @default 0 + */ + total_compression_savings_spend: number; /** * Total Failed Requests * @default 0 @@ -23254,6 +23259,11 @@ export interface components { * @default 1 */ total_pages: number; + /** + * Total Prompt Caching Savings Spend + * @default 0 + */ + total_prompt_caching_savings_spend: number; /** * Total Prompt Tokens * @default 0 @@ -30874,11 +30884,21 @@ export interface components { * @default 0 */ compression_saved_tokens: number; + /** + * Compression Savings Spend + * @default 0 + */ + compression_savings_spend: number; /** * Failed Requests * @default 0 */ failed_requests: number; + /** + * Prompt Caching Savings Spend + * @default 0 + */ + prompt_caching_savings_spend: number; /** * Prompt Tokens * @default 0