diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 83a0e6f9e05..4156b9b0b7d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -877,9 +877,9 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -913,9 +913,9 @@ model LiteLLM_DailyGlobalSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -946,9 +946,9 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -984,9 +984,9 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1021,9 +1021,9 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1058,9 +1058,9 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1098,9 +1098,9 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dd91bb6eef3..d5ca0f5cd05 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5190,10 +5190,14 @@ class BaseDailySpendTransaction(TypedDict): cache_creation_input_tokens: int compression_saved_tokens: int - # cost-savings metrics (dollars, priced per request before aggregation) - compression_savings_spend: float - prompt_caching_savings_spend: float - gateway_injected_caching_savings_spend: float # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field + # cost-savings metrics (dollars, priced per request before aggregation). + # None means the request's savings existed but could not be priced (e.g. an + # unmapped openrouter/* model since the cost-map removals, #46677): the + # write path records NULL so the row never claims a measured $0.00 it never + # made. The aggregation merges None as "no dollars from this row", not zero. + compression_savings_spend: float | None # writable-ok: accumulated in place by the rollup queue + prompt_caching_savings_spend: float | None # writable-ok: accumulated in place by the rollup queue + gateway_injected_caching_savings_spend: float | None # writable-ok: accumulated in place by the rollup queue # Not required: rows queued by a pod running the previous release, or replayed from # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a # missing value to zero, so requiring it here would describe a shape the aggregation diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..e7d438f25f5 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -99,6 +99,17 @@ _SPEND_COLUMNS: Final = ( "gateway_injected_caching_savings_spend", "autorouter_savings_spend", ) +# Savings columns may carry no price for what they saved (#46677): the request +# recorded saved tokens but its model resolved to no rate, so the row reaches +# the database as NULL. Readers already coalesce NULL to zero for their totals; +# the merge treats NULL as "no dollars from this row", never as a measured zero. +_UNPRICED_SPEND_COLUMNS: Final = frozenset( + ( + "compression_savings_spend", + "prompt_caching_savings_spend", + "gateway_injected_caching_savings_spend", + ) +) _CASTS: Final[Mapping[str, str]] = MappingProxyType( { @@ -124,6 +135,24 @@ def _as_float(value: object) -> float: return float(value) if isinstance(value, (int, float)) else 0.0 +def _as_float_or_none(value: object) -> float | None: + """``value`` as a float, keeping ``None`` for a savings figure that was never priced.""" + if value is None: + return None + return _as_float(value) + + +def _sum_spend(column: str, group: Sequence[SpendRow]) -> float | None: + if column in _UNPRICED_SPEND_COLUMNS: + # A row that carries no such key at all (older release, synthetic caller) + # keeps its old zero, so only an explicit None marks an unpriced figure. + figures: Final = [row.get(column, 0.0) for row in group] + if all(figure is None for figure in figures): + return None + return sum(_as_float_or_none(figure) or 0.0 for figure in figures) + return sum(_as_float(row.get(column)) for row in group) + + def conflict_key(table: DailySpendTable, transaction: SpendRow) -> tuple[str, ...]: """The tuple the database arbitrates the upsert on, normalized free of NULLs.""" return tuple(_as_text(transaction.get(column)) for column in (table.entity_id_column, *_KEY_COLUMNS)) @@ -135,7 +164,7 @@ def _merge(group: Sequence[SpendRow]) -> SpendRow: return { **group[0], **{column: sum(_as_int(row.get(column)) for row in group) for column in _COUNTER_COLUMNS}, - **{column: sum(_as_float(row.get(column)) for row in group) for column in _SPEND_COLUMNS}, + **{column: _sum_spend(column, group) for column in _SPEND_COLUMNS}, } @@ -166,7 +195,12 @@ def _row_params( *key, None if transaction.get("model_group") is None else _as_text(transaction.get("model_group")), *(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS), - *(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS), + *( + _as_float_or_none(transaction.get(column, 0.0)) + if column in _UNPRICED_SPEND_COLUMNS + else _as_float(transaction.get(column)) + for column in _SPEND_COLUMNS + ), *((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()), ) @@ -199,10 +233,21 @@ def build_bulk_upsert( + ", (NOW() AT TIME ZONE 'UTC'))" for row_index in range(len(batch)) ) - increments: Final = ", ".join( - f'"{column}" = {quoted_table}."{column}" + EXCLUDED."{column}"' - for column in (*_COUNTER_COLUMNS, *_SPEND_COLUMNS) - ) + + def _increment_for(column: str) -> str: + if column in _UNPRICED_SPEND_COLUMNS: + # NULL-aware merge: NULL + x is NULL, so the coalesce falls back to + # whichever side carried dollars. An unpriced request landing on a + # priced row contributes nothing instead of wiping it, a priced one + # landing on an all-unpriced row prices it, and two unpriced rows + # stay unpriced. + return ( + f'"{column}" = COALESCE({quoted_table}."{column}" + EXCLUDED."{column}", ' + f'{quoted_table}."{column}", EXCLUDED."{column}")' + ) + return f'"{column}" = {quoted_table}."{column}" + EXCLUDED."{column}"' + + increments: Final = ", ".join(_increment_for(column) for column in (*_COUNTER_COLUMNS, *_SPEND_COLUMNS)) # request_id names one arbitrary contributing request, so an entry carrying none must # not blank out the one already recorded. request_id_update: Final = ( diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 288c85c3513..1295eea441d 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -13,6 +13,26 @@ from litellm.proxy.db.db_transaction_queue.base_update_queue import ( from litellm.types.services import ServiceTypes +def _add_savings_dollars(incoming: object, accumulated: object) -> float | None: + """Merge one request's savings dollars into the queued row's running figure. + + ``None`` marks savings that could not be priced (an unmapped model, a model + with no input rate): an unpriced contribution adds no dollars -- so it never + dilutes a priced figure -- and never turns an unpriced row into a measured + zero. The row stays ``None`` exactly while every contributing request was + unpriced, which is what the daily tables record as NULL. + """ + incoming_value: Final[float | None] = ( + float(incoming) if isinstance(incoming, (int, float)) and not isinstance(incoming, bool) else None + ) + accumulated_value: Final[float | None] = ( + float(accumulated) if isinstance(accumulated, (int, float)) and not isinstance(accumulated, bool) else None + ) + if incoming_value is None and accumulated_value is None: + return None + return (incoming_value or 0.0) + (accumulated_value or 0.0) + + class DailySpendUpdateQueue(BaseUpdateQueue): """ In memory buffer for daily spend updates that should be committed to the database @@ -138,17 +158,20 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("compression_saved_tokens", 0) or 0 ) + daily_transaction.get("compression_saved_tokens", 0) - daily_transaction["compression_savings_spend"] = ( - payload.get("compression_savings_spend", 0) or 0 - ) + daily_transaction.get("compression_savings_spend", 0) + daily_transaction["compression_savings_spend"] = _add_savings_dollars( + payload.get("compression_savings_spend", 0.0), + daily_transaction.get("compression_savings_spend", 0.0), + ) - daily_transaction["prompt_caching_savings_spend"] = ( - payload.get("prompt_caching_savings_spend", 0) or 0 - ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["prompt_caching_savings_spend"] = _add_savings_dollars( + payload.get("prompt_caching_savings_spend", 0.0), + daily_transaction.get("prompt_caching_savings_spend", 0.0), + ) - daily_transaction["gateway_injected_caching_savings_spend"] = ( - payload.get("gateway_injected_caching_savings_spend", 0) or 0 - ) + daily_transaction.get("gateway_injected_caching_savings_spend", 0) + daily_transaction["gateway_injected_caching_savings_spend"] = _add_savings_dollars( + payload.get("gateway_injected_caching_savings_spend", 0.0), + daily_transaction.get("gateway_injected_caching_savings_spend", 0.0), + ) daily_transaction["autorouter_savings_spend"] = ( payload.get("autorouter_savings_spend", 0) or 0 diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 83a0e6f9e05..4156b9b0b7d 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -877,9 +877,9 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -913,9 +913,9 @@ model LiteLLM_DailyGlobalSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -946,9 +946,9 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -984,9 +984,9 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1021,9 +1021,9 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1058,9 +1058,9 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1098,9 +1098,9 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 604a7a389e6..48eae9e558f 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -37,10 +37,39 @@ from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): - compression: float - prompt_caching: float + """Dollar savings for one request, split by optimization driver. + + ``compression``, ``prompt_caching`` and ``gateway_injected_caching`` carry + ``None`` when the request's savings could not be priced: an unmapped model, + a model with no input rate, or a usage record that never parsed. Such a + request has no dollar figure, and a flat ``0.0`` would read as a measured + zero -- the cost-map removals turned every ``openrouter/*`` deployment into + exactly that misreport. ``None`` reaches the daily tables as NULL, which the + readers already treat as "not priced" rather than as "saved nothing". + ``autorouter`` stays a plain ``float``: an un-routed request genuinely has + no auto-router savings, and unknown estimates are already kept apart by + ``recorded_estimated_autorouter_savings``. + """ + + compression: float | None + prompt_caching: float | None autorouter: float = 0.0 - gateway_injected_caching: float = 0.0 + gateway_injected_caching: float | None = 0.0 + + +def _priced_tokens(tokens: int, rate: float | None) -> float | None: + """Dollar cost of ``tokens`` at ``rate``, or ``None`` when no rate resolves. + + Zero tokens are a priced zero whatever the rate -- nothing was saved, so + there is nothing to price -- while tokens saved against a request the cost + map cannot price stay ``None`` so the write layer records them as unpriced + instead of a flat ``$0.00``. + """ + if tokens <= 0: + return 0.0 + if rate is None: + return None + return max(tokens, 0) * rate def _coerce_billed_at(value: datetime | str | None) -> datetime | None: @@ -620,6 +649,16 @@ def compute_savings_spend( Savings stay signed: a write-only request can lose money, and daily rollups net those losses against read savings. + Savings the request's pricing cannot resolve report ``None``, not ``0.0``: + since the OpenRouter rows left the cost map, an ``openrouter/*`` deployment + still records the tokens it saved, and a flat zero dollar value alongside + them reads as "saved nothing" where the truth is "could not price". ``None`` + is what the daily tables store as NULL, so the dashboard keeps its visible + totals (readers sum the priced dollars) while the row stops claiming a + measurement it never made. Deployment-configured rates still price it, as + before, and a priced request whose usage record is missing keeps reporting + ``0.0``: that zero is about the usage, not the price. + Caching is reported twice. ``prompt_caching`` is every net dollar caching saved, whoever caused it, which is what a customer means by "what did caching save me". ``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying @@ -657,9 +696,13 @@ def compute_savings_spend( request_pricing: Final = _request_savings_pricing(model, custom_llm_provider, model_id, llm_router) provider: Final = request_pricing[0] pricing: Final = request_pricing[1] - input_cost: Final = (get_cost_per_unit(pricing, "input_cost_per_token") or 0.0) if pricing else 0.0 - compression: Final = max(compression_saved_tokens, 0) * input_cost - prompt_caching: Final = _prompt_caching_savings(pricing, provider, usage_object, cost_breakdown, billed_at) or 0.0 + input_cost: Final = get_cost_per_unit(pricing, "input_cost_per_token", default_value=None) if pricing else None + compression: Final = _priced_tokens(compression_saved_tokens, input_cost) + prompt_caching: Final = ( + None + if pricing is None + else (_prompt_caching_savings(pricing, provider, usage_object, cost_breakdown, billed_at) or 0.0) + ) gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 # The figure the logging path recorded wins, before the usage gate on purpose: a row diff --git a/schema.prisma b/schema.prisma index 83a0e6f9e05..4156b9b0b7d 100644 --- a/schema.prisma +++ b/schema.prisma @@ -877,9 +877,9 @@ model LiteLLM_DailyUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -913,9 +913,9 @@ model LiteLLM_DailyGlobalSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -946,9 +946,9 @@ model LiteLLM_DailyOrganizationSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -984,9 +984,9 @@ model LiteLLM_DailyEndUserSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1021,9 +1021,9 @@ model LiteLLM_DailyAgentSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1058,9 +1058,9 @@ model LiteLLM_DailyTeamSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -1098,9 +1098,9 @@ model LiteLLM_DailyTagSpend { cache_read_input_tokens BigInt @default(0) cache_creation_input_tokens BigInt @default(0) compression_saved_tokens BigInt @default(0) - compression_savings_spend Float @default(0.0) - prompt_caching_savings_spend Float @default(0.0) - gateway_injected_caching_savings_spend Float @default(0.0) + compression_savings_spend Float? + prompt_caching_savings_spend Float? + gateway_injected_caching_savings_spend Float? autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index 79447d0fc59..f2984101502 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -592,3 +592,98 @@ async def test_optional_metric_missing_from_an_older_payload_still_aggregates( assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) assert updates[0][test_key]["total_response_time_ms"] == 900 assert updates[0][test_key]["timed_requests"] == 1 + +def test_get_aggregated_daily_spend_update_transactions_unpriced_savings_stay_none(): + """#46677: two requests whose savings could not be priced aggregate to an + unpriced row, not to a measured $0.00.""" + test_key = "user1_2026-10-15_key123_openrouter-model_openrouter" + unpriced = { + "spend": 10.0, + "prompt_tokens": 100, + "completion_tokens": 50, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "compression_savings_spend": None, + "prompt_caching_savings_spend": None, + "gateway_injected_caching_savings_spend": None, + } + result = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions( + [{test_key: dict(unpriced)}, {test_key: dict(unpriced)}] + ) + assert len(result) == 1 + merged = result[test_key] + assert merged["compression_savings_spend"] is None + assert merged["prompt_caching_savings_spend"] is None + assert merged["gateway_injected_caching_savings_spend"] is None + + +@pytest.mark.parametrize("order", ["priced_first", "unpriced_first"]) +def test_get_aggregated_daily_spend_update_transactions_unpriced_keeps_priced_dollars(order): + """An unpriced request landing on a priced row contributes no dollars and + never dilutes the figure the priced requests already earned.""" + test_key = "user1_2026-10-15_key123_model_provider" + priced = { + "spend": 10.0, + "prompt_tokens": 100, + "completion_tokens": 50, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "compression_savings_spend": 0.11, + "prompt_caching_savings_spend": 0.22, + "gateway_injected_caching_savings_spend": 0.33, + } + unpriced = { + "spend": 5.0, + "prompt_tokens": 200, + "completion_tokens": 30, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "compression_savings_spend": None, + "prompt_caching_savings_spend": None, + "gateway_injected_caching_savings_spend": None, + } + updates = ( + [{test_key: dict(priced)}, {test_key: dict(unpriced)}] + if order == "priced_first" + else [{test_key: dict(unpriced)}, {test_key: dict(priced)}] + ) + result = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(updates) + merged = result[test_key] + assert merged["compression_savings_spend"] == pytest.approx(0.11) + assert merged["prompt_caching_savings_spend"] == pytest.approx(0.22) + assert merged["gateway_injected_caching_savings_spend"] == pytest.approx(0.33) + assert merged["spend"] == pytest.approx(15.0) + + +def test_get_aggregated_daily_spend_update_transactions_missing_savings_key_stays_zero(): + """A transaction that carries no savings keys keeps its old zero semantics.""" + test_key = "user1_2023-01-01_key123_gpt-4_openai" + bare = { + "spend": 10.0, + "prompt_tokens": 100, + "completion_tokens": 50, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + unpriced = { + "spend": 5.0, + "prompt_tokens": 200, + "completion_tokens": 30, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + "compression_savings_spend": None, + "prompt_caching_savings_spend": None, + "gateway_injected_caching_savings_spend": None, + } + result = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions( + [{test_key: dict(bare)}, {test_key: dict(unpriced)}] + ) + merged = result[test_key] + assert merged["compression_savings_spend"] == 0.0 + assert merged["prompt_caching_savings_spend"] == 0.0 + assert merged["gateway_injected_caching_savings_spend"] == 0.0 diff --git a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py index 5bce9137666..5fff88241e1 100644 --- a/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/unit/proxy/db/test_daily_spend_bulk_upsert.py @@ -208,3 +208,103 @@ async def test_writer_survives_a_transaction_whose_key_columns_are_null(): _, params = prisma_client.db.statements[0] assert None not in params[:9] assert transactions == {} + +def test_unpriced_savings_survive_the_conflict_merge_as_none(): + """#46677: a row whose request saved tokens the cost map cannot price must + not come out of the merge as a measured $0.00.""" + merged = merge_by_conflict_key( + TAG_TABLE, + ( + tag_txn( + compression_savings_spend=None, + prompt_caching_savings_spend=None, + gateway_injected_caching_savings_spend=None, + ), + ), + ) + _, folded = merged[0] + assert folded["compression_savings_spend"] is None + assert folded["prompt_caching_savings_spend"] is None + assert folded["gateway_injected_caching_savings_spend"] is None + + +@pytest.mark.parametrize("order", ["priced_first", "unpriced_first"]) +def test_unpriced_savings_contribute_no_dollars_to_a_priced_row(order): + """An unpriced contribution must not dilute, wipe, or zero a priced figure.""" + unpriced = tag_txn( + compression_savings_spend=None, + prompt_caching_savings_spend=None, + gateway_injected_caching_savings_spend=None, + ) + priced = tag_txn( + compression_savings_spend=0.11, + prompt_caching_savings_spend=0.22, + gateway_injected_caching_savings_spend=0.33, + ) + transactions = (priced, unpriced) if order == "priced_first" else (unpriced, priced) + merged = merge_by_conflict_key(TAG_TABLE, transactions) + _, folded = merged[0] + assert folded["compression_savings_spend"] == pytest.approx(0.11) + assert folded["prompt_caching_savings_spend"] == pytest.approx(0.22) + assert folded["gateway_injected_caching_savings_spend"] == pytest.approx(0.33) + + +def test_two_unpriced_rows_stay_unpriced_not_zero(): + merged = merge_by_conflict_key( + TAG_TABLE, + ( + tag_txn(compression_savings_spend=None, prompt_caching_savings_spend=None, gateway_injected_caching_savings_spend=None), + tag_txn(compression_savings_spend=None, prompt_caching_savings_spend=None, gateway_injected_caching_savings_spend=None), + ), + ) + _, folded = merged[0] + assert folded["compression_savings_spend"] is None + assert folded["prompt_caching_savings_spend"] is None + assert folded["gateway_injected_caching_savings_spend"] is None + + +def test_rows_without_savings_keys_still_merge_to_zero(): + """Rows that never carried the savings keys keep their old measured zero.""" + merged = merge_by_conflict_key(TAG_TABLE, (tag_txn(), tag_txn())) + _, folded = merged[0] + assert folded["compression_savings_spend"] == 0.0 + assert folded["prompt_caching_savings_spend"] == 0.0 + assert folded["gateway_injected_caching_savings_spend"] == 0.0 + + +def test_unpriced_savings_bind_as_null_and_merge_null_aware(): + """NULL must bind for an unpriced row, and the increment must be NULL-aware: + NULL + x reads x on either side, so an unpriced request neither wipes a + priced row nor turns an all-unpriced row into a measured zero.""" + sql, params = build_bulk_upsert( + TAG_TABLE, + merge_by_conflict_key( + TAG_TABLE, + (tag_txn(compression_savings_spend=None, prompt_caching_savings_spend=None, gateway_injected_caching_savings_spend=None),), + ), + ) + for column in ( + "compression_savings_spend", + "prompt_caching_savings_spend", + "gateway_injected_caching_savings_spend", + ): + assert ( + f'"{column}" = COALESCE("LiteLLM_DailyTagSpend"."{column}" + EXCLUDED."{column}", ' + f'"LiteLLM_DailyTagSpend"."{column}", EXCLUDED."{column}")' in sql + ) + assert '"spend" = "LiteLLM_DailyTagSpend"."spend" + EXCLUDED."spend"' in sql + assert params.count(None) == 3 + + +def test_priced_savings_still_bind_dollars(): + sql, params = build_bulk_upsert( + TAG_TABLE, + merge_by_conflict_key( + TAG_TABLE, + (tag_txn(compression_savings_spend=0.11, prompt_caching_savings_spend=0.22, gateway_injected_caching_savings_spend=0.33),), + ), + ) + assert None not in params + assert 0.11 in params + assert 0.22 in params + assert 0.33 in params diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index a42b6404278..c68bff6090f 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -5317,3 +5317,57 @@ def test_record_request_error_feeds_the_failure_rollup_with_the_logged_status() status_code=503, ): 1 } + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.asyncio +async def test_add_spend_log_transaction_to_daily_user_transaction_leaves_unpriced_savings_null(): + """#46677: an openrouter/* deployment's compression savings cannot be priced + (the cost map no longer carries its rows), so the queued row must record the + dollars as unpriced (None -> NULL), never as a measured $0.00.""" + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-unpriced", + "user": "test-user", + "startTime": "2026-10-15T00:00:00", + "api_key": "test-key", + "model": "openrouter/z-ai/glm-5.3-flash", + "custom_llm_provider": "openrouter", + "model_group": "openrouter/z-ai/glm-5.3-flash", + "prompt_tokens": 100, + "completion_tokens": 10, + "spend": 0.02, + "metadata": json.dumps( + { + "compression_savings": {"tokens_saved": 5000}, + "usage_object": { + "prompt_tokens": 100, + "completion_tokens": 10, + "total_tokens": 110, + "prompt_tokens_details": {"cached_tokens": 80, "text_tokens": 20}, + }, + } + ), + } + + writer.daily_spend_update_queue.add_update = AsyncMock() + + await writer.add_spend_log_transaction_to_daily_user_transaction( + payload=payload, + prisma_client=mock_prisma, + ) + + writer.daily_spend_update_queue.add_update.assert_called_once() + update_dict = writer.daily_spend_update_queue.add_update.call_args[1]["update"] + assert len(update_dict) == 1 + for key, transaction in update_dict.items(): + # the tokens still count... + assert transaction["compression_saved_tokens"] == 5000 + # ...but their dollars are unpriced, not a fabricated zero + assert transaction["compression_savings_spend"] is None + assert transaction["prompt_caching_savings_spend"] is None + # no gateway injection: the attributed figure stays a priced zero + assert transaction["gateway_injected_caching_savings_spend"] == 0.0 + assert transaction["spend"] == pytest.approx(0.02) diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..d068376ea36 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -22,12 +22,23 @@ pytestmark = pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("model,usage", [ (None, {"cache_read_input_tokens": 100}), - ("claude-sonnet-5", None), - ("claude-sonnet-5", {"prompt_tokens": "invalid"}), + ("totally-made-up-model-xyz", {"cache_read_input_tokens": 100}), ]) def test_prompt_cache_estimate_distinguishes_unknown_from_zero(model: str | None, usage: dict[str, object] | None) -> None: assert prompt_caching_savings_for_request(model, "anthropic", usage) is None - assert compute_savings_spend(model, "anthropic", 0, False, usage_object=usage).prompt_caching == 0 + # The rollup now carries the unknown forward instead of recording a measured zero: + # a request whose price cannot be resolved has no dollar figure to write. + assert compute_savings_spend(model, "anthropic", 0, False, usage_object=usage).prompt_caching is None + assert compute_savings_spend(model, "anthropic", 500, False, usage_object=usage).compression is None + + +def test_priced_request_with_missing_or_unusable_usage_records_zero() -> None: + # The estimate reports unknown; the rollup has a price but no usage it can + # apply it to, so the savings are zero, not unpriced. + assert prompt_caching_savings_for_request("claude-sonnet-5", "anthropic", None) is None + assert prompt_caching_savings_for_request("claude-sonnet-5", "anthropic", {"prompt_tokens": "invalid"}) is None + assert compute_savings_spend("claude-sonnet-5", "anthropic", 0, False, usage_object=None).prompt_caching == 0 + assert compute_savings_spend("claude-sonnet-5", "anthropic", 0, False, usage_object={"prompt_tokens": "invalid"}).prompt_caching == 0 assert prompt_caching_savings_for_request("claude-sonnet-5", "anthropic", {"prompt_tokens": 100}) == 0 @@ -407,7 +418,10 @@ def test_negative_cache_write_count_clamps_to_zero(): assert result.prompt_caching == pytest.approx(1000 * (input_cost - cache_read_cost)) -def test_unknown_model_fails_open_to_zero(): +def test_unknown_model_reports_unpriced_savings(): + # Cost-map removals (#45665-#45669) made "unknown model" a whole provider's + # steady state, not a typo: a flat $0.00 there reads as "saved nothing" + # while the token counters keep climbing (#46677). result = compute_savings_spend( model="totally-made-up-model-xyz", custom_llm_provider="anthropic", @@ -415,11 +429,12 @@ def test_unknown_model_fails_open_to_zero(): gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 1000}, ) - assert result.compression == 0.0 - assert result.prompt_caching == 0.0 + assert result.compression is None + assert result.prompt_caching is None + assert result.gateway_injected_caching is None -def test_missing_model_fails_open_to_zero(): +def test_missing_model_reports_unpriced_savings(): result = compute_savings_spend( model=None, custom_llm_provider=None, @@ -427,8 +442,60 @@ def test_missing_model_fails_open_to_zero(): gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 1000}, ) + assert result.compression is None + assert result.prompt_caching is None + assert result.gateway_injected_caching is None + + +def test_unmapped_provider_model_savings_are_unpriced_not_zero(): + """Regression for #46677: openrouter/* left the cost map, so every + openrouter deployment's savings dashboard flatlined at $0.00.""" + result = compute_savings_spend( + model="openrouter/z-ai/glm-5.3-flash", + custom_llm_provider="openrouter", + compression_saved_tokens=5000, + gateway_injected_cache=False, + usage_object={ + "prompt_tokens": 100, + "completion_tokens": 10, + "total_tokens": 110, + "prompt_tokens_details": {"cached_tokens": 80, "text_tokens": 20}, + }, + ) + assert result.compression is None + assert result.prompt_caching is None + # not gateway-injected: the attributed figure stays a priced zero + assert result.gateway_injected_caching == 0.0 + + +def test_zero_saved_tokens_stay_a_priced_zero_on_an_unpriced_model(): + # Nothing was saved: the dollar figure is a measured zero even when the + # model itself cannot be priced. + result = compute_savings_spend( + model="openrouter/z-ai/glm-5.3-flash", + custom_llm_provider="openrouter", + compression_saved_tokens=0, + gateway_injected_cache=False, + ) assert result.compression == 0.0 - assert result.prompt_caching == 0.0 + assert result.gateway_injected_caching == 0.0 + + +@pytest.mark.parametrize("tokens,rate,expected", [ + (100, 0.0, 0.0), # an explicit zero rate is a price, not an unknown + (0, None, 0.0), # nothing saved is a priced zero + (-5, None, 0.0), # negative counts clamp, and stay zero + (100, None, None), # tokens saved against no resolvable rate are unpriced + (100, 2e-06, 2e-04), +]) +def test_priced_tokens_distinguishes_a_zero_rate_from_no_rate(tokens: int, rate: float | None, expected: float | None) -> None: + from litellm.proxy.spend_tracking.savings import _priced_tokens + + result = _priced_tokens(tokens, rate) + if expected is None: + assert result is None + else: + assert result == pytest.approx(expected) def test_negative_token_counts_clamp_to_zero():