From ca85e7e413df204a52047d10c0d986339f809384 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:46:11 -0700 Subject: [PATCH] fix(spend): record non-finite savings as 0 and warn when a parked NaN batch drains An overflowing price on a compressed request made compression savings infinite, which still wedged the daily spend batch. Every savings column now records a non-finite value as 0 with a warning. The Redis drain warns when it reads a NaN an older version parked, since the spend summed into it is lost from daily spend. The writer test now runs the real payload builder and spend log insert --- .../redis_update_buffer.py | 5 ++- litellm/proxy/spend_tracking/savings.py | 6 ++- .../test_redis_update_buffer.py | 8 +++- .../proxy/db/test_db_spend_update_writer.py | 41 ++++++++----------- .../unit/proxy/spend_tracking/test_savings.py | 28 +++++++++++++ 5 files changed, 60 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index a9159739e25..699f559f16f 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -109,7 +109,10 @@ _ValueT = TypeVar("_ValueT") _DECODED_JSON: Final = TypeAdapter(object) -def _zero_for_non_finite_constant(_constant: str) -> float: +def _zero_for_non_finite_constant(constant: str) -> float: + verbose_proxy_logger.warning( + "Redis spend buffer - draining a %s spend an older version parked as 0, its summed spend is lost", constant + ) return 0.0 diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 604a7a389e6..d73689a1207 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -689,9 +689,13 @@ def compute_savings_spend( cost_breakdown=cost_breakdown, ) ) - return SavingsSpend( + savings: Final = SavingsSpend( compression=compression, prompt_caching=prompt_caching, autorouter=0.0 if autorouter is None else autorouter, gateway_injected_caching=gateway_injected_caching, ) + if all(isfinite(value) for value in savings): + return savings + verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model) + return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings)) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 164c322d3aa..23dab2fa14c 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -1,10 +1,12 @@ import json +import logging from datetime import datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm._logging import verbose_proxy_logger from litellm.constants import REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DailyTagSpendTransaction @@ -449,16 +451,18 @@ async def test_a_nan_spend_parked_by_an_older_version_drains_from_the_pipeline_a @pytest.mark.asyncio async def test_a_nan_spend_parked_by_an_older_version_drains_from_a_daily_key_as_zero( - redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock, caplog: pytest.LogCaptureFixture ): mock_redis_cache.async_lpop = AsyncMock( return_value=[_parked_daily_batch(float("inf")), _parked_daily_batch(0.0042)] ) - daily_tag: Final = await redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + daily_tag: Final = await redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() assert daily_tag is not None assert daily_tag[PARKED_DAILY_KEY]["spend"] == 0.0042 + assert [record.getMessage() for record in caplog.records if "Infinity" in record.getMessage()] != [] @pytest.mark.asyncio diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index f273711d9dc..67a97170686 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -2229,31 +2229,17 @@ async def test_shutdown_drain_finishes_the_rollups_update_database_started(): @pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")]) async def test_update_database_records_a_non_finite_response_cost_as_zero(response_cost: float) -> None: db_writer: Final = DBSpendUpdateWriter() - db_writer._insert_spend_log_to_db = AsyncMock() - prisma: Final = MagicMock() + prisma: Final = _tool_usage_prisma() prisma.get_request_status = MagicMock(return_value="success") - fake_payload: Final = { - "request_id": "req-non-finite-cost", - "api_key": "hashed-key", - "user": "test-user", - "team_id": "test-team", - "agent_id": None, - "startTime": "2026-01-01T00:00:00", - "endTime": "2026-01-01T00:00:01", - "model": "gpt-4o-mini", - "custom_llm_provider": "openai", - "model_group": "gpt-4o-mini", - "prompt_tokens": 10, - "completion_tokens": 5, - "spend": response_cost, - "metadata": '{"usage_object": {}}', - } + response: Final = litellm.ModelResponse( + model="gpt-5.6-luna", + usage=litellm.Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) with ( patch("litellm.proxy.proxy_server.disable_spend_logs", False), patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), - patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload), ): await db_writer.update_database( token="test-token", @@ -2261,15 +2247,22 @@ async def test_update_database_records_a_non_finite_response_cost_as_zero(respon end_user_id=None, team_id="test-team", org_id=None, - kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, - completion_response=MagicMock(), - start_time=datetime.now(), - end_time=datetime.now(), + kwargs={ + "model": "gpt-5.6-luna", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "req-non-finite-cost", + "response_cost": response_cost, + "litellm_params": {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "test-team"}}, + }, + completion_response=response, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), response_cost=response_cost, ) await drain_logging_tasks() - assert db_writer._insert_spend_log_to_db.call_args.kwargs["payload"]["spend"] == 0 + assert [row["spend"] for row in prisma.spend_log_transactions] == [0] spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() assert spend["key_list_transactions"] == {"test-token": 0} assert spend["team_list_transactions"] == {"test-team": 0} diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..8503a124b01 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -137,6 +137,34 @@ def test_compression_savings_priced_at_input_rate(): assert result.prompt_caching == 0.0 +def test_an_overflowing_deployment_price_records_finite_savings() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "overflowing-sonnet", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "input_cost_per_token": 1e308, + "cache_read_input_token_cost": 1e-07, + }, + }, + ] + ) + deployment_id: Final = router.get_model_list(model_name="overflowing-sonnet")[0]["model_info"]["id"] + + result: Final = compute_savings_spend( + model="claude-sonnet-5-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + gateway_injected_cache=True, + usage_object=_caching_usage(read=1000, written=0), + model_id=deployment_id, + llm_router=lambda: router, + ) + + assert result == (0.0, 0.0, 0.0, 0.0) + + def test_prompt_caching_savings_priced_at_input_minus_cache_read(): input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") # A model that supports prompt caching must charge less to read from cache;