From f49b20161f4c09f12be243919aaaf1517ac49327 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:16:12 -0700 Subject: [PATCH] fix(spend): drop non-finite Responses and OpenRouter costs and the breakdown of an unpriceable request --- litellm/litellm_core_utils/core_helpers.py | 4 +-- litellm/litellm_core_utils/litellm_logging.py | 3 +- .../llms/openrouter/chat/transformation.py | 8 +++-- litellm/proxy/db/db_spend_update_writer.py | 4 +-- .../redis_update_buffer.py | 8 ++--- litellm/types/llms/openai.py | 10 +++++-- litellm/types/utils.py | 2 +- .../test_litellm_logging.py | 20 +++++++++++-- .../test_openrouter_chat_transformation.py | 29 +++++++++++++++++++ ...est_openrouter_responses_transformation.py | 25 ++++++++++++++++ .../test_redis_update_buffer.py | 20 +++++++++++++ 11 files changed, 115 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 778ca84ce14..7be0d80d296 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -951,9 +951,7 @@ def single_log_line(value: object) -> str: def finite_reported_cost(value: object) -> float | None: cost: Final = finite_cost(value) if cost is None and isinstance(value, float): - verbose_logger.warning( - "Ignoring a non-finite provider-reported cost %s, the cost map prices the request", value - ) + verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value) return cost diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c5075366051..a7914f50e38 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2198,6 +2198,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.warning( "Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id) ) + self.cost_breakdown = None return 0.0 def uncached_price(self, result: object) -> UncachedPrice: @@ -7465,7 +7466,7 @@ def get_standard_logging_object_payload( verbose_logger.warning( "Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s", numeric_response_cost, - logging_obj.litellm_call_id, + single_log_line(logging_obj.litellm_call_id), ) llm_response_cost: Final[float] = priced_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 9aaf552e79a..cca6815399f 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.core_helpers import finite_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig): additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata dict[str, object], hidden_params["additional_headers"] ) - if isinstance(response_cost, str | int | float): - additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost) + reported_cost: Final = ( + finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None + ) + if reported_cost is not None: + additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost except Exception: # If we can't extract cost, continue without it - don't fail the response pass diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 6221eaeb8bc..4b328593487 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -34,7 +34,7 @@ from litellm.constants import ( DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, ) -from litellm.litellm_core_utils.core_helpers import finite_cost +from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( @@ -653,7 +653,7 @@ class DBSpendUpdateWriter: verbose_proxy_logger.warning( "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", response_cost, - payload["request_id"], + single_log_line(payload["request_id"]), ) payload["spend"] = billable_cost or 0.0 if isinstance(payload["startTime"], datetime): diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 31a3e21b635..459f95972ce 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -111,7 +111,7 @@ _DECODED_JSON: Final = TypeAdapter(object) def _zero_for_non_finite_constant(constant: str) -> float: verbose_proxy_logger.warning( - "Redis spend buffer - reading a %s spend an older version parked as 0, the spend it held is lost", constant + "Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant ) return 0.0 @@ -208,7 +208,7 @@ class RedisUpdateBuffer: if transactions is None or len(transactions) == 0: return - list_of_transactions: Final = [safe_dumps(transactions, non_finite_as_null=True)] + list_of_transactions: Final = [safe_dumps(transactions)] if self.redis_cache is None: return try: @@ -369,7 +369,7 @@ class RedisUpdateBuffer: rpush_list.append( RedisPipelineRpushOperation( key=redis_key, - values=[safe_dumps(transactions, non_finite_as_null=True)], + values=[safe_dumps(transactions)], ) ) service_types.append(service_type) @@ -559,7 +559,7 @@ class RedisUpdateBuffer: ) rpush_list: Final = tuple( - RedisPipelineRpushOperation(key=redis_key, values=(safe_dumps(transactions, non_finite_as_null=True),)) + RedisPipelineRpushOperation(key=redis_key, values=(safe_dumps(transactions),)) for transactions, redis_key in restore_configs if transactions ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4912f9f1375..5697f60ef87 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,4 +1,5 @@ import builtins +import math from collections.abc import Iterable, Mapping from enum import Enum from os import PathLike @@ -82,6 +83,7 @@ from typing_extensions import ( override, ) +from litellm._logging import verbose_logger from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel from litellm.types.responses.main import ( CustomToolCallOutputItem, @@ -1440,9 +1442,11 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): @classmethod def parse_cost(cls, v: object) -> object: """Normalise cost: accept either a float or a dict with a ``total_cost`` key.""" - if isinstance(v, dict): - return v.get("total_cost") - return v + cost: Final = v.get("total_cost") if isinstance(v, dict) else v + if isinstance(cost, float) and not math.isfinite(cost): + verbose_logger.warning("Dropping a non-finite provider usage cost") + return None + return cost model_config = ConfigDict(extra="allow") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bad2d05af14..78eec3ee1f2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2062,7 +2062,7 @@ class Usage(SafeAttributeModel, CompletionUsage): if cost is None: del self.cost elif isinstance(cost, float) and not math.isfinite(cost): - verbose_logger.warning("Dropping a non-finite provider usage cost, the cost map prices the request") + verbose_logger.warning("Dropping a non-finite provider usage cost") del self.cost else: self.cost = cost diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index d2d7cb0fb48..072cad00863 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6135,17 +6135,33 @@ def test_get_standard_logging_object_payload_logs_a_non_finite_response_cost_as_ def test_an_overflowing_custom_price_records_a_zero_response_cost(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [{"role": "user", "content": "hello world"}] + start_time: Final = datetime.datetime.now() + logging_obj: Final = Logging( + model="gpt-4o", + messages=messages, + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="overflow\nforged log line", + function_id="overflow", + ) with caplog.at_level(logging.WARNING, logger="LiteLLM"): response: Final = litellm.completion( model="gpt-4o", mock_response="hi there", - messages=[{"role": "user", "content": "hello world"}], + messages=messages, input_cost_per_token=1e308, output_cost_per_token=1e308, - litellm_call_id="overflow\nforged log line", + litellm_logging_obj=logging_obj, ) + logging_obj._process_hidden_params_and_response_cost( + response, start_time=start_time, end_time=datetime.datetime.now() + ) assert response._hidden_params["response_cost"] == 0.0 + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert json.loads(json.dumps(payload, allow_nan=False, default=str))["response_cost"] == 0.0 warnings: Final = [r.getMessage() for r in caplog.records if "non-finite response_cost" in r.getMessage()] assert warnings == ["Recording a non-finite response_cost as 0 for litellm_call_id=overflowforged log line"] diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 59355dfabcb..888c47505a4 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,6 +1,7 @@ import json from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -464,6 +465,34 @@ def test_openrouter_cost_tracking_non_streaming(): ) + +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_chat_leaves_a_non_finite_provider_cost_out_of_the_response_headers(cost: float) -> None: + body: Final = { + "id": "gen-123", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-5.6-luna", + "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", "index": 0}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11, "cost": cost}, + } + + result: Final = OpenrouterConfig().transform_response( + model="openrouter/openai/gpt-5.6-luna", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + model_response=litellm.ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "Reply with ok"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {}) + def test_openrouter_cost_tracking_streaming(): """ Test OpenRouter cost tracking for streaming completions. diff --git a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py index 19517e58b31..78d7161c388 100644 --- a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py +++ b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py @@ -9,6 +9,8 @@ reasoning.encrypted_content for multi-turn stateless workflows. Related issue: https://github.com/BerriAI/litellm/issues/22189 """ +import json +from typing import Final from unittest.mock import MagicMock import httpx @@ -157,6 +159,29 @@ def test_openrouter_responses_returns_provider_reported_cost(cost): assert get_response_cost_from_hidden_params(response.hidden_params) == cost +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_responses_drops_a_non_finite_provider_cost(cost: float) -> None: + body: Final = { + "id": "resp_test", + "created_at": 1, + "model": "unmapped-model", + "object": "response", + "output": [], + "status": "completed", + "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5, "cost": cost}, + } + response: Final = OpenRouterResponsesAPIConfig().transform_response_api_response( + model="openrouter/unmapped-model", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + logging_obj=MagicMock(), + ) + + assert get_response_cost_from_hidden_params(response.hidden_params) is None + assert json.loads(json.dumps(response.model_dump(), allow_nan=False, default=str))["usage"]["cost"] is None + + @pytest.mark.parametrize( "event_type", ["response.completed", "response.incomplete", "response.failed"], diff --git a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py index b5630170371..65da5630f33 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -824,6 +824,10 @@ class _ListRedis: def __init__(self) -> None: self.rows: list[str] = [] + async def async_rpush(self, key: str, values: list[str]) -> int: + self.rows.extend(values) + return len(self.rows) + async def async_rpush_and_trim(self, key: str, values: list[str], max_len: int) -> int: self.rows.extend(values) pushed_len = len(self.rows) @@ -852,6 +856,22 @@ async def test_store_spend_logs_in_redis_drops_oldest_rows_past_the_cap(): assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == () +@pytest.mark.asyncio +@pytest.mark.parametrize("spend", [float("nan"), float("inf")]) +async def test_a_non_finite_daily_spend_buffered_in_redis_drains_as_zero(spend: float): + buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis()) + queue: Final = DailySpendUpdateQueue() + await queue.add_update(json.loads(_parked_daily_batch(spend))) + await queue.add_update(json.loads(_parked_daily_batch(0.0042))) + + await buffer.store_in_memory_daily_tag_spend_updates_in_redis(queue) + drained: Final = await buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + + assert drained is not None + assert drained[PARKED_DAILY_KEY]["spend"] == 0.0 + assert drained[PARKED_DAILY_KEY]["api_requests"] == 2 + + @pytest.mark.asyncio async def test_a_spend_log_row_parked_with_a_nan_spend_is_read_back_with_zero_spend(): buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis())