From 655d10775ca88dca9fc0bed63a2042ab3da5fe54 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:01:16 -0700 Subject: [PATCH] fix(cost): keep mid-stream pricing from leaking a breakdown into the spend log Pricing a frame through the request's own logging object is what makes custom deployment pricing work, but _response_cost_calculator does not only return a number. It also stamps cost_breakdown onto the live logging object, and on a pricing failure it writes response_cost_failure_debug_information into model_call_details. On an ordinary proxy stream that is harmless, because the success handler recomputes cost_breakdown at end of stream and overwrites whatever the frames left behind. The pass-through handlers are the problem: they compute their final cost with a bare completion_cost call and never touch cost_breakdown again, so a breakdown derived from one mid-stream frame would survive to the end and land in the spend log's metadata. response_cost itself is unaffected either way, so this was a reporting surface bug rather than a billing one, but the spend row would have gone from null to a populated breakdown for a partial frame. Snapshot both writes and put them back once the cost is read, so pricing a frame stays a read as far as the rest of the request is concerned. The returned cost is unchanged, so nothing about the injected usage.cost moves. --- litellm/proxy/common_request_processing.py | 17 ++++++ .../proxy/test_common_request_processing.py | 60 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7975db5cb85..d0dad1a4204 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3560,10 +3560,27 @@ class ProxyBaseLLMRequestProcessing: def _logging_obj_cost_or_none( model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj ) -> float | None: + # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto + # the live logging object. The pass-through handlers never recompute either one, so a + # frame-derived breakdown would outlive the stream and land in the spend log. Snapshot + # both and put them back, so pricing here stays a read as far as the request is concerned + breakdown_before: Final = getattr(litellm_logging_obj, "cost_breakdown", None) + call_details: Final = getattr(litellm_logging_obj, "model_call_details", None) + debug_key: Final = "response_cost_failure_debug_information" + debug_missing: Final = object() + debug_before: Final = call_details.get(debug_key, debug_missing) if isinstance(call_details, dict) else None try: cost: Final = litellm_logging_obj._response_cost_calculator(result=model_response) # pyright: ignore[reportPrivateUsage] # reuse the call's own cost calc for pricing parity with the logging callback except Exception: # noqa: BLE001 # a pricing failure falls back to model-name pricing instead of breaking the stream return None + finally: + if hasattr(litellm_logging_obj, "cost_breakdown"): + litellm_logging_obj.cost_breakdown = breakdown_before + if isinstance(call_details, dict): + if debug_before is debug_missing: + call_details.pop(debug_key, None) + else: + call_details[debug_key] = debug_before return float(cost) if isinstance(cost, (int, float)) and not isinstance(cost, bool) else None @staticmethod diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 58ea736ba12..6b762d7bd74 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6222,6 +6222,66 @@ class TestInjectCostIntoUsageDict: + 8 * pricing["output_cost_per_token"] ) + def test_pricing_a_frame_leaves_the_real_logging_obj_unchanged(self): + """Pricing runs against the live logging object, and the pass-through handlers never + recompute cost_breakdown, so a frame-derived breakdown would reach the spend log.""" + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + from litellm.types.utils import ModelResponse, Usage + + logging_obj = LiteLLMLoggingObj( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "test"}], + stream=True, + call_type="completion", + start_time=None, + litellm_call_id="lit4902-breakdown-test", + function_id="lit4902-breakdown-test", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + assert logging_obj.cost_breakdown is None + + model_response = ModelResponse( + usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224) + ) + cost = ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, logging_obj) + + assert cost is not None and cost > 0 + assert logging_obj.cost_breakdown is None + assert "response_cost_failure_debug_information" not in logging_obj.model_call_details + + def test_pricing_a_frame_restores_a_breakdown_the_request_already_had(self): + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + from litellm.types.utils import ModelResponse, Usage + + logging_obj = LiteLLMLoggingObj( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "test"}], + stream=True, + call_type="completion", + start_time=None, + litellm_call_id="lit4902-breakdown-restore", + function_id="lit4902-breakdown-restore", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + logging_obj.set_cost_breakdown( + input_cost=0.5, output_cost=0.25, total_cost=0.75, cost_for_built_in_tools_cost_usd_dollar=0.0 + ) + existing = logging_obj.cost_breakdown + + model_response = ModelResponse( + usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224) + ) + ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, logging_obj) + + assert logging_obj.cost_breakdown is existing + assert logging_obj.cost_breakdown["total_cost"] == 0.75 + def test_openai_chunk_prices_through_the_logging_obj_so_custom_pricing_applies(self): """The chat.completion.chunk path rides the same pricer, so a discounted deployment streaming /v1/chat/completions gets its negotiated price instead of sticker."""