From ce9dc9fd48f0b6e8bb67c9bcc237f9acb8656211 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Thu, 29 Oct 2026 01:18:26 +0000 Subject: [PATCH] fix(streaming): don't treat usage.cost=0 as Vertex stream spend Ignore non-positive provider usage.cost on stream assembly so Vertex Anthropic cache-read streams price from tokens instead of logging spend=0 Fixes #37923 --- .../streaming_chunk_builder_utils.py | 30 +- .../litellm_core_utils/streaming_handler.py | 65 +++- litellm/types/utils.py | 5 + .../test_streaming_handler.py | 317 ++++++++++++++++++ 4 files changed, 396 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8d243e572fa..081c55b5780 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -50,6 +50,7 @@ from litellm.types.utils import ( TextCompletionResponse, UnreportedUsage, Usage, + hidden_response_cost_value, ) from litellm.utils import print_verbose, token_counter @@ -1376,7 +1377,12 @@ def _stream_builder_response_cost( ) return next((cost for cost in reported if cost is not None), None) usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): + # A non-positive leftover (common on Vertex Anthropic stream assembly) is not a provider total. + if ( + isinstance(usage_cost, (int, float)) + and usage_cost > 0 + and not _reported_cost_is_priced_by_calculator(logging_obj) + ): return float(usage_cost) if logging_obj is not None: return None @@ -1416,15 +1422,31 @@ def _set_stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> None: response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks) - if response_cost is None: + if response_cost is not None: + _set_hidden_response_cost(response, response_cost) return - _set_hidden_response_cost(response, response_cost) + usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) + hidden_cost: Final = hidden_response_cost_value(response) + # xAI is priced by the calculator, so the provider total here is None even after + # the stamp wrote the same positive number onto usage.cost and hidden response_cost. + # Keep that pair. Any other leftover, including a Vertex 0, has to go so token pricing runs. + if ( + isinstance(usage_cost, (int, float)) + and not isinstance(usage_cost, bool) + and isinstance(hidden_cost, (int, float)) + and not isinstance(hidden_cost, bool) + and usage_cost > 0 + and float(usage_cost) == float(hidden_cost) + ): + return + response.hidden_params.pop("response_cost", None) # pyright: ignore[reportUnknownMemberType] # legacy dict def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): + existing_cost: Final = getattr(usage, "cost", None) + if isinstance(existing_cost, (int, float)) and existing_cost > 0: return computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2348d8e57d6..de6bac5664f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1720,10 +1720,28 @@ class CustomStreamWrapper: self.tool_call = True chunk_usage: Final = getattr(chunk, "usage", None) - if isinstance(chunk_usage, Usage): - model_response.usage = chunk_usage - elif isinstance(chunk_usage, BaseModel): - model_response.usage = Usage(**chunk_usage.model_dump()) + parsed_usage: Final = ( + chunk_usage + if isinstance(chunk_usage, Usage) + else Usage(**chunk_usage.model_dump()) + if isinstance(chunk_usage, BaseModel) + else None + ) + # vertex_ai returns before cost propagation, so a leftover usage.cost=0 would + # be read downstream as a provider total. Drop it so token pricing runs. + reported_cost: Final = getattr(parsed_usage, "cost", None) + drop_reported_cost: Final = ( + self.custom_llm_provider == "vertex_ai" + and reported_cost is not None + and _usable_provider_reported_cost(reported_cost) is None + ) + attached_usage: Final = ( + parsed_usage.model_copy(update={"cost": None}) + if parsed_usage is not None and drop_reported_cost + else parsed_usage + ) + if attached_usage is not None: + model_response.usage = attached_usage ## RETURN ARG if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields: @@ -1963,17 +1981,7 @@ class CustomStreamWrapper: @staticmethod def _resolve_provider_reported_cost(usage_cost: object) -> float | None: - """ - Providers report usage.cost either as a number or as a breakdown object - whose total lives under ``total_cost``. - """ - if isinstance(usage_cost, bool): - return None - if isinstance(usage_cost, (int, float)): - return float(usage_cost) - if isinstance(usage_cost, dict): - return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost")) - return None + return _usable_provider_reported_cost(usage_cost) @staticmethod def _propagate_usage_cost_to_hidden_params( @@ -2733,6 +2741,28 @@ def _reported_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) return calculate_total_usage(chunks=chunks) +def _positive_provider_cost(usage_cost: object) -> float | None: + if isinstance(usage_cost, bool) or not isinstance(usage_cost, (int, float)) or usage_cost <= 0: + return None + return float(usage_cost) + + +def _usable_provider_reported_cost(usage_cost: object) -> float | None: + """ + Providers report usage.cost either as a number or as a breakdown object + whose total lives under ``total_cost``. + + Zero / negative is not a real provider total. Stream assembly can leave + ``usage.cost = 0`` on Vertex Anthropic chunks that still have tokens + (including cache-read). Treating that 0 as authoritative made the + stream spend path log spend=0 while ``cost_per_token`` on the same + Usage priced correctly. Leave it absent so token-based pricing runs. + """ + if isinstance(usage_cost, dict): + return _positive_provider_cost(usage_cost.get("total_cost")) + return _positive_provider_cost(usage_cost) + + def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) -> Usage: """Assume most recent usage chunk has total usage uptil then.""" from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( @@ -2781,8 +2811,9 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) if isinstance(latest_usage_chunk, dict) else getattr(latest_usage_chunk, "cost", None) ) - if latest_cost is not None: - returned_usage_chunk.cost = latest_cost + resolved_cost: Final = _usable_provider_reported_cost(latest_cost) + if resolved_cost is not None: + returned_usage_chunk.cost = resolved_cost return returned_usage_chunk diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b21bf0e14d4..ad6123ad229 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2392,6 +2392,11 @@ class ModelResponse(ModelResponseBase): return self.dict() +def hidden_response_cost_value(response: ModelResponse) -> object | None: + # Module scope so `object` is the builtin. Inside ModelResponse it is shadowed by the `object` field. + return response.hidden_params.get("response_cost") + + class Embedding(OpenAIObject): embedding: list | str = [] index: int diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 79978ecfe8e..97c852ad77d 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2728,6 +2728,323 @@ def test_provider_reported_cost_ignores_unusable_shapes(): assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5 + # 0 is not a real provider total — fall through to token-based pricing + assert CustomStreamWrapper._resolve_provider_reported_cost(0) is None + assert CustomStreamWrapper._resolve_provider_reported_cost(0.0) is None + assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": 0}) is None + + +def _vertex_anthropic_stream_chunks(*, cache_read: int, prompt_tokens: int, completion_tokens: int): + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + config = AnthropicConfig() + raw_input = prompt_tokens - cache_read + start_usage = config.calculate_usage( + usage_object={ + "input_tokens": raw_input, + "cache_read_input_tokens": cache_read, + "cache_creation_input_tokens": 0, + "output_tokens": 1, + }, + reasoning_content=None, + ) + delta_usage = config.calculate_usage( + usage_object={ + "input_tokens": raw_input, + "cache_read_input_tokens": cache_read, + "cache_creation_input_tokens": 0, + "output_tokens": completion_tokens, + }, + reasoning_content=None, + ) + return [ + ModelResponseStream( + id="chatcmpl-vertex-anthropic-cache", + created=1745513206, + model="claude-opus-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hi", role="assistant"), + ) + ], + usage=start_usage, + ), + ModelResponseStream( + id="chatcmpl-vertex-anthropic-cache", + created=1745513207, + model="claude-opus-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=""), + ) + ], + usage=delta_usage, + ), + ] + + +def _stream_spend_via_logging(complete_response, *, model: str, custom_llm_provider: str) -> float: + """Price an assembled stream the same way success_handler does.""" + logging_obj = Logging( + model=model, + messages=[{"role": "user", "content": "count to five"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="stream-spend-cache-read", + function_id="1245", + ) + logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + logging_obj.optional_params = {} + CustomStreamWrapper._propagate_usage_cost_to_hidden_params( + complete_response, custom_llm_provider + ) + cost = logging_obj._response_cost_calculator(result=complete_response) + assert cost is not None + return cost + + +def test_set_stream_builder_response_cost_keeps_matching_xai_stamp(): + """A calculator-priced stamp is not a provider total, but it is the spend figure. + + Dropping it made the assembled response report usage.cost and then lose + hidden response_cost, so the xAI stream spend row came back as None. + """ + from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost + + stamped: Final = 0.006152 + response: Final = ModelResponse( + id="chatcmpl-xai-stamp", + model="grok-4.7", + choices=[], + usage=Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252, cost=stamped), + ) + response._hidden_params["response_cost"] = stamped + logging_obj: Final = Logging( + model="xai/grok-4.7", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="xai-stamp-kept", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "xai" + logging_obj.optional_params = {} + + _set_stream_builder_response_cost(response, logging_obj, chunks=()) + + assert response._hidden_params.get("response_cost") == pytest.approx(stamped) + + +def test_set_stream_builder_response_cost_drops_xai_hidden_cost_that_differs_from_usage(): + from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost + + response: Final = ModelResponse( + id="chatcmpl-xai-stale", + model="grok-4.7", + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5), + ) + response._hidden_params["response_cost"] = 0.1 + logging_obj: Final = Logging( + model="xai/grok-4.7", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="xai-stamp-dropped", + function_id="1", + ) + logging_obj.model_call_details["custom_llm_provider"] = "xai" + logging_obj.optional_params = {} + + _set_stream_builder_response_cost(response, logging_obj, chunks=()) + + assert response._hidden_params.get("response_cost") is None + + +def test_stream_spend_prices_vertex_anthropic_cache_read_tokens(): + """#37923: streamed Vertex Anthropic usage with cache_read_input_tokens > 0 + must be priced the same way as cost_per_token on that Usage, not logged as $0. + """ + from litellm.cost_calculator import get_response_cost_from_hidden_params + + cache_read = 44616 + prompt_tokens = 70961 + completion_tokens = 4096 + chunks = _vertex_anthropic_stream_chunks( + cache_read=cache_read, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + complete_response = litellm.stream_chunk_builder( + chunks=chunks, + messages=[{"role": "user", "content": "count to five"}], + ) + assert complete_response is not None + usage = complete_response.usage + assert usage.cache_read_input_tokens == cache_read + assert usage.prompt_tokens == prompt_tokens + assert usage.completion_tokens == completion_tokens + + token_prompt, token_completion = litellm.cost_per_token( + model="claude-opus-5", + custom_llm_provider="vertex_ai", + usage_object=usage, + ) + token_total = token_prompt + token_completion + assert token_total > 0 + + # A leftover usage.cost=0 must not override token-based stream spend. + # Production path: stream_chunk_builder stamps hidden response_cost from usage.cost. + zero_cost_response = complete_response.model_copy(update={"usage": usage.model_copy(update={"cost": 0.0})}) + logging_for_builder = Logging( + model="claude-opus-5", + messages=[{"role": "user", "content": "count to five"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="stream-spend-cache-read-builder", + function_id="1245", + ) + logging_for_builder.model_call_details["custom_llm_provider"] = "vertex_ai" + logging_for_builder.optional_params = {} + from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost + + _set_stream_builder_response_cost(zero_cost_response, logging_for_builder, chunks=chunks) + assert zero_cost_response._hidden_params.get("response_cost") is None + assert get_response_cost_from_hidden_params(zero_cost_response._hidden_params) is None + + stream_spend = _stream_spend_via_logging( + zero_cost_response, + model="claude-opus-5", + custom_llm_provider="vertex_ai", + ) + # Stream spend uses the same Usage; regional Vertex uplift may raise the + # logged figure above the raw cost_per_token total, but it must not be 0. + assert stream_spend > 0 + assert stream_spend >= token_total - 1e-12 + + +def test_stream_spend_prices_vertex_anthropic_without_cache_read(): + """Stream without cache-read tokens must still price from the assembled usage.""" + chunks = _vertex_anthropic_stream_chunks( + cache_read=0, prompt_tokens=9, completion_tokens=16 + ) + complete_response = litellm.stream_chunk_builder( + chunks=chunks, + messages=[{"role": "user", "content": "count to five"}], + ) + assert complete_response is not None + token_prompt, token_completion = litellm.cost_per_token( + model="claude-opus-5", + custom_llm_provider="vertex_ai", + usage_object=complete_response.usage, + ) + token_total = token_prompt + token_completion + assert token_total == pytest.approx(9 * 5e-6 + 16 * 2.5e-5) + stream_spend = _stream_spend_via_logging( + complete_response, + model="claude-opus-5", + custom_llm_provider="vertex_ai", + ) + assert stream_spend > 0 + assert stream_spend >= token_total - 1e-12 + + +def test_vertex_chunk_creator_drops_zero_cost_and_prices_tokens(): + """Vertex stream chunks can carry usage.cost=0 with real token counts. + + chunk_creator must not keep that 0 as a provider total. Token pricing on + the assembled stream still has to be a positive spend. + """ + logging_obj = Logging( + model="claude-opus-5", + messages=[{"role": "user", "content": "count to five"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="vertex-zero-cost-chunk", + function_id="1245", + ) + logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai" + logging_obj.optional_params = {} + wrapper = CustomStreamWrapper( + completion_stream=None, + model="claude-opus-5", + logging_obj=logging_obj, + custom_llm_provider="vertex_ai", + ) + source_chunks = _vertex_anthropic_stream_chunks(cache_read=44616, prompt_tokens=70961, completion_tokens=4096) + zero_cost_usage = source_chunks[-1].usage.model_copy(update={"cost": 0}) + zero_cost_chunk = ModelResponseStream( + id="chatcmpl-vertex-zero-cost", + created=1745513207, + model="claude-opus-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="Hi", role="assistant"), + ) + ], + usage=zero_cost_usage, + ) + + processed = wrapper.chunk_creator(chunk=zero_cost_chunk) + + assert processed is not None + assert processed.usage.prompt_tokens == 70961 + assert processed.usage.completion_tokens == 4096 + assert processed.usage.cache_read_input_tokens == 44616 + assert getattr(processed.usage, "cost", None) in (None,) + + priced_chunk = ModelResponseStream( + id="chatcmpl-vertex-priced", + created=1745513208, + model="claude-opus-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="Hi", role="assistant"), + ) + ], + usage=zero_cost_usage.model_copy(update={"cost": 0.01}), + ) + priced = wrapper.chunk_creator(chunk=priced_chunk) + assert priced is not None + assert priced.usage.cost == 0.01 + + complete_response = litellm.stream_chunk_builder( + chunks=[processed], + messages=[{"role": "user", "content": "count to five"}], + logging_obj=logging_obj, + ) + assert complete_response is not None + token_prompt, token_completion = litellm.cost_per_token( + model="claude-opus-5", + custom_llm_provider="vertex_ai", + usage_object=complete_response.usage, + ) + token_total = token_prompt + token_completion + assert token_total > 0 + stream_spend = _stream_spend_via_logging( + complete_response, + model="claude-opus-5", + custom_llm_provider="vertex_ai", + ) + assert stream_spend > 0 + assert stream_spend >= token_total - 1e-12 def test_handle_special_delta_attributes(