This commit is contained in:
leilei3167 2026-10-29 14:01:27 +00:00 • committed by GitHub
commit 5c35a42178
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 396 additions and 21 deletions

View file

@ -55,6 +55,7 @@ from litellm.types.utils import (
TextCompletionResponse,
UnreportedUsage,
Usage,
hidden_response_cost_value,
)
from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
@ -1381,7 +1382,12 @@ def _stream_builder_response_cost(
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
# A non-positive leftover (common on Vertex Anthropic stream assembly) is not a provider total.
if (
isinstance(usage_cost, (int, float))
and usage_cost > 0
and not _reported_cost_is_priced_by_calculator(logging_obj)
):
return float(usage_cost)
if logging_obj is not None:
return None
@ -1421,15 +1427,31 @@ def _set_stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> None:
response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks)
if response_cost is None:
if response_cost is not None:
_set_hidden_response_cost(response, response_cost)
return
_set_hidden_response_cost(response, response_cost)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
hidden_cost: Final = hidden_response_cost_value(response)
# xAI is priced by the calculator, so the provider total here is None even after
# the stamp wrote the same positive number onto usage.cost and hidden response_cost.
# Keep that pair. Any other leftover, including a Vertex 0, has to go so token pricing runs.
if (
isinstance(usage_cost, (int, float))
and not isinstance(usage_cost, bool)
and isinstance(hidden_cost, (int, float))
and not isinstance(hidden_cost, bool)
and usage_cost > 0
and float(usage_cost) == float(hidden_cost)
):
return
response.hidden_params.pop("response_cost", None) # pyright: ignore[reportUnknownMemberType] # legacy dict
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
existing_cost: Final = getattr(usage, "cost", None)
if isinstance(existing_cost, (int, float)) and existing_cost > 0:
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:

View file

@ -1732,10 +1732,28 @@ class CustomStreamWrapper:
self.tool_call = True
chunk_usage: Final = getattr(chunk, "usage", None)
if isinstance(chunk_usage, Usage):
model_response.usage = chunk_usage
elif isinstance(chunk_usage, BaseModel):
model_response.usage = Usage(**chunk_usage.model_dump())
parsed_usage: Final = (
chunk_usage
if isinstance(chunk_usage, Usage)
else Usage(**chunk_usage.model_dump())
if isinstance(chunk_usage, BaseModel)
else None
)
# vertex_ai returns before cost propagation, so a leftover usage.cost=0 would
# be read downstream as a provider total. Drop it so token pricing runs.
reported_cost: Final = getattr(parsed_usage, "cost", None)
drop_reported_cost: Final = (
self.custom_llm_provider == "vertex_ai"
and reported_cost is not None
and _usable_provider_reported_cost(reported_cost) is None
)
attached_usage: Final = (
parsed_usage.model_copy(update={"cost": None})
if parsed_usage is not None and drop_reported_cost
else parsed_usage
)
if attached_usage is not None:
model_response.usage = attached_usage
## RETURN ARG
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
@ -1985,17 +2003,7 @@ class CustomStreamWrapper:
@staticmethod
def _resolve_provider_reported_cost(usage_cost: object) -> float | None:
"""
Providers report usage.cost either as a number or as a breakdown object
whose total lives under ``total_cost``.
"""
if isinstance(usage_cost, bool):
return None
if isinstance(usage_cost, (int, float)):
return float(usage_cost)
if isinstance(usage_cost, dict):
return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost"))
return None
return _usable_provider_reported_cost(usage_cost)
@staticmethod
def _propagate_usage_cost_to_hidden_params(
@ -2757,6 +2765,28 @@ def _reported_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream])
return calculate_total_usage(chunks=chunks)
def _positive_provider_cost(usage_cost: object) -> float | None:
if isinstance(usage_cost, bool) or not isinstance(usage_cost, (int, float)) or usage_cost <= 0:
return None
return float(usage_cost)
def _usable_provider_reported_cost(usage_cost: object) -> float | None:
"""
Providers report usage.cost either as a number or as a breakdown object
whose total lives under ``total_cost``.
Zero / negative is not a real provider total. Stream assembly can leave
``usage.cost = 0`` on Vertex Anthropic chunks that still have tokens
(including cache-read). Treating that 0 as authoritative made the
stream spend path log spend=0 while ``cost_per_token`` on the same
Usage priced correctly. Leave it absent so token-based pricing runs.
"""
if isinstance(usage_cost, dict):
return _positive_provider_cost(usage_cost.get("total_cost"))
return _positive_provider_cost(usage_cost)
def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) -> Usage:
"""Assume most recent usage chunk has total usage uptil then."""
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
@ -2805,8 +2835,9 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream])
if isinstance(latest_usage_chunk, dict)
else getattr(latest_usage_chunk, "cost", None)
)
if latest_cost is not None:
returned_usage_chunk.cost = latest_cost
resolved_cost: Final = _usable_provider_reported_cost(latest_cost)
if resolved_cost is not None:
returned_usage_chunk.cost = resolved_cost
return returned_usage_chunk

View file

@ -2392,6 +2392,11 @@ class ModelResponse(ModelResponseBase):
return self.dict()
def hidden_response_cost_value(response: ModelResponse) -> object | None:
# Module scope so `object` is the builtin. Inside ModelResponse it is shadowed by the `object` field.
return response.hidden_params.get("response_cost")
class Embedding(OpenAIObject):
embedding: list | str = []
index: int

View file

@ -2728,6 +2728,323 @@ def test_provider_reported_cost_ignores_unusable_shapes():
assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None
assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None
assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5
# 0 is not a real provider total — fall through to token-based pricing
assert CustomStreamWrapper._resolve_provider_reported_cost(0) is None
assert CustomStreamWrapper._resolve_provider_reported_cost(0.0) is None
assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": 0}) is None
def _vertex_anthropic_stream_chunks(*, cache_read: int, prompt_tokens: int, completion_tokens: int):
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
config = AnthropicConfig()
raw_input = prompt_tokens - cache_read
start_usage = config.calculate_usage(
usage_object={
"input_tokens": raw_input,
"cache_read_input_tokens": cache_read,
"cache_creation_input_tokens": 0,
"output_tokens": 1,
},
reasoning_content=None,
)
delta_usage = config.calculate_usage(
usage_object={
"input_tokens": raw_input,
"cache_read_input_tokens": cache_read,
"cache_creation_input_tokens": 0,
"output_tokens": completion_tokens,
},
reasoning_content=None,
)
return [
ModelResponseStream(
id="chatcmpl-vertex-anthropic-cache",
created=1745513206,
model="claude-opus-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content="Hi", role="assistant"),
)
],
usage=start_usage,
),
ModelResponseStream(
id="chatcmpl-vertex-anthropic-cache",
created=1745513207,
model="claude-opus-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(content=""),
)
],
usage=delta_usage,
),
]
def _stream_spend_via_logging(complete_response, *, model: str, custom_llm_provider: str) -> float:
"""Price an assembled stream the same way success_handler does."""
logging_obj = Logging(
model=model,
messages=[{"role": "user", "content": "count to five"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="stream-spend-cache-read",
function_id="1245",
)
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
logging_obj.optional_params = {}
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(
complete_response, custom_llm_provider
)
cost = logging_obj._response_cost_calculator(result=complete_response)
assert cost is not None
return cost
def test_set_stream_builder_response_cost_keeps_matching_xai_stamp():
"""A calculator-priced stamp is not a provider total, but it is the spend figure.
Dropping it made the assembled response report usage.cost and then lose
hidden response_cost, so the xAI stream spend row came back as None.
"""
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
stamped: Final = 0.006152
response: Final = ModelResponse(
id="chatcmpl-xai-stamp",
model="grok-4.7",
choices=[],
usage=Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252, cost=stamped),
)
response._hidden_params["response_cost"] = stamped
logging_obj: Final = Logging(
model="xai/grok-4.7",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="xai-stamp-kept",
function_id="1",
)
logging_obj.model_call_details["custom_llm_provider"] = "xai"
logging_obj.optional_params = {}
_set_stream_builder_response_cost(response, logging_obj, chunks=())
assert response._hidden_params.get("response_cost") == pytest.approx(stamped)
def test_set_stream_builder_response_cost_drops_xai_hidden_cost_that_differs_from_usage():
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
response: Final = ModelResponse(
id="chatcmpl-xai-stale",
model="grok-4.7",
choices=[],
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5),
)
response._hidden_params["response_cost"] = 0.1
logging_obj: Final = Logging(
model="xai/grok-4.7",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="xai-stamp-dropped",
function_id="1",
)
logging_obj.model_call_details["custom_llm_provider"] = "xai"
logging_obj.optional_params = {}
_set_stream_builder_response_cost(response, logging_obj, chunks=())
assert response._hidden_params.get("response_cost") is None
def test_stream_spend_prices_vertex_anthropic_cache_read_tokens():
"""#37923: streamed Vertex Anthropic usage with cache_read_input_tokens > 0
must be priced the same way as cost_per_token on that Usage, not logged as $0.
"""
from litellm.cost_calculator import get_response_cost_from_hidden_params
cache_read = 44616
prompt_tokens = 70961
completion_tokens = 4096
chunks = _vertex_anthropic_stream_chunks(
cache_read=cache_read,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
)
complete_response = litellm.stream_chunk_builder(
chunks=chunks,
messages=[{"role": "user", "content": "count to five"}],
)
assert complete_response is not None
usage = complete_response.usage
assert usage.cache_read_input_tokens == cache_read
assert usage.prompt_tokens == prompt_tokens
assert usage.completion_tokens == completion_tokens
token_prompt, token_completion = litellm.cost_per_token(
model="claude-opus-5",
custom_llm_provider="vertex_ai",
usage_object=usage,
)
token_total = token_prompt + token_completion
assert token_total > 0
# A leftover usage.cost=0 must not override token-based stream spend.
# Production path: stream_chunk_builder stamps hidden response_cost from usage.cost.
zero_cost_response = complete_response.model_copy(update={"usage": usage.model_copy(update={"cost": 0.0})})
logging_for_builder = Logging(
model="claude-opus-5",
messages=[{"role": "user", "content": "count to five"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="stream-spend-cache-read-builder",
function_id="1245",
)
logging_for_builder.model_call_details["custom_llm_provider"] = "vertex_ai"
logging_for_builder.optional_params = {}
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
_set_stream_builder_response_cost(zero_cost_response, logging_for_builder, chunks=chunks)
assert zero_cost_response._hidden_params.get("response_cost") is None
assert get_response_cost_from_hidden_params(zero_cost_response._hidden_params) is None
stream_spend = _stream_spend_via_logging(
zero_cost_response,
model="claude-opus-5",
custom_llm_provider="vertex_ai",
)
# Stream spend uses the same Usage; regional Vertex uplift may raise the
# logged figure above the raw cost_per_token total, but it must not be 0.
assert stream_spend > 0
assert stream_spend >= token_total - 1e-12
def test_stream_spend_prices_vertex_anthropic_without_cache_read():
"""Stream without cache-read tokens must still price from the assembled usage."""
chunks = _vertex_anthropic_stream_chunks(
cache_read=0, prompt_tokens=9, completion_tokens=16
)
complete_response = litellm.stream_chunk_builder(
chunks=chunks,
messages=[{"role": "user", "content": "count to five"}],
)
assert complete_response is not None
token_prompt, token_completion = litellm.cost_per_token(
model="claude-opus-5",
custom_llm_provider="vertex_ai",
usage_object=complete_response.usage,
)
token_total = token_prompt + token_completion
assert token_total == pytest.approx(9 * 5e-6 + 16 * 2.5e-5)
stream_spend = _stream_spend_via_logging(
complete_response,
model="claude-opus-5",
custom_llm_provider="vertex_ai",
)
assert stream_spend > 0
assert stream_spend >= token_total - 1e-12
def test_vertex_chunk_creator_drops_zero_cost_and_prices_tokens():
"""Vertex stream chunks can carry usage.cost=0 with real token counts.
chunk_creator must not keep that 0 as a provider total. Token pricing on
the assembled stream still has to be a positive spend.
"""
logging_obj = Logging(
model="claude-opus-5",
messages=[{"role": "user", "content": "count to five"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="vertex-zero-cost-chunk",
function_id="1245",
)
logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai"
logging_obj.optional_params = {}
wrapper = CustomStreamWrapper(
completion_stream=None,
model="claude-opus-5",
logging_obj=logging_obj,
custom_llm_provider="vertex_ai",
)
source_chunks = _vertex_anthropic_stream_chunks(cache_read=44616, prompt_tokens=70961, completion_tokens=4096)
zero_cost_usage = source_chunks[-1].usage.model_copy(update={"cost": 0})
zero_cost_chunk = ModelResponseStream(
id="chatcmpl-vertex-zero-cost",
created=1745513207,
model="claude-opus-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(content="Hi", role="assistant"),
)
],
usage=zero_cost_usage,
)
processed = wrapper.chunk_creator(chunk=zero_cost_chunk)
assert processed is not None
assert processed.usage.prompt_tokens == 70961
assert processed.usage.completion_tokens == 4096
assert processed.usage.cache_read_input_tokens == 44616
assert getattr(processed.usage, "cost", None) in (None,)
priced_chunk = ModelResponseStream(
id="chatcmpl-vertex-priced",
created=1745513208,
model="claude-opus-5",
object="chat.completion.chunk",
choices=[
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(content="Hi", role="assistant"),
)
],
usage=zero_cost_usage.model_copy(update={"cost": 0.01}),
)
priced = wrapper.chunk_creator(chunk=priced_chunk)
assert priced is not None
assert priced.usage.cost == 0.01
complete_response = litellm.stream_chunk_builder(
chunks=[processed],
messages=[{"role": "user", "content": "count to five"}],
logging_obj=logging_obj,
)
assert complete_response is not None
token_prompt, token_completion = litellm.cost_per_token(
model="claude-opus-5",
custom_llm_provider="vertex_ai",
usage_object=complete_response.usage,
)
token_total = token_prompt + token_completion
assert token_total > 0
stream_spend = _stream_spend_via_logging(
complete_response,
model="claude-opus-5",
custom_llm_provider="vertex_ai",
)
assert stream_spend > 0
assert stream_spend >= token_total - 1e-12
def test_handle_special_delta_attributes(