mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge ce9dc9fd48 into b25ea48c4b
This commit is contained in:
commit
5c35a42178
4 changed files with 396 additions and 21 deletions
|
|
@ -55,6 +55,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
UnreportedUsage,
|
||||
Usage,
|
||||
hidden_response_cost_value,
|
||||
)
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
|
||||
|
||||
|
|
@ -1381,7 +1382,12 @@ def _stream_builder_response_cost(
|
|||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
# A non-positive leftover (common on Vertex Anthropic stream assembly) is not a provider total.
|
||||
if (
|
||||
isinstance(usage_cost, (int, float))
|
||||
and usage_cost > 0
|
||||
and not _reported_cost_is_priced_by_calculator(logging_obj)
|
||||
):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
|
|
@ -1421,15 +1427,31 @@ def _set_stream_builder_response_cost(
|
|||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> None:
|
||||
response_cost: Final = _stream_builder_response_cost(response, logging_obj, chunks)
|
||||
if response_cost is None:
|
||||
if response_cost is not None:
|
||||
_set_hidden_response_cost(response, response_cost)
|
||||
return
|
||||
_set_hidden_response_cost(response, response_cost)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
hidden_cost: Final = hidden_response_cost_value(response)
|
||||
# xAI is priced by the calculator, so the provider total here is None even after
|
||||
# the stamp wrote the same positive number onto usage.cost and hidden response_cost.
|
||||
# Keep that pair. Any other leftover, including a Vertex 0, has to go so token pricing runs.
|
||||
if (
|
||||
isinstance(usage_cost, (int, float))
|
||||
and not isinstance(usage_cost, bool)
|
||||
and isinstance(hidden_cost, (int, float))
|
||||
and not isinstance(hidden_cost, bool)
|
||||
and usage_cost > 0
|
||||
and float(usage_cost) == float(hidden_cost)
|
||||
):
|
||||
return
|
||||
response.hidden_params.pop("response_cost", None) # pyright: ignore[reportUnknownMemberType] # legacy dict
|
||||
|
||||
|
||||
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
|
||||
if logging_obj is None:
|
||||
return
|
||||
if isinstance(getattr(usage, "cost", None), (int, float)):
|
||||
existing_cost: Final = getattr(usage, "cost", None)
|
||||
if isinstance(existing_cost, (int, float)) and existing_cost > 0:
|
||||
return
|
||||
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
|
||||
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:
|
||||
|
|
|
|||
|
|
@ -1732,10 +1732,28 @@ class CustomStreamWrapper:
|
|||
self.tool_call = True
|
||||
|
||||
chunk_usage: Final = getattr(chunk, "usage", None)
|
||||
if isinstance(chunk_usage, Usage):
|
||||
model_response.usage = chunk_usage
|
||||
elif isinstance(chunk_usage, BaseModel):
|
||||
model_response.usage = Usage(**chunk_usage.model_dump())
|
||||
parsed_usage: Final = (
|
||||
chunk_usage
|
||||
if isinstance(chunk_usage, Usage)
|
||||
else Usage(**chunk_usage.model_dump())
|
||||
if isinstance(chunk_usage, BaseModel)
|
||||
else None
|
||||
)
|
||||
# vertex_ai returns before cost propagation, so a leftover usage.cost=0 would
|
||||
# be read downstream as a provider total. Drop it so token pricing runs.
|
||||
reported_cost: Final = getattr(parsed_usage, "cost", None)
|
||||
drop_reported_cost: Final = (
|
||||
self.custom_llm_provider == "vertex_ai"
|
||||
and reported_cost is not None
|
||||
and _usable_provider_reported_cost(reported_cost) is None
|
||||
)
|
||||
attached_usage: Final = (
|
||||
parsed_usage.model_copy(update={"cost": None})
|
||||
if parsed_usage is not None and drop_reported_cost
|
||||
else parsed_usage
|
||||
)
|
||||
if attached_usage is not None:
|
||||
model_response.usage = attached_usage
|
||||
|
||||
## RETURN ARG
|
||||
if isinstance(chunk, ModelResponseStream) and chunk.provider_specific_fields:
|
||||
|
|
@ -1985,17 +2003,7 @@ class CustomStreamWrapper:
|
|||
|
||||
@staticmethod
|
||||
def _resolve_provider_reported_cost(usage_cost: object) -> float | None:
|
||||
"""
|
||||
Providers report usage.cost either as a number or as a breakdown object
|
||||
whose total lives under ``total_cost``.
|
||||
"""
|
||||
if isinstance(usage_cost, bool):
|
||||
return None
|
||||
if isinstance(usage_cost, (int, float)):
|
||||
return float(usage_cost)
|
||||
if isinstance(usage_cost, dict):
|
||||
return CustomStreamWrapper._resolve_provider_reported_cost(usage_cost.get("total_cost"))
|
||||
return None
|
||||
return _usable_provider_reported_cost(usage_cost)
|
||||
|
||||
@staticmethod
|
||||
def _propagate_usage_cost_to_hidden_params(
|
||||
|
|
@ -2757,6 +2765,28 @@ def _reported_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream])
|
|||
return calculate_total_usage(chunks=chunks)
|
||||
|
||||
|
||||
def _positive_provider_cost(usage_cost: object) -> float | None:
|
||||
if isinstance(usage_cost, bool) or not isinstance(usage_cost, (int, float)) or usage_cost <= 0:
|
||||
return None
|
||||
return float(usage_cost)
|
||||
|
||||
|
||||
def _usable_provider_reported_cost(usage_cost: object) -> float | None:
|
||||
"""
|
||||
Providers report usage.cost either as a number or as a breakdown object
|
||||
whose total lives under ``total_cost``.
|
||||
|
||||
Zero / negative is not a real provider total. Stream assembly can leave
|
||||
``usage.cost = 0`` on Vertex Anthropic chunks that still have tokens
|
||||
(including cache-read). Treating that 0 as authoritative made the
|
||||
stream spend path log spend=0 while ``cost_per_token`` on the same
|
||||
Usage priced correctly. Leave it absent so token-based pricing runs.
|
||||
"""
|
||||
if isinstance(usage_cost, dict):
|
||||
return _positive_provider_cost(usage_cost.get("total_cost"))
|
||||
return _positive_provider_cost(usage_cost)
|
||||
|
||||
|
||||
def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) -> Usage:
|
||||
"""Assume most recent usage chunk has total usage uptil then."""
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import (
|
||||
|
|
@ -2805,8 +2835,9 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream])
|
|||
if isinstance(latest_usage_chunk, dict)
|
||||
else getattr(latest_usage_chunk, "cost", None)
|
||||
)
|
||||
if latest_cost is not None:
|
||||
returned_usage_chunk.cost = latest_cost
|
||||
resolved_cost: Final = _usable_provider_reported_cost(latest_cost)
|
||||
if resolved_cost is not None:
|
||||
returned_usage_chunk.cost = resolved_cost
|
||||
|
||||
return returned_usage_chunk
|
||||
|
||||
|
|
|
|||
|
|
@ -2392,6 +2392,11 @@ class ModelResponse(ModelResponseBase):
|
|||
return self.dict()
|
||||
|
||||
|
||||
def hidden_response_cost_value(response: ModelResponse) -> object | None:
|
||||
# Module scope so `object` is the builtin. Inside ModelResponse it is shadowed by the `object` field.
|
||||
return response.hidden_params.get("response_cost")
|
||||
|
||||
|
||||
class Embedding(OpenAIObject):
|
||||
embedding: list | str = []
|
||||
index: int
|
||||
|
|
|
|||
|
|
@ -2728,6 +2728,323 @@ def test_provider_reported_cost_ignores_unusable_shapes():
|
|||
assert CustomStreamWrapper._resolve_provider_reported_cost({}) is None
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": None}) is None
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost(0.5) == 0.5
|
||||
# 0 is not a real provider total — fall through to token-based pricing
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost(0) is None
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost(0.0) is None
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost({"total_cost": 0}) is None
|
||||
|
||||
|
||||
def _vertex_anthropic_stream_chunks(*, cache_read: int, prompt_tokens: int, completion_tokens: int):
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
config = AnthropicConfig()
|
||||
raw_input = prompt_tokens - cache_read
|
||||
start_usage = config.calculate_usage(
|
||||
usage_object={
|
||||
"input_tokens": raw_input,
|
||||
"cache_read_input_tokens": cache_read,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"output_tokens": 1,
|
||||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
delta_usage = config.calculate_usage(
|
||||
usage_object={
|
||||
"input_tokens": raw_input,
|
||||
"cache_read_input_tokens": cache_read,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"output_tokens": completion_tokens,
|
||||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
return [
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-vertex-anthropic-cache",
|
||||
created=1745513206,
|
||||
model="claude-opus-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason=None,
|
||||
index=0,
|
||||
delta=Delta(content="Hi", role="assistant"),
|
||||
)
|
||||
],
|
||||
usage=start_usage,
|
||||
),
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-vertex-anthropic-cache",
|
||||
created=1745513207,
|
||||
model="claude-opus-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content=""),
|
||||
)
|
||||
],
|
||||
usage=delta_usage,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _stream_spend_via_logging(complete_response, *, model: str, custom_llm_provider: str) -> float:
|
||||
"""Price an assembled stream the same way success_handler does."""
|
||||
logging_obj = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="stream-spend-cache-read",
|
||||
function_id="1245",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
|
||||
logging_obj.optional_params = {}
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(
|
||||
complete_response, custom_llm_provider
|
||||
)
|
||||
cost = logging_obj._response_cost_calculator(result=complete_response)
|
||||
assert cost is not None
|
||||
return cost
|
||||
|
||||
|
||||
def test_set_stream_builder_response_cost_keeps_matching_xai_stamp():
|
||||
"""A calculator-priced stamp is not a provider total, but it is the spend figure.
|
||||
|
||||
Dropping it made the assembled response report usage.cost and then lose
|
||||
hidden response_cost, so the xAI stream spend row came back as None.
|
||||
"""
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
|
||||
|
||||
stamped: Final = 0.006152
|
||||
response: Final = ModelResponse(
|
||||
id="chatcmpl-xai-stamp",
|
||||
model="grok-4.7",
|
||||
choices=[],
|
||||
usage=Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252, cost=stamped),
|
||||
)
|
||||
response._hidden_params["response_cost"] = stamped
|
||||
logging_obj: Final = Logging(
|
||||
model="xai/grok-4.7",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="xai-stamp-kept",
|
||||
function_id="1",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "xai"
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
_set_stream_builder_response_cost(response, logging_obj, chunks=())
|
||||
|
||||
assert response._hidden_params.get("response_cost") == pytest.approx(stamped)
|
||||
|
||||
|
||||
def test_set_stream_builder_response_cost_drops_xai_hidden_cost_that_differs_from_usage():
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
|
||||
|
||||
response: Final = ModelResponse(
|
||||
id="chatcmpl-xai-stale",
|
||||
model="grok-4.7",
|
||||
choices=[],
|
||||
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5),
|
||||
)
|
||||
response._hidden_params["response_cost"] = 0.1
|
||||
logging_obj: Final = Logging(
|
||||
model="xai/grok-4.7",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="xai-stamp-dropped",
|
||||
function_id="1",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "xai"
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
_set_stream_builder_response_cost(response, logging_obj, chunks=())
|
||||
|
||||
assert response._hidden_params.get("response_cost") is None
|
||||
|
||||
|
||||
def test_stream_spend_prices_vertex_anthropic_cache_read_tokens():
|
||||
"""#37923: streamed Vertex Anthropic usage with cache_read_input_tokens > 0
|
||||
must be priced the same way as cost_per_token on that Usage, not logged as $0.
|
||||
"""
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
|
||||
cache_read = 44616
|
||||
prompt_tokens = 70961
|
||||
completion_tokens = 4096
|
||||
chunks = _vertex_anthropic_stream_chunks(
|
||||
cache_read=cache_read,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
)
|
||||
complete_response = litellm.stream_chunk_builder(
|
||||
chunks=chunks,
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
)
|
||||
assert complete_response is not None
|
||||
usage = complete_response.usage
|
||||
assert usage.cache_read_input_tokens == cache_read
|
||||
assert usage.prompt_tokens == prompt_tokens
|
||||
assert usage.completion_tokens == completion_tokens
|
||||
|
||||
token_prompt, token_completion = litellm.cost_per_token(
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage_object=usage,
|
||||
)
|
||||
token_total = token_prompt + token_completion
|
||||
assert token_total > 0
|
||||
|
||||
# A leftover usage.cost=0 must not override token-based stream spend.
|
||||
# Production path: stream_chunk_builder stamps hidden response_cost from usage.cost.
|
||||
zero_cost_response = complete_response.model_copy(update={"usage": usage.model_copy(update={"cost": 0.0})})
|
||||
logging_for_builder = Logging(
|
||||
model="claude-opus-5",
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="stream-spend-cache-read-builder",
|
||||
function_id="1245",
|
||||
)
|
||||
logging_for_builder.model_call_details["custom_llm_provider"] = "vertex_ai"
|
||||
logging_for_builder.optional_params = {}
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import _set_stream_builder_response_cost
|
||||
|
||||
_set_stream_builder_response_cost(zero_cost_response, logging_for_builder, chunks=chunks)
|
||||
assert zero_cost_response._hidden_params.get("response_cost") is None
|
||||
assert get_response_cost_from_hidden_params(zero_cost_response._hidden_params) is None
|
||||
|
||||
stream_spend = _stream_spend_via_logging(
|
||||
zero_cost_response,
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
# Stream spend uses the same Usage; regional Vertex uplift may raise the
|
||||
# logged figure above the raw cost_per_token total, but it must not be 0.
|
||||
assert stream_spend > 0
|
||||
assert stream_spend >= token_total - 1e-12
|
||||
|
||||
|
||||
def test_stream_spend_prices_vertex_anthropic_without_cache_read():
|
||||
"""Stream without cache-read tokens must still price from the assembled usage."""
|
||||
chunks = _vertex_anthropic_stream_chunks(
|
||||
cache_read=0, prompt_tokens=9, completion_tokens=16
|
||||
)
|
||||
complete_response = litellm.stream_chunk_builder(
|
||||
chunks=chunks,
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
)
|
||||
assert complete_response is not None
|
||||
token_prompt, token_completion = litellm.cost_per_token(
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage_object=complete_response.usage,
|
||||
)
|
||||
token_total = token_prompt + token_completion
|
||||
assert token_total == pytest.approx(9 * 5e-6 + 16 * 2.5e-5)
|
||||
stream_spend = _stream_spend_via_logging(
|
||||
complete_response,
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
assert stream_spend > 0
|
||||
assert stream_spend >= token_total - 1e-12
|
||||
|
||||
|
||||
def test_vertex_chunk_creator_drops_zero_cost_and_prices_tokens():
|
||||
"""Vertex stream chunks can carry usage.cost=0 with real token counts.
|
||||
|
||||
chunk_creator must not keep that 0 as a provider total. Token pricing on
|
||||
the assembled stream still has to be a positive spend.
|
||||
"""
|
||||
logging_obj = Logging(
|
||||
model="claude-opus-5",
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="vertex-zero-cost-chunk",
|
||||
function_id="1245",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai"
|
||||
logging_obj.optional_params = {}
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="claude-opus-5",
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
source_chunks = _vertex_anthropic_stream_chunks(cache_read=44616, prompt_tokens=70961, completion_tokens=4096)
|
||||
zero_cost_usage = source_chunks[-1].usage.model_copy(update={"cost": 0})
|
||||
zero_cost_chunk = ModelResponseStream(
|
||||
id="chatcmpl-vertex-zero-cost",
|
||||
created=1745513207,
|
||||
model="claude-opus-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content="Hi", role="assistant"),
|
||||
)
|
||||
],
|
||||
usage=zero_cost_usage,
|
||||
)
|
||||
|
||||
processed = wrapper.chunk_creator(chunk=zero_cost_chunk)
|
||||
|
||||
assert processed is not None
|
||||
assert processed.usage.prompt_tokens == 70961
|
||||
assert processed.usage.completion_tokens == 4096
|
||||
assert processed.usage.cache_read_input_tokens == 44616
|
||||
assert getattr(processed.usage, "cost", None) in (None,)
|
||||
|
||||
priced_chunk = ModelResponseStream(
|
||||
id="chatcmpl-vertex-priced",
|
||||
created=1745513208,
|
||||
model="claude-opus-5",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content="Hi", role="assistant"),
|
||||
)
|
||||
],
|
||||
usage=zero_cost_usage.model_copy(update={"cost": 0.01}),
|
||||
)
|
||||
priced = wrapper.chunk_creator(chunk=priced_chunk)
|
||||
assert priced is not None
|
||||
assert priced.usage.cost == 0.01
|
||||
|
||||
complete_response = litellm.stream_chunk_builder(
|
||||
chunks=[processed],
|
||||
messages=[{"role": "user", "content": "count to five"}],
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
assert complete_response is not None
|
||||
token_prompt, token_completion = litellm.cost_per_token(
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage_object=complete_response.usage,
|
||||
)
|
||||
token_total = token_prompt + token_completion
|
||||
assert token_total > 0
|
||||
stream_spend = _stream_spend_via_logging(
|
||||
complete_response,
|
||||
model="claude-opus-5",
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
assert stream_spend > 0
|
||||
assert stream_spend >= token_total - 1e-12
|
||||
|
||||
|
||||
def test_handle_special_delta_attributes(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue