mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(anthropic): keep usage absent when a response omits it (#44535)
Instead of injecting `{}` so calculate_usage fabricates a 0/0 usage, a
successful non-streaming response that carries no usage object now keeps
`model_response.usage` as None and marks `_hidden_params["usage_missing"]`,
so spend tracking and budgets can distinguish "no usage reported" from a
genuine zero instead of recording the call as free.
Signed-off-by: apex-mochen <2756823972@qq.com>
This commit is contained in:
parent
a3236ba947
commit
b5132b7c23
2 changed files with 70 additions and 10 deletions
|
|
@ -6,9 +6,6 @@ from types import MappingProxyType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
|
||||
|
|
@ -95,6 +92,8 @@ from litellm.utils import (
|
|||
supports_reasoning,
|
||||
token_counter,
|
||||
)
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from ..common_utils import (
|
||||
AnthropicError,
|
||||
|
|
@ -2740,13 +2739,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
map_finish_reason(completion_response["stop_reason"]),
|
||||
)
|
||||
|
||||
usage: Final = self.calculate_usage(
|
||||
usage_object=completion_response["usage"],
|
||||
reasoning_content=reasoning_content,
|
||||
completion_response=completion_response,
|
||||
speed=speed,
|
||||
)
|
||||
setattr(model_response, "usage", usage)
|
||||
usage_object: Final = completion_response.get("usage")
|
||||
if usage_object is None:
|
||||
# #44535: keep usage absent (None) instead of fabricating a 0/0
|
||||
# usage object. Spend tracking and budgets must be able to tell
|
||||
# "no usage reported" apart from a genuine zero, otherwise a
|
||||
# response that omits usage would be recorded as free even though
|
||||
# the upstream consumed tokens.
|
||||
setattr(model_response, "usage", None)
|
||||
_hidden_params["usage_missing"] = True
|
||||
else:
|
||||
setattr(
|
||||
model_response,
|
||||
"usage",
|
||||
self.calculate_usage(
|
||||
usage_object=usage_object,
|
||||
reasoning_content=reasoning_content,
|
||||
completion_response=completion_response,
|
||||
speed=speed,
|
||||
),
|
||||
)
|
||||
|
||||
model_response.created = int(time.time())
|
||||
model_response.model = completion_response["model"]
|
||||
|
|
|
|||
|
|
@ -6805,6 +6805,54 @@ def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identic
|
|||
assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result"
|
||||
|
||||
|
||||
def test_transform_parsed_response_without_usage():
|
||||
"""#44535: a non-streaming Anthropic response without a usage object
|
||||
must not raise KeyError. LiteLLM keeps the usage absent (None) instead
|
||||
of fabricating a 0/0 usage, so spend tracking and budgets can tell "no
|
||||
usage reported" apart from a genuine zero, and mark it in hidden params.
|
||||
|
||||
Regression for: KeyError: 'usage' in transform_parsed_response, which
|
||||
the router mapped to APIConnectionError (HTTP 500), retried
|
||||
num_retries times, and surfaced as a 500 to the client.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
config = AnthropicConfig()
|
||||
|
||||
completion_response = {
|
||||
"id": "msg_01",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"content": [{"type": "text", "text": "Hello there"}],
|
||||
"stop_reason": "end_turn",
|
||||
}
|
||||
|
||||
for variant in ("missing", "none"):
|
||||
response = copy.deepcopy(completion_response)
|
||||
if variant == "none":
|
||||
response["usage"] = None
|
||||
|
||||
raw_response = httpx.Response(status_code=200, headers={})
|
||||
model_response = ModelResponse()
|
||||
|
||||
result = config.transform_parsed_response(
|
||||
completion_response=response,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
json_mode=False,
|
||||
prefix_prompt=None,
|
||||
)
|
||||
|
||||
# Usage stays absent (None), not a fabricated 0/0, and the response
|
||||
# path is explicitly marked so cost tracking can distinguish "no
|
||||
# usage reported" from a genuine zero.
|
||||
assert result.usage is None
|
||||
assert result._hidden_params.get("usage_missing") is True
|
||||
|
||||
|
||||
anthropic_chunk_list: Final = [
|
||||
{
|
||||
"type": "content_block_start",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue