mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(cost): price native Claude message bodies the header path cannot strictly parse
The /v1/messages header cost path validated the provider body against the strict AnthropicResponse model, whose content union has no server_tool_use, web_search_tool_result, compaction, or MCP blocks and whose stop_reason has no refusal or pause_turn, so every Claude web search reply fell back to output-rate pricing for its thinking tokens with only a debug line. Hand the raw dict to transform_parsed_response, the same shape the native httpx path already uses, and log the fallback at warning level. Also drop the verbose docstrings Greptile flagged on the new tests
This commit is contained in:
parent
4cefdf4f72
commit
6381af94bb
5 changed files with 71 additions and 26 deletions
|
|
@ -17,7 +17,7 @@ from types import MappingProxyType, TracebackType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
|
||||
|
||||
from httpx import Response
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import (
|
||||
|
|
@ -414,6 +414,9 @@ def _resolve_vertex_location_for_cost(
|
|||
return VertexBase.get_vertex_region(configured_location, model)
|
||||
|
||||
|
||||
_ANTHROPIC_MESSAGE_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _provider_response_id(source: object) -> str | None:
|
||||
candidate: Final = source.get("id") if isinstance(source, dict) else getattr(source, "id", None)
|
||||
return candidate if isinstance(candidate, str) and candidate else None
|
||||
|
|
@ -3891,17 +3894,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
try:
|
||||
return self._anthropic_message_body_as_model_response(result)
|
||||
except Exception as e: # noqa: BLE001 # cost normalization must never break the response path
|
||||
verbose_logger.debug("anthropic message response cost normalization failed: %s", e)
|
||||
verbose_logger.warning("anthropic message response cost normalization failed: %s", e)
|
||||
return None
|
||||
|
||||
def _anthropic_message_body_as_model_response(self, body: object) -> ModelResponse:
|
||||
import httpx
|
||||
|
||||
from litellm.types.llms.anthropic import AnthropicResponse
|
||||
|
||||
parsed: Final = AnthropicResponse.model_validate(body)
|
||||
return litellm.AnthropicConfig().transform_parsed_response(
|
||||
completion_response=parsed.model_dump(),
|
||||
completion_response=_ANTHROPIC_MESSAGE_BODY.validate_python(body),
|
||||
raw_response=httpx.Response(status_code=200, headers={}),
|
||||
model_response=litellm.ModelResponse(id=_provider_response_id(body)),
|
||||
json_mode=None,
|
||||
|
|
|
|||
|
|
@ -2014,14 +2014,6 @@ def test_response_cost_calculator_does_not_transform_non_generate_content_dict()
|
|||
|
||||
|
||||
def test_response_cost_calculator_anthropic_message_body_bills_thinking_at_the_reasoning_rate(monkeypatch):
|
||||
"""
|
||||
Regression for LIT-6908: the ``x-litellm-response-cost`` header for a non-streaming
|
||||
/v1/messages call is priced synchronously from the raw Anthropic message body, whose
|
||||
usage carries no reasoning split, so thinking tokens were billed at the plain output
|
||||
rate while the spend log (priced from the converted ``ModelResponse``) billed them at
|
||||
``output_cost_per_reasoning_token``. The calculator now converts the body first, so
|
||||
both surfaces agree.
|
||||
"""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"lit6908-thinker",
|
||||
|
|
@ -2068,6 +2060,71 @@ def test_response_cost_calculator_anthropic_message_body_bills_thinking_at_the_r
|
|||
)
|
||||
|
||||
|
||||
def test_response_cost_calculator_prices_web_search_message_bodies_without_falling_back(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"lit6908-searcher",
|
||||
{
|
||||
"litellm_provider": "anthropic",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"output_cost_per_reasoning_token": 1e-05,
|
||||
},
|
||||
)
|
||||
logging_obj = LitellmLogging(
|
||||
model="lit6908-searcher",
|
||||
messages=[{"role": "user", "content": "Who won the most recent Formula 1 race?"}],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="lit6908-search",
|
||||
function_id="lit6908-search",
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
|
||||
logging_obj.optional_params = {}
|
||||
body = {
|
||||
"id": "msg_lit6908_search",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "lit6908-searcher",
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "Search first, then answer.", "signature": ""},
|
||||
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "latest F1 race winner"}},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_1",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"title": "Race report",
|
||||
"url": "https://example.com/race",
|
||||
"encrypted_content": "abc",
|
||||
"page_age": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "The most recent race was won by the driver named in the report."},
|
||||
],
|
||||
"stop_reason": "pause_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {
|
||||
"input_tokens": 120,
|
||||
"output_tokens": 60,
|
||||
"output_tokens_details": {"thinking_tokens": 12},
|
||||
"server_tool_use": {"web_search_requests": 1},
|
||||
},
|
||||
}
|
||||
|
||||
normalized = logging_obj._anthropic_message_result_as_model_response(body)
|
||||
|
||||
assert isinstance(normalized, litellm.ModelResponse)
|
||||
assert normalized.usage.completion_tokens_details.reasoning_tokens == 12
|
||||
assert normalized.usage.server_tool_use.web_search_requests == 1
|
||||
assert logging_obj._response_cost_calculator(result=body) == pytest.approx(
|
||||
120 * 1e-06 + 48 * 2e-06 + 12 * 1e-05
|
||||
)
|
||||
|
||||
|
||||
def _file_content_logging_obj(call_type: str) -> LitellmLogging:
|
||||
logging_obj = LitellmLogging(
|
||||
model="gemini-3-flash-preview",
|
||||
|
|
|
|||
|
|
@ -6412,9 +6412,6 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc
|
|||
|
||||
|
||||
def test_is_anthropic_usage_object_accepts_cache_free_usage_reporting_thinking_tokens():
|
||||
"""Only Anthropic-shaped usage nests ``thinking_tokens`` under ``output_tokens_details``,
|
||||
so a cache-free payload carrying it must take the Anthropic mapping, which is the only
|
||||
converter that turns the count into billable reasoning tokens."""
|
||||
assert AnthropicConfig.is_anthropic_usage_object(
|
||||
{"input_tokens": 105, "output_tokens": 77, "output_tokens_details": {"thinking_tokens": 75}}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4662,9 +4662,6 @@ def _reasoning_model_response(model: str, reasoning_tokens: int | None) -> Model
|
|||
|
||||
|
||||
def test_translate_openai_response_to_anthropic_reports_provider_reasoning_tokens_as_thinking_tokens():
|
||||
"""A non-Anthropic model answering over /v1/messages reports its reasoning tokens in the
|
||||
OpenAI usage; the Anthropic usage must carry that count as ``output_tokens_details.thinking_tokens``
|
||||
so cost calculation can bill them at the reasoning rate instead of the output rate."""
|
||||
anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
|
||||
response=_reasoning_model_response(model="deepseek-v4-flash", reasoning_tokens=75)
|
||||
)
|
||||
|
|
@ -4686,8 +4683,6 @@ def test_translate_openai_response_to_anthropic_omits_output_tokens_details_with
|
|||
|
||||
|
||||
def test_translated_messages_response_bills_reasoning_tokens_at_the_reasoning_rate(monkeypatch):
|
||||
"""The response-cost header path prices the translated Anthropic usage; with reasoning tokens
|
||||
carried through, /v1/messages must cost the same as the identical /v1/chat/completions call."""
|
||||
from litellm.cost_calculator import _get_usage_object
|
||||
|
||||
model = "lit6908-reasoning-model"
|
||||
|
|
@ -4724,8 +4719,6 @@ def test_translated_messages_response_bills_reasoning_tokens_at_the_reasoning_ra
|
|||
|
||||
|
||||
def test_spend_log_usage_for_translated_messages_response_uses_the_reported_thinking_tokens():
|
||||
"""The spend row is built by re-parsing the translated Anthropic response; it must bill the
|
||||
provider's reported reasoning count rather than a token-counter estimate of the thinking text."""
|
||||
import httpx
|
||||
|
||||
from litellm.types.llms.anthropic import AnthropicResponse
|
||||
|
|
|
|||
|
|
@ -88,8 +88,6 @@ def test_final_usage_chunk_without_choices_is_merged_into_message_delta():
|
|||
|
||||
|
||||
def test_final_usage_chunk_reasoning_tokens_land_on_message_delta_as_thinking_tokens():
|
||||
"""A reasoning model's final usage chunk reports ``reasoning_tokens``; the Anthropic
|
||||
``message_delta`` usage must carry them as ``output_tokens_details.thinking_tokens``."""
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper
|
||||
|
||||
usage = Usage(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue