mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
fix(cost): price bridged /v1/messages reasoning tokens from the chat completion, not the served Anthropic body (#46726)
* fix(cost): price bridged /v1/messages reasoning tokens from the chat completion, not the served Anthropic body * fix(cost): keep the type gate flat for the bridged /v1/messages billing path --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
54ff82dd14
commit
11e9fcc033
5 changed files with 148 additions and 10 deletions
|
|
@ -572,6 +572,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
streamed_anthropic_message_id: str | None = None
|
||||
bridged_model_response: ModelResponse | None = None
|
||||
classifier_input: Mapping[str, JsonValue] | None = None
|
||||
baseline_cache_context: "BaselineCacheContext | None" = None
|
||||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
|
|
@ -1864,7 +1865,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return 0.0
|
||||
|
||||
transformed_result: Final = self._generate_content_result_as_model_response(result)
|
||||
response_result: Final[object] = transformed_result if transformed_result is not None else result
|
||||
response_result: Final[object] = (
|
||||
transformed_result if transformed_result is not None else self.billable_result(result)
|
||||
)
|
||||
|
||||
priced_result: Final = (
|
||||
response_result.response
|
||||
|
|
@ -4243,6 +4246,22 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
def record_streamed_anthropic_message_id(self, message_id: str) -> None:
|
||||
self.streamed_anthropic_message_id = message_id
|
||||
|
||||
def record_bridged_model_response(self, response: ModelResponse) -> None:
|
||||
"""The chat completion a /v1/messages response was translated from.
|
||||
|
||||
The Anthropic shape served to the caller has no field for reasoning tokens, so the spend
|
||||
row is built from this response rather than re-parsed from the served body."""
|
||||
self.bridged_model_response = response
|
||||
|
||||
def billable_result(self, result: object) -> object:
|
||||
"""The response the cost calculator prices: a /v1/messages body served through the chat
|
||||
bridge is replaced by the ModelResponse its spend row is built from, anything else is
|
||||
priced as is."""
|
||||
served: Final[object] = result
|
||||
if self.bridged_model_response is None or not isinstance(result, dict):
|
||||
return result
|
||||
return self._anthropic_messages_logged_response(result=served)
|
||||
|
||||
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
|
||||
"""
|
||||
The ModelResponse a /v1/messages spend_logs row is built from.
|
||||
|
|
@ -4284,6 +4303,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return self._translate_responses_api_response_to_model_response(result)
|
||||
|
||||
provider_response_id: Final = _provider_response_id(result)
|
||||
bridged: Final = self.bridged_model_response
|
||||
if bridged is not None and isinstance(result, dict):
|
||||
return bridged if provider_response_id is None else bridged.model_copy(update={"id": provider_response_id})
|
||||
httpx_response: Final = self.model_call_details.get("httpx_response", None)
|
||||
if httpx_response and isinstance(httpx_response, httpx.Response):
|
||||
result = litellm.AnthropicConfig().transform_response(
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing_extensions import TypedDict
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
|
|
@ -368,6 +369,11 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter()
|
|||
########################################################
|
||||
|
||||
|
||||
def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None:
|
||||
if logging_obj is not None and isinstance(completion_response, ModelResponse):
|
||||
logging_obj.record_bridged_model_response(completion_response)
|
||||
|
||||
|
||||
class LiteLLMMessagesToCompletionTransformationHandler:
|
||||
@staticmethod
|
||||
def _route_openai_thinking_to_responses_api_if_needed(
|
||||
|
|
@ -726,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
)
|
||||
|
||||
completion_response: Final = await litellm.acompletion(**completion_kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
|
||||
if stream:
|
||||
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
|
||||
|
|
@ -734,13 +741,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
polyfill_result=polyfill_result,
|
||||
is_async=True,
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
|
||||
litellm_logging_obj=logging_obj,
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
)
|
||||
if transformed_stream is not None:
|
||||
return transformed_stream
|
||||
raise ValueError("Failed to transform streaming response")
|
||||
else:
|
||||
_record_bridged_model_response(logging_obj, completion_response)
|
||||
anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
||||
cast(ModelResponse, completion_response),
|
||||
tool_name_mapping=tool_name_mapping,
|
||||
|
|
@ -865,6 +873,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
)
|
||||
|
||||
completion_response: Final = litellm.completion(**completion_kwargs)
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
|
||||
if stream:
|
||||
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
|
||||
|
|
@ -873,12 +882,13 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
polyfill_result=polyfill_result,
|
||||
is_async=False,
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs),
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
if transformed_stream is not None:
|
||||
return transformed_stream
|
||||
raise ValueError("Failed to transform streaming response")
|
||||
else:
|
||||
_record_bridged_model_response(logging_obj, completion_response)
|
||||
anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
||||
cast(ModelResponse, completion_response),
|
||||
tool_name_mapping=tool_name_mapping,
|
||||
|
|
|
|||
|
|
@ -6319,15 +6319,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs((AZURE_GPT_5_6_INPUT_TEXT, AZURE_GPT_5_6_CACHE_READ, AZURE_GPT_5_6_STREAM), ("/v1/messages",)),
|
||||
*bridge_pairs(
|
||||
(AZURE_GPT_5_6_REASONING,),
|
||||
(AZURE_GPT_5_6_INPUT_TEXT, AZURE_GPT_5_6_CACHE_READ, AZURE_GPT_5_6_STREAM, AZURE_GPT_5_6_REASONING),
|
||||
("/v1/messages",),
|
||||
known_bug=(
|
||||
"the /v1/messages -> chat bridge usually drops completion_tokens_details.reasoning_tokens on azure, so"
|
||||
" reasoning tokens bill at output_cost_per_token instead of output_cost_per_reasoning_token"
|
||||
" (output_cost 0.058176 instead of 0.065484)"
|
||||
),
|
||||
),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_SOL_INPUT_TEXT, bridged=AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_INPUT_TEXT, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON),
|
||||
|
|
|
|||
|
|
@ -5918,6 +5918,58 @@ def test_handle_anthropic_messages_response_logging_keeps_the_served_response_id
|
|||
assert result.service_tier == "flex"
|
||||
|
||||
|
||||
def test_handle_anthropic_messages_response_logging_bills_the_bridged_chat_completion_not_the_served_body():
|
||||
"""A chat deployment answers /v1/messages through the chat bridge. The Anthropic body served to the
|
||||
caller has no reasoning token field, so the spend row must come from the recorded chat completion."""
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, Usage
|
||||
|
||||
chat_completion = ModelResponse(
|
||||
id="chatcmpl-upstream",
|
||||
model="gpt-5.6",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}],
|
||||
usage=Usage(
|
||||
prompt_tokens=1240,
|
||||
completion_tokens=4040,
|
||||
total_tokens=5280,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3480),
|
||||
),
|
||||
)
|
||||
served = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response=chat_completion)
|
||||
assert served is not None
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
logging_obj.record_bridged_model_response(chat_completion)
|
||||
|
||||
logged = logging_obj._handle_anthropic_messages_response_logging(result=dict(served))
|
||||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == served["id"], "the spend log row keeps the id the caller was served"
|
||||
assert logged.usage.completion_tokens == 4040 # type: ignore[attr-defined]
|
||||
assert logged.usage.completion_tokens_details.reasoning_tokens == 3480 # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_handle_anthropic_messages_response_logging_reparses_the_served_body_without_a_bridged_completion():
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
logged = logging_obj._handle_anthropic_messages_response_logging(
|
||||
result={
|
||||
"id": "msg_native",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [{"type": "text", "text": "answer"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 7},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(logged, ModelResponse)
|
||||
assert logged.id == "msg_native"
|
||||
assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_handle_anthropic_messages_response_logging_passes_model_response_through():
|
||||
"""Anthropic-native path already yields a ModelResponse; it must be returned unchanged."""
|
||||
logging_obj = _anthropic_messages_logging_obj()
|
||||
|
|
|
|||
|
|
@ -10919,3 +10919,63 @@ class TestStreamingContainerOwnershipRecordedBeforeDone:
|
|||
assert tuple(chunk for chunk, _ in observed) == self.CHUNKS
|
||||
assert tuple(count for _, count in observed) == (0, 0, 0, 0)
|
||||
recorder.assert_awaited_once()
|
||||
|
||||
|
||||
class TestResponseCostFromLoggingObjForBridgedMessages:
|
||||
"""A /v1/messages request answered through the chat bridge arrives here as an Anthropic body whose
|
||||
usage has no reasoning field, while the cost it recomputes must still price the reasoning tokens
|
||||
the provider reported (LIT-9666)."""
|
||||
|
||||
@staticmethod
|
||||
def _logging_obj(model: str):
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
logging_obj = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="lit-9666",
|
||||
function_id="lit-9666",
|
||||
)
|
||||
logging_obj.optional_params = {}
|
||||
return logging_obj
|
||||
|
||||
def test_recomputed_cost_prices_the_reasoning_tokens_of_the_bridged_chat_completion(self):
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
model = "dashscope/qwen-turbo"
|
||||
pricing = litellm.get_model_info(model)
|
||||
input_rate = pricing["input_cost_per_token"]
|
||||
output_rate = pricing["output_cost_per_token"]
|
||||
reasoning_rate = pricing["output_cost_per_reasoning_token"]
|
||||
assert input_rate is not None and output_rate is not None and reasoning_rate is not None
|
||||
assert reasoning_rate != output_rate
|
||||
chat_completion = ModelResponse(
|
||||
id="chatcmpl-upstream",
|
||||
model="qwen-turbo",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}],
|
||||
usage=Usage(
|
||||
prompt_tokens=1240,
|
||||
completion_tokens=4040,
|
||||
total_tokens=5280,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3480),
|
||||
),
|
||||
)
|
||||
served = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response=chat_completion)
|
||||
assert served is not None and "reasoning" not in json.dumps(served["usage"])
|
||||
logging_obj = self._logging_obj(model)
|
||||
logging_obj.record_bridged_model_response(chat_completion)
|
||||
|
||||
cost = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj(
|
||||
response=dict(served), logging_obj=logging_obj
|
||||
)
|
||||
|
||||
expected_reasoning = 3480 * reasoning_rate
|
||||
expected_output = expected_reasoning + (4040 - 3480) * output_rate
|
||||
assert cost == pytest.approx(1240 * input_rate + expected_output)
|
||||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output)
|
||||
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue