diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ebb981546c7..bbeacc3c584 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -572,6 +572,7 @@ class Logging(LiteLLMLoggingBaseClass): stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None + bridged_model_response: ModelResponse | None = None classifier_input: Mapping[str, JsonValue] | None = None baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None @@ -1864,7 +1865,9 @@ class Logging(LiteLLMLoggingBaseClass): return 0.0 transformed_result: Final = self._generate_content_result_as_model_response(result) - response_result: Final[object] = transformed_result if transformed_result is not None else result + response_result: Final[object] = ( + transformed_result if transformed_result is not None else self.billable_result(result) + ) priced_result: Final = ( response_result.response @@ -4243,6 +4246,22 @@ class Logging(LiteLLMLoggingBaseClass): def record_streamed_anthropic_message_id(self, message_id: str) -> None: self.streamed_anthropic_message_id = message_id + def record_bridged_model_response(self, response: ModelResponse) -> None: + """The chat completion a /v1/messages response was translated from. + + The Anthropic shape served to the caller has no field for reasoning tokens, so the spend + row is built from this response rather than re-parsed from the served body.""" + self.bridged_model_response = response + + def billable_result(self, result: object) -> object: + """The response the cost calculator prices: a /v1/messages body served through the chat + bridge is replaced by the ModelResponse its spend row is built from, anything else is + priced as is.""" + served: Final[object] = result + if self.bridged_model_response is None or not isinstance(result, dict): + return result + return self._anthropic_messages_logged_response(result=served) + def _anthropic_messages_logged_response(self, result: object) -> ModelResponse: """ The ModelResponse a /v1/messages spend_logs row is built from. @@ -4284,6 +4303,9 @@ class Logging(LiteLLMLoggingBaseClass): return self._translate_responses_api_response_to_model_response(result) provider_response_id: Final = _provider_response_id(result) + bridged: Final = self.bridged_model_response + if bridged is not None and isinstance(result, dict): + return bridged if provider_response_id is None else bridged.model_copy(update={"id": provider_response_id}) httpx_response: Final = self.model_call_details.get("httpx_response", None) if httpx_response and isinstance(httpx_response, httpx.Response): result = litellm.AnthropicConfig().transform_response( diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 9c8ab02d552..4c0613777c6 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, ) @@ -368,6 +369,11 @@ ANTHROPIC_ADAPTER: Final = AnthropicAdapter() ######################################################## +def _record_bridged_model_response(logging_obj: LiteLLMLoggingObject | None, completion_response: object) -> None: + if logging_obj is not None and isinstance(completion_response, ModelResponse): + logging_obj.record_bridged_model_response(completion_response) + + class LiteLLMMessagesToCompletionTransformationHandler: @staticmethod def _route_openai_thinking_to_responses_api_if_needed( @@ -726,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) completion_response: Final = await litellm.acompletion(**completion_kwargs) + logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -734,13 +741,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, - litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs), + litellm_logging_obj=logging_obj, safeguards_evaluator=safeguards_evaluator, ) if transformed_stream is not None: return transformed_stream raise ValueError("Failed to transform streaming response") else: + _record_bridged_model_response(logging_obj, completion_response) anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params( cast(ModelResponse, completion_response), tool_name_mapping=tool_name_mapping, @@ -865,6 +873,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) completion_response: Final = litellm.completion(**completion_kwargs) + logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -873,12 +882,13 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, - litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs), + litellm_logging_obj=logging_obj, ) if transformed_stream is not None: return transformed_stream raise ValueError("Failed to transform streaming response") else: + _record_bridged_model_response(logging_obj, completion_response) anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params( cast(ModelResponse, completion_response), tool_name_mapping=tool_name_mapping, diff --git a/tests/integration/cost_calculation/cases/azure.py b/tests/integration/cost_calculation/cases/azure.py index 6cf7cb23d2e..14c9647dc08 100644 --- a/tests/integration/cost_calculation/cases/azure.py +++ b/tests/integration/cost_calculation/cases/azure.py @@ -6319,15 +6319,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( - *bridge_pairs((AZURE_GPT_5_6_INPUT_TEXT, AZURE_GPT_5_6_CACHE_READ, AZURE_GPT_5_6_STREAM), ("/v1/messages",)), *bridge_pairs( - (AZURE_GPT_5_6_REASONING,), + (AZURE_GPT_5_6_INPUT_TEXT, AZURE_GPT_5_6_CACHE_READ, AZURE_GPT_5_6_STREAM, AZURE_GPT_5_6_REASONING), ("/v1/messages",), - known_bug=( - "the /v1/messages -> chat bridge usually drops completion_tokens_details.reasoning_tokens on azure, so" - " reasoning tokens bill at output_cost_per_token instead of output_cost_per_reasoning_token" - " (output_cost 0.058176 instead of 0.065484)" - ), ), BridgeCostParityTestCase(native=AZURE_GPT_6_SOL_INPUT_TEXT, bridged=AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON), BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_INPUT_TEXT, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON), diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index fbe3766f654..12cc57ada4c 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -5918,6 +5918,58 @@ def test_handle_anthropic_messages_response_logging_keeps_the_served_response_id assert result.service_tier == "flex" +def test_handle_anthropic_messages_response_logging_bills_the_bridged_chat_completion_not_the_served_body(): + """A chat deployment answers /v1/messages through the chat bridge. The Anthropic body served to the + caller has no reasoning token field, so the spend row must come from the recorded chat completion.""" + from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.types.utils import CompletionTokensDetailsWrapper, Usage + + chat_completion = ModelResponse( + id="chatcmpl-upstream", + model="gpt-5.6", + choices=[{"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}], + usage=Usage( + prompt_tokens=1240, + completion_tokens=4040, + total_tokens=5280, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3480), + ), + ) + served = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response=chat_completion) + assert served is not None + logging_obj = _anthropic_messages_logging_obj() + logging_obj.record_bridged_model_response(chat_completion) + + logged = logging_obj._handle_anthropic_messages_response_logging(result=dict(served)) + + assert isinstance(logged, ModelResponse) + assert logged.id == served["id"], "the spend log row keeps the id the caller was served" + assert logged.usage.completion_tokens == 4040 # type: ignore[attr-defined] + assert logged.usage.completion_tokens_details.reasoning_tokens == 3480 # type: ignore[attr-defined] + + +def test_handle_anthropic_messages_response_logging_reparses_the_served_body_without_a_bridged_completion(): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_native", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "answer"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 7}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_native" + assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined] + + def test_handle_anthropic_messages_response_logging_passes_model_response_through(): """Anthropic-native path already yields a ModelResponse; it must be returned unchanged.""" logging_obj = _anthropic_messages_logging_obj() diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index bf6eb062507..ec8e2e0438a 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10919,3 +10919,63 @@ class TestStreamingContainerOwnershipRecordedBeforeDone: assert tuple(chunk for chunk, _ in observed) == self.CHUNKS assert tuple(count for _, count in observed) == (0, 0, 0, 0) recorder.assert_awaited_once() + + +class TestResponseCostFromLoggingObjForBridgedMessages: + """A /v1/messages request answered through the chat bridge arrives here as an Anthropic body whose + usage has no reasoning field, while the cost it recomputes must still price the reasoning tokens + the provider reported (LIT-9666).""" + + @staticmethod + def _logging_obj(model: str): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="lit-9666", + function_id="lit-9666", + ) + logging_obj.optional_params = {} + return logging_obj + + def test_recomputed_cost_prices_the_reasoning_tokens_of_the_bridged_chat_completion(self): + from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage + from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + model = "dashscope/qwen-turbo" + pricing = litellm.get_model_info(model) + input_rate = pricing["input_cost_per_token"] + output_rate = pricing["output_cost_per_token"] + reasoning_rate = pricing["output_cost_per_reasoning_token"] + assert input_rate is not None and output_rate is not None and reasoning_rate is not None + assert reasoning_rate != output_rate + chat_completion = ModelResponse( + id="chatcmpl-upstream", + model="qwen-turbo", + choices=[{"index": 0, "message": {"role": "assistant", "content": "answer"}, "finish_reason": "stop"}], + usage=Usage( + prompt_tokens=1240, + completion_tokens=4040, + total_tokens=5280, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3480), + ), + ) + served = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response=chat_completion) + assert served is not None and "reasoning" not in json.dumps(served["usage"]) + logging_obj = self._logging_obj(model) + logging_obj.record_bridged_model_response(chat_completion) + + cost = ProxyBaseLLMRequestProcessing._response_cost_from_logging_obj( + response=dict(served), logging_obj=logging_obj + ) + + expected_reasoning = 3480 * reasoning_rate + expected_output = expected_reasoning + (4040 - 3480) * output_rate + assert cost == pytest.approx(1240 * input_rate + expected_output) + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) + assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning)