From e0e7c793438c25eaee7f1f1925698674144d80f6 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Tue, 6 Oct 2026 01:48:36 -0400 Subject: [PATCH 1/3] fix(anthropic): keep the image-priced cost on bridged /v1/messages responses When /v1/messages is bridged to a non-Anthropic model, the proxy builds x-litellm-response-cost from logging_obj.model_call_details["response_cost"]. That value is stored by the success handlers, which run on another task. When they have not run yet, the proxy recomputes the cost from the Anthropic-shaped response, whose usage no longer carries the image token breakdown, so image output tokens were priced at the text rate. Store the cost the ModelResponse already carries on the logging object before translating it, in both the async and sync handlers. Fixes #44743 --- .../pass_through/adapters/handler.py | 30 ++++++- ...erimental_pass_through_messages_handler.py | 90 +++++++++++++++++++ 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 5bda3437a40..4ca03244291 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -24,6 +24,7 @@ from litellm.llms.anthropic.pass_through.utils import ( litellm_logging_obj_from_kwargs, local_model_name, ) +from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -32,6 +33,7 @@ from litellm.types.utils import ModelResponse from litellm.utils import get_model_info if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router @@ -60,6 +62,20 @@ def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool: return False +def _record_bridged_response_cost( + completion_response: ModelResponse, logging_obj: "LiteLLMLoggingObject | None" +) -> None: + """ + The Anthropic-shaped response can no longer be priced correctly (its usage drops the image token breakdown), + and the success handlers that store this cost run on another thread, so the proxy's x-litellm-response-cost + header could read the logging object before they did + """ + response_cost: Final = get_hidden_params_dict(completion_response).get("response_cost") + if logging_obj is None or not isinstance(response_cost, (int, float)): + return + logging_obj.model_call_details["response_cost"] = response_cost + + def _proxy_router_fallback() -> "Router | None": try: from litellm.proxy.proxy_server import llm_router as _proxy_router @@ -665,6 +681,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) completion_response: Final = await litellm.acompletion(**completion_kwargs) + bridged_logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -673,14 +690,16 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=True, - litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs), + litellm_logging_obj=bridged_logging_obj, ) if transformed_stream is not None: return transformed_stream raise ValueError("Failed to transform streaming response") else: + bridged_response: Final = cast(ModelResponse, completion_response) + _record_bridged_response_cost(bridged_response, bridged_logging_obj) anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params( - cast(ModelResponse, completion_response), + bridged_response, tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, ) @@ -800,6 +819,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) completion_response: Final = litellm.completion(**completion_kwargs) + bridged_logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( @@ -808,14 +828,16 @@ class LiteLLMMessagesToCompletionTransformationHandler: tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, is_async=False, - litellm_logging_obj=litellm_logging_obj_from_kwargs(kwargs), + litellm_logging_obj=bridged_logging_obj, ) if transformed_stream is not None: return transformed_stream raise ValueError("Failed to transform streaming response") else: + bridged_response: Final = cast(ModelResponse, completion_response) + _record_bridged_response_cost(bridged_response, bridged_logging_obj) anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params( - cast(ModelResponse, completion_response), + bridged_response, tool_name_mapping=tool_name_mapping, polyfill_result=polyfill_result, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index c3d4dba7376..f18d372e5b7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -2,10 +2,13 @@ import asyncio import json import os import uuid +from collections.abc import Iterator +from datetime import datetime from typing import Any, Dict, Final, List import httpx import pytest +import respx from fastapi.testclient import TestClient from pydantic import ValidationError @@ -15,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm from litellm.anthropic_interface import messages from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.utils import ( @@ -1685,3 +1689,89 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet assert "anthropic_beta" not in captured["body"] assert captured["anthropic-beta"].split(",").count("dangerous-tool-use-2026-09-03") == 1 assert response["safeguard_results"] == safeguard_results + + +_IMAGE_MODEL = "gemini-image-cost-probe" +_IMAGE_INPUT_RATE = 1e-06 +_IMAGE_TEXT_OUTPUT_RATE = 2e-06 +_IMAGE_OUTPUT_RATE = 5e-05 +_IMAGE_PROMPT_TOKENS = 7 +_IMAGE_OUTPUT_TOKENS = 1120 + + +class _LoggingBeforeSuccessHandlersRan(Logging): + """The proxy's logging object as the response headers see it when the success handlers have not run yet""" + + def success_handler(self, *args, **kwargs): + return None + + async def async_success_handler(self, *args, **kwargs): + return None + + +@pytest.fixture +def gemini_image_response(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setitem( + litellm.model_cost, + f"gemini/{_IMAGE_MODEL}", + { + "input_cost_per_token": _IMAGE_INPUT_RATE, + "output_cost_per_token": _IMAGE_TEXT_OUTPUT_RATE, + "output_cost_per_image_token": _IMAGE_OUTPUT_RATE, + "litellm_provider": "gemini", + "mode": "chat", + }, + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.get_model_info.cache_clear() + body = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "a red square"}]}, "finishReason": "STOP"}], + "usageMetadata": { + "promptTokenCount": _IMAGE_PROMPT_TOKENS, + "candidatesTokenCount": _IMAGE_OUTPUT_TOKENS, + "candidatesTokensDetails": [{"modality": "IMAGE", "tokenCount": _IMAGE_OUTPUT_TOKENS}], + "totalTokenCount": _IMAGE_PROMPT_TOKENS + _IMAGE_OUTPUT_TOKENS, + }, + "modelVersion": _IMAGE_MODEL, + } + with respx.mock(assert_all_called=True) as router: + router.route(host="generativelanguage.googleapis.com").mock(return_value=httpx.Response(200, json=body)) + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [True, False], ids=["async", "sync"]) +async def test_bridged_messages_response_leaves_its_image_priced_cost_on_the_logging_object( + gemini_image_response, is_async +): + from litellm.llms.anthropic.pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + logging_obj: Final = _LoggingBeforeSuccessHandlersRan( + model=_IMAGE_MODEL, + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1), + litellm_call_id="call-1", + function_id="fn-1", + ) + request: Final = { + "max_tokens": 2048, + "messages": [{"role": "user", "content": "a red square on a white background"}], + "model": f"gemini/{_IMAGE_MODEL}", + "api_key": "test-key", + "litellm_logging_obj": logging_obj, + } + + if is_async: + response = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(**request) + else: + response = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(**request) + + assert response["usage"]["output_tokens"] == _IMAGE_OUTPUT_TOKENS, response + assert logging_obj.model_call_details.get("response_cost") == pytest.approx( + _IMAGE_PROMPT_TOKENS * _IMAGE_INPUT_RATE + _IMAGE_OUTPUT_TOKENS * _IMAGE_OUTPUT_RATE + ), "x-litellm-response-cost reads this before the success handlers run, so it must already hold the image rate" From c261db2f02fe54787f805322ee0d2f1ce751a680 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 11 Oct 2026 02:09:40 -0400 Subject: [PATCH 2/3] style(anthropic): mark the response_cost write to model_call_details as intended The type discipline gate counts writing into a parameter as LIT011. Writing the cost onto the logging object is the point of this helper, so it gets the same rebind-ok marker other providers use. --- litellm/llms/anthropic/pass_through/adapters/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 1b1dd9ae145..6a7c3812d97 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -74,7 +74,7 @@ def _record_bridged_response_cost( response_cost: Final = get_hidden_params_dict(completion_response).get("response_cost") if logging_obj is None or not isinstance(response_cost, (int, float)): return - logging_obj.model_call_details["response_cost"] = response_cost + logging_obj.model_call_details["response_cost"] = response_cost # rebind-ok: the record spend logging reads def _proxy_router_fallback() -> "Router | None": From 6994fbfab56ac5ec85f4d1004c0e0b7c73a8c10a Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Fri, 16 Oct 2026 14:32:26 -0400 Subject: [PATCH 3/3] fix(anthropic): drop the type-checking Logging import now imported at runtime Upstream's bridged-response change imports Logging at module level, so the TYPE_CHECKING copy trips TC004 in the strict ruff gate --- litellm/llms/anthropic/pass_through/adapters/handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index ae2ac23a804..f9816ed4d6b 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -47,7 +47,6 @@ from litellm.types.utils import ModelResponse from litellm.utils import get_model_info if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.router import Router from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -83,9 +82,7 @@ def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool: return False -def _record_bridged_response_cost( - completion_response: ModelResponse, logging_obj: "LiteLLMLoggingObject | None" -) -> None: +def _record_bridged_response_cost(completion_response: ModelResponse, logging_obj: LiteLLMLoggingObject | None) -> None: """ The Anthropic-shaped response can no longer be priced correctly (its usage drops the image token breakdown), and the success handlers that store this cost run on another thread, so the proxy's x-litellm-response-cost