mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 7d022d62d6 into 3b8ad37296
This commit is contained in:
commit
61fe38f8c4
2 changed files with 194 additions and 1 deletions
|
|
@ -44,7 +44,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
|
|||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIWebSearchOptions
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -91,6 +91,32 @@ def _proxy_router_fallback() -> "Router | None":
|
|||
return _proxy_router
|
||||
|
||||
|
||||
def _store_response_cost_before_usage_flattening(completion_response: object, kwargs: Mapping[str, object]) -> None:
|
||||
"""Price the response while ``completion_response`` still carries its full usage breakdown.
|
||||
|
||||
``translate_completion_output_params`` flattens the OpenAI-shaped usage into the
|
||||
Anthropic response TypedDict, dropping ``completion_tokens_details.image_tokens``.
|
||||
The proxy recomputes the ``x-litellm-response-cost`` header from the translated
|
||||
response whenever the async success handler has not stored a cost yet, and that
|
||||
recompute prices image output tokens at the text rate. Recording the usage and its
|
||||
cost here, from the untouched ModelResponse, makes the header and the spend row
|
||||
price the same tokens.
|
||||
"""
|
||||
litellm_logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
|
||||
if litellm_logging_obj is None:
|
||||
return
|
||||
model_response: Final = cast(ModelResponse, completion_response) # cast-ok: adapter returns ModelResponse
|
||||
usage: Final = getattr(model_response, "usage", None)
|
||||
if not isinstance(usage, Usage):
|
||||
return
|
||||
try:
|
||||
response_cost: Final = litellm_logging_obj._response_cost_calculator(result=model_response) # pyright: ignore[reportPrivateUsage] # as the HTTP streaming iterator does
|
||||
except Exception: # noqa: BLE001 # pricing lookup failures vary by provider; never fail the response over them
|
||||
verbose_logger.exception("Anthropic Adapter - failed to pre-compute response cost")
|
||||
return
|
||||
litellm_logging_obj.record_partial_usage_for_failure(usage, response_cost or 0.0)
|
||||
|
||||
|
||||
def _router_or_proxy_fallback(requested: "Router | None") -> "Router | None":
|
||||
return requested if requested is not None else _proxy_router_fallback()
|
||||
|
||||
|
|
@ -813,6 +839,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
return transformed_stream
|
||||
raise ValueError("Failed to transform streaming response")
|
||||
else:
|
||||
_store_response_cost_before_usage_flattening(
|
||||
completion_response,
|
||||
cast(Mapping[str, object], kwargs), # cast-ok: framework kwargs are a string-keyed request mapping
|
||||
)
|
||||
_record_bridged_model_response(logging_obj, completion_response)
|
||||
anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
||||
cast(ModelResponse, completion_response),
|
||||
|
|
@ -958,6 +988,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
return transformed_stream
|
||||
raise ValueError("Failed to transform streaming response")
|
||||
else:
|
||||
_store_response_cost_before_usage_flattening(
|
||||
completion_response,
|
||||
cast(Mapping[str, object], kwargs), # cast-ok: framework kwargs are a string-keyed request mapping
|
||||
)
|
||||
_record_bridged_model_response(logging_obj, completion_response)
|
||||
anthropic_response: Final = ANTHROPIC_ADAPTER.translate_completion_output_params(
|
||||
cast(ModelResponse, completion_response),
|
||||
|
|
|
|||
|
|
@ -250,6 +250,165 @@ class TestPromptCacheOptionsForwarded:
|
|||
assert completion_kwargs["prompt_cache_options"] == {"mode": "explicit"}
|
||||
|
||||
|
||||
# --- response cost captured before usage flattening (#44743) -------------------
|
||||
#
|
||||
# ``translate_completion_output_params`` flattens the OpenAI-shaped usage into the
|
||||
# Anthropic response TypedDict, dropping ``completion_tokens_details.image_tokens``.
|
||||
# The proxy rebuilds the ``x-litellm-response-cost`` header from that translated
|
||||
# response whenever the async success handler has not stored a cost yet, and the
|
||||
# rebuild cannot tell image tokens from text tokens, so they get priced at the text
|
||||
# rate. The adapter therefore records usage plus cost from the untouched
|
||||
# ModelResponse first, which is what makes the header and the spend row agree.
|
||||
|
||||
|
||||
class _StubCostCalculator:
|
||||
"""Stands in for the pricing lookup, handed to a real ``Logging`` instance.
|
||||
|
||||
Binding it to the instance means no class attribute is replaced, so a
|
||||
concurrent test using another ``Logging`` object is unaffected.
|
||||
"""
|
||||
|
||||
def __init__(self, cost=None, raises=False):
|
||||
self.cost = cost
|
||||
self.raises = raises
|
||||
self.seen: list[object] = []
|
||||
|
||||
def __call__(self, result, **kwargs):
|
||||
self.seen.append(result)
|
||||
if self.raises:
|
||||
raise RuntimeError("no pricing configured")
|
||||
return self.cost
|
||||
|
||||
|
||||
def _real_logging_obj(calculator):
|
||||
"""A real ``Logging`` object, which ``litellm_logging_obj_from_kwargs`` requires.
|
||||
|
||||
The cost calculator is bound on the instance, so nothing at class level is
|
||||
patched and the object still satisfies the ``isinstance`` guard.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
obj = Logging(
|
||||
model="gemini-3.1-flash-image",
|
||||
messages=[{"role": "user", "content": "a red square"}],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=None,
|
||||
litellm_call_id="cost-probe",
|
||||
function_id="cost-probe",
|
||||
)
|
||||
obj._response_cost_calculator = calculator
|
||||
return obj
|
||||
|
||||
|
||||
def _image_usage_response():
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage
|
||||
|
||||
return ModelResponse(
|
||||
id="resp_1",
|
||||
model="gemini-3.1-flash-image",
|
||||
choices=[],
|
||||
usage=Usage(
|
||||
prompt_tokens=7,
|
||||
completion_tokens=1120,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=1120),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_response_cost_is_recorded_while_image_tokens_are_still_visible():
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
_store_response_cost_before_usage_flattening,
|
||||
)
|
||||
|
||||
calculator = _StubCostCalculator(cost=0.0672035)
|
||||
logging_obj = _real_logging_obj(calculator)
|
||||
|
||||
_store_response_cost_before_usage_flattening(_image_usage_response(), {"litellm_logging_obj": logging_obj})
|
||||
|
||||
stored_usage = logging_obj.model_call_details.get("combined_usage_object")
|
||||
assert stored_usage is not None, "adapter skipped response-cost bookkeeping on the non-streaming path"
|
||||
# The whole point of the fix: the usage handed to the cost calculator, and the
|
||||
# one recorded for the spend row, still carry image_tokens.
|
||||
assert stored_usage.completion_tokens_details.image_tokens == 1120
|
||||
assert calculator.seen[0].usage.completion_tokens_details.image_tokens == 1120
|
||||
assert logging_obj.model_call_details["response_cost"] == 0.0672035
|
||||
|
||||
|
||||
def test_response_cost_bookkeeping_is_a_noop_without_logging_obj():
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
_store_response_cost_before_usage_flattening,
|
||||
)
|
||||
|
||||
response: Final = _image_usage_response()
|
||||
before: Final = response.model_dump()
|
||||
_store_response_cost_before_usage_flattening(response, {})
|
||||
assert response.model_dump() == before
|
||||
|
||||
|
||||
def test_response_cost_bookkeeping_swallows_calculator_failure():
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
_store_response_cost_before_usage_flattening,
|
||||
)
|
||||
|
||||
logging_obj = _real_logging_obj(_StubCostCalculator(raises=True))
|
||||
|
||||
_store_response_cost_before_usage_flattening(_image_usage_response(), {"litellm_logging_obj": logging_obj})
|
||||
|
||||
assert "combined_usage_object" not in logging_obj.model_call_details
|
||||
|
||||
|
||||
def test_translated_response_cannot_be_repriced_for_image_tokens():
|
||||
"""Pins why the cost has to be captured before translation: the Anthropic
|
||||
response the proxy recomputes from has no image-token breakdown left, so any
|
||||
recompute from it necessarily charges the text rate."""
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import ANTHROPIC_ADAPTER
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
translated = ANTHROPIC_ADAPTER.translate_completion_output_params(_image_usage_response())
|
||||
assert translated is not None
|
||||
usage = translated["usage"]
|
||||
assert usage["output_tokens"] == 1120
|
||||
assert "completion_tokens_details" not in usage
|
||||
assert isinstance(_image_usage_response(), ModelResponse)
|
||||
|
||||
|
||||
def test_anthropic_messages_handler_records_cost_before_flattening():
|
||||
"""Drives the real non-streaming handler end to end.
|
||||
|
||||
The helper-level tests above pass even when the handler stops calling the
|
||||
helper, so this one asserts on what the handler leaves on the logging object.
|
||||
That is the regression the fix is for: if the handler stops recording before
|
||||
flattening, the proxy's recompute prices image tokens at the text rate.
|
||||
"""
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
|
||||
calculator = _StubCostCalculator(cost=0.0672035)
|
||||
logging_obj = _real_logging_obj(calculator)
|
||||
|
||||
with patch("litellm.completion", return_value=_image_usage_response()):
|
||||
result = LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "a red square"}],
|
||||
model="gemini-3.1-flash-image",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
stored_usage = logging_obj.model_call_details.get("combined_usage_object")
|
||||
assert stored_usage is not None, (
|
||||
"the handler returned an Anthropic response without recording image-token usage first, "
|
||||
"so the proxy's response-cost recompute cannot price image tokens correctly"
|
||||
)
|
||||
assert stored_usage.completion_tokens_details.image_tokens == 1120
|
||||
assert logging_obj.model_call_details["response_cost"] == 0.0672035
|
||||
# The returned Anthropic response is what the proxy recomputes from, and it is
|
||||
# exactly the shape that no longer carries the image-token breakdown.
|
||||
assert "completion_tokens_details" not in result["usage"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("converted_stream_key", sorted(CONVERTED_STREAM_KEYS))
|
||||
async def test_messages_layer_converted_stream_flag_still_gets_a_message_back(converted_stream_key: str):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue