fix(proxy): price streamed usage frames under the deployment's model so usage.cost matches the spend row

This commit is contained in:
mateo-berri 2026-10-30 09:28:55 -07:00
parent 1ca8c8c94a
commit 744b3be87f
3 changed files with 72 additions and 3 deletions

View file

@ -4869,8 +4869,9 @@ class ProxyBaseLLMRequestProcessing:
if stream_usage is None:
return None
service_tier: Final = obj.get("service_tier")
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
ModelResponse(usage=stream_usage),
ModelResponse(model=served_model, usage=stream_usage),
model_name,
service_tier if isinstance(service_tier, str) else None,
litellm_logging_obj,

View file

@ -41,12 +41,13 @@ def _make_streaming_response(chunks):
return mock
def _unarmed_logging_obj():
def _unarmed_logging_obj(model: str = "unknown"):
"""Real Logging objects only carry _on_deferred_stream_complete when the
proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
and would spuriously trigger the deferral branch."""
obj = MagicMock()
obj._on_deferred_stream_complete = None
obj.model = model
return obj
@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=response,
request_body={"model": "gpt-4o-mini", "stream": True},
litellm_logging_obj=_unarmed_logging_obj(),
litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"),
endpoint_type=endpoint_type,
start_time=datetime(2026, 1, 1),
passthrough_success_handler_obj=MagicMock(),

View file

@ -68,6 +68,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro
from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
@ -10148,6 +10149,8 @@ class TestInjectCostIntoUsageDict:
discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``."""
class _StubLoggingObj:
model = "claude-haiku-4-5"
def __init__(self, cost):
self._cost = cost
self.captured_result = None
@ -10184,6 +10187,8 @@ class TestInjectCostIntoUsageDict:
def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
return None
@ -10209,6 +10214,8 @@ class TestInjectCostIntoUsageDict:
model-name pricing rather than propagating into the response body."""
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
raise ValueError("no pricing for this deployment")
@ -10294,6 +10301,8 @@ class TestInjectCostIntoUsageDict:
streaming /v1/chat/completions gets its negotiated price instead of sticker."""
class _StubLoggingObj:
model = "gpt-4o-mini"
def __init__(self, cost):
self._cost = cost
self.captured_result = None
@ -10322,6 +10331,8 @@ class TestInjectCostIntoUsageDict:
def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
class _StubLoggingObj:
model = "gpt-4o-mini"
def response_cost_calculator(self, result):
return None
@ -10337,6 +10348,60 @@ class TestInjectCostIntoUsageDict:
assert result is not None
assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4))
@pytest.mark.parametrize(
("deployment_model", "served_model", "custom_llm_provider"),
[
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
],
)
def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment(
self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str
) -> None:
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
router: Final = Router(
model_list=[
{
"model_name": "priced-claude",
"litellm_params": {
"model": deployment_model,
"api_key": "k",
"vertex_project": "p",
"vertex_location": "global",
},
"model_info": token_rates,
}
]
)
deployment_id: Final = router.model_list[0]["model_info"]["id"]
logging_obj: Final = LiteLLMLoggingObj(
model=deployment_model,
messages=[{"role": "user", "content": "test"}],
stream=True,
call_type="anthropic_messages",
start_time=None,
litellm_call_id="token-priced-claude-search",
function_id="token-priced-claude-search",
)
logging_obj.update_environment_variables(
litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={}
)
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}}
event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage}
result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj)
stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None)
logged_cost: Final = logging_obj.response_cost_calculator(
result=litellm.ModelResponse(model=served_model, usage=stream_usage)
)
token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"]
assert logged_cost is not None and logged_cost > token_cost
assert result is not None
assert result["usage"]["cost"] == pytest.approx(logged_cost)
class TestProcessChunkWithCostInjection:
@pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"])
@ -10403,6 +10468,8 @@ class TestProcessChunkWithCostInjection:
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
class _StubLoggingObj:
model = "claude-haiku-4-5"
def response_cost_calculator(self, result):
return 0.00042