mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): price streamed usage frames under the deployment's model so usage.cost matches the spend row
This commit is contained in:
parent
1ca8c8c94a
commit
744b3be87f
3 changed files with 72 additions and 3 deletions
|
|
@ -4869,8 +4869,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
if stream_usage is None:
|
||||
return None
|
||||
service_tier: Final = obj.get("service_tier")
|
||||
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
|
||||
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
|
||||
ModelResponse(usage=stream_usage),
|
||||
ModelResponse(model=served_model, usage=stream_usage),
|
||||
model_name,
|
||||
service_tier if isinstance(service_tier, str) else None,
|
||||
litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -41,12 +41,13 @@ def _make_streaming_response(chunks):
|
|||
return mock
|
||||
|
||||
|
||||
def _unarmed_logging_obj():
|
||||
def _unarmed_logging_obj(model: str = "unknown"):
|
||||
"""Real Logging objects only carry _on_deferred_stream_complete when the
|
||||
proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
|
||||
and would spuriously trigger the deferral branch."""
|
||||
obj = MagicMock()
|
||||
obj._on_deferred_stream_complete = None
|
||||
obj.model = model
|
||||
return obj
|
||||
|
||||
|
||||
|
|
@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
|
|||
async for chunk in PassThroughStreamingHandler.chunk_processor(
|
||||
response=response,
|
||||
request_body={"model": "gpt-4o-mini", "stream": True},
|
||||
litellm_logging_obj=_unarmed_logging_obj(),
|
||||
litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"),
|
||||
endpoint_type=endpoint_type,
|
||||
start_time=datetime(2026, 1, 1),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro
|
|||
from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
|
||||
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
|
||||
|
|
@ -10148,6 +10149,8 @@ class TestInjectCostIntoUsageDict:
|
|||
discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def __init__(self, cost):
|
||||
self._cost = cost
|
||||
self.captured_result = None
|
||||
|
|
@ -10184,6 +10187,8 @@ class TestInjectCostIntoUsageDict:
|
|||
|
||||
def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return None
|
||||
|
||||
|
|
@ -10209,6 +10214,8 @@ class TestInjectCostIntoUsageDict:
|
|||
model-name pricing rather than propagating into the response body."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
raise ValueError("no pricing for this deployment")
|
||||
|
||||
|
|
@ -10294,6 +10301,8 @@ class TestInjectCostIntoUsageDict:
|
|||
streaming /v1/chat/completions gets its negotiated price instead of sticker."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
def __init__(self, cost):
|
||||
self._cost = cost
|
||||
self.captured_result = None
|
||||
|
|
@ -10322,6 +10331,8 @@ class TestInjectCostIntoUsageDict:
|
|||
|
||||
def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
|
||||
class _StubLoggingObj:
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return None
|
||||
|
||||
|
|
@ -10337,6 +10348,60 @@ class TestInjectCostIntoUsageDict:
|
|||
assert result is not None
|
||||
assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "served_model", "custom_llm_provider"),
|
||||
[
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
|
||||
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
|
||||
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
|
||||
],
|
||||
)
|
||||
def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment(
|
||||
self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str
|
||||
) -> None:
|
||||
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-claude",
|
||||
"litellm_params": {
|
||||
"model": deployment_model,
|
||||
"api_key": "k",
|
||||
"vertex_project": "p",
|
||||
"vertex_location": "global",
|
||||
},
|
||||
"model_info": token_rates,
|
||||
}
|
||||
]
|
||||
)
|
||||
deployment_id: Final = router.model_list[0]["model_info"]["id"]
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model=deployment_model,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=None,
|
||||
litellm_call_id="token-priced-claude-search",
|
||||
function_id="token-priced-claude-search",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={}
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
|
||||
wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}}
|
||||
event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage}
|
||||
|
||||
result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj)
|
||||
|
||||
stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None)
|
||||
logged_cost: Final = logging_obj.response_cost_calculator(
|
||||
result=litellm.ModelResponse(model=served_model, usage=stream_usage)
|
||||
)
|
||||
token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"]
|
||||
assert logged_cost is not None and logged_cost > token_cost
|
||||
assert result is not None
|
||||
assert result["usage"]["cost"] == pytest.approx(logged_cost)
|
||||
|
||||
|
||||
class TestProcessChunkWithCostInjection:
|
||||
@pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"])
|
||||
|
|
@ -10403,6 +10468,8 @@ class TestProcessChunkWithCostInjection:
|
|||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return 0.00042
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue