mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
Merge pull request #36965 from erensh27/feat/per-component-cost-headers
feat(proxy): per-component response cost headers
This commit is contained in:
Mateo Wang
GitHub
commit
cba2beaf42
2 changed files with 296 additions and 68 deletions
|
|
@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, Callable, Mapping
|
|||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, overload
|
||||
|
||||
import anyio
|
||||
import httpx
|
||||
|
|
@ -928,28 +928,57 @@ def _override_openai_response_model(
|
|||
)
|
||||
|
||||
|
||||
class CostBreakdownHeaderValues(NamedTuple):
|
||||
original_cost: float | None = None
|
||||
discount_amount: float | None = None
|
||||
margin_total_amount: float | None = None
|
||||
margin_percent: float | None = None
|
||||
input_cost: float | None = None
|
||||
output_cost: float | None = None
|
||||
cache_read_cost: float | None = None
|
||||
cache_creation_cost: float | None = None
|
||||
reasoning_cost: float | None = None
|
||||
tool_usage_cost: float | None = None
|
||||
|
||||
|
||||
def _uncached_input_cost(
|
||||
input_cost: float | None,
|
||||
cache_read_cost: float | None,
|
||||
cache_creation_cost: float | None,
|
||||
) -> float | None:
|
||||
"""The stored input cost nests the cache costs inside it; headers advertise the additive split instead."""
|
||||
if input_cost is None:
|
||||
return None
|
||||
return input_cost - (cache_read_cost or 0.0) - (cache_creation_cost or 0.0)
|
||||
|
||||
|
||||
def _get_cost_breakdown_from_logging_obj(
|
||||
litellm_logging_obj: LiteLLMLoggingObj | None,
|
||||
) -> tuple[float | None, float | None, float | None, float | None]:
|
||||
"""
|
||||
Extract discount and margin information from logging object's cost breakdown.
|
||||
|
||||
Returns:
|
||||
Tuple of (original_cost, discount_amount, margin_total_amount, margin_percent)
|
||||
"""
|
||||
) -> CostBreakdownHeaderValues:
|
||||
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown."""
|
||||
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
|
||||
return None, None, None, None
|
||||
return CostBreakdownHeaderValues()
|
||||
|
||||
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
|
||||
if not cost_breakdown:
|
||||
return None, None, None, None
|
||||
return CostBreakdownHeaderValues()
|
||||
|
||||
original_cost: Final = cost_breakdown.get("original_cost")
|
||||
discount_amount: Final = cost_breakdown.get("discount_amount")
|
||||
margin_total_amount: Final = cost_breakdown.get("margin_total_amount")
|
||||
margin_percent: Final = cost_breakdown.get("margin_percent")
|
||||
|
||||
return original_cost, discount_amount, margin_total_amount, margin_percent
|
||||
return CostBreakdownHeaderValues(
|
||||
original_cost=cost_breakdown.get("original_cost"),
|
||||
discount_amount=cost_breakdown.get("discount_amount"),
|
||||
margin_total_amount=cost_breakdown.get("margin_total_amount"),
|
||||
margin_percent=cost_breakdown.get("margin_percent"),
|
||||
input_cost=_uncached_input_cost(
|
||||
input_cost=cost_breakdown.get("input_cost"),
|
||||
cache_read_cost=cost_breakdown.get("cache_read_cost"),
|
||||
cache_creation_cost=cost_breakdown.get("cache_creation_cost"),
|
||||
),
|
||||
output_cost=cost_breakdown.get("output_cost"),
|
||||
cache_read_cost=cost_breakdown.get("cache_read_cost"),
|
||||
cache_creation_cost=cost_breakdown.get("cache_creation_cost"),
|
||||
reasoning_cost=cost_breakdown.get("reasoning_cost"),
|
||||
tool_usage_cost=cost_breakdown.get("tool_usage_cost"),
|
||||
)
|
||||
|
||||
|
||||
def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None) -> float | None:
|
||||
|
|
@ -1075,13 +1104,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
exclude_values: Final = {"", None, "None"}
|
||||
hidden_params = hidden_params or {}
|
||||
|
||||
# Extract discount and margin info from cost_breakdown if available
|
||||
(
|
||||
original_cost,
|
||||
discount_amount,
|
||||
margin_total_amount,
|
||||
margin_percent,
|
||||
) = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj)
|
||||
cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj)
|
||||
|
||||
# Calculate updated spend for header (include current response_cost)
|
||||
current_spend: Final = user_api_key_dict.spend or 0.0
|
||||
|
|
@ -1110,12 +1133,36 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"x-litellm-version": version,
|
||||
"x-litellm-model-region": model_region,
|
||||
"x-litellm-response-cost": str(response_cost),
|
||||
"x-litellm-response-cost-original": (str(original_cost) if original_cost is not None else None),
|
||||
"x-litellm-response-cost-discount-amount": (str(discount_amount) if discount_amount is not None else None),
|
||||
"x-litellm-response-cost-margin-amount": (
|
||||
str(margin_total_amount) if margin_total_amount is not None else None
|
||||
"x-litellm-response-cost-original": (
|
||||
str(cost_breakdown.original_cost) if cost_breakdown.original_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-discount-amount": (
|
||||
str(cost_breakdown.discount_amount) if cost_breakdown.discount_amount is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-margin-amount": (
|
||||
str(cost_breakdown.margin_total_amount) if cost_breakdown.margin_total_amount is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-margin-percent": (
|
||||
str(cost_breakdown.margin_percent) if cost_breakdown.margin_percent is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-input": (
|
||||
str(cost_breakdown.input_cost) if cost_breakdown.input_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-output": (
|
||||
str(cost_breakdown.output_cost) if cost_breakdown.output_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-cache-read": (
|
||||
str(cost_breakdown.cache_read_cost) if cost_breakdown.cache_read_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-cache-creation": (
|
||||
str(cost_breakdown.cache_creation_cost) if cost_breakdown.cache_creation_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-reasoning": (
|
||||
str(cost_breakdown.reasoning_cost) if cost_breakdown.reasoning_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-tool-usage": (
|
||||
str(cost_breakdown.tool_usage_cost) if cost_breakdown.tool_usage_cost is not None else None
|
||||
),
|
||||
"x-litellm-response-cost-margin-percent": (str(margin_percent) if margin_percent is not None else None),
|
||||
"x-litellm-classifier-cost": (str(classifier_cost) if classifier_cost is not None else None),
|
||||
"x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit),
|
||||
"x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit),
|
||||
|
|
|
|||
|
|
@ -834,6 +834,180 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
assert "x-litellm-response-cost-margin-amount" not in headers
|
||||
assert "x-litellm-response-cost-margin-percent" not in headers
|
||||
|
||||
def test_get_custom_headers_per_component_cost_breakdown(self):
|
||||
"""Test per-component cost headers against the stored production breakdown.
|
||||
|
||||
cost_calculator stores full prompt cost (cache pricing included) as input_cost
|
||||
and full completion cost (reasoning included) as output_cost. The input header
|
||||
subtracts the cache components so the emitted contract is additive:
|
||||
input + cache_read + cache_creation + output + tool_usage == total, with
|
||||
reasoning remaining a subset of output.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObj,
|
||||
)
|
||||
|
||||
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
|
||||
mock_user_api_key_dict.tpm_limit = None
|
||||
mock_user_api_key_dict.rpm_limit = None
|
||||
mock_user_api_key_dict.max_budget = None
|
||||
mock_user_api_key_dict.spend = 0
|
||||
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="test-call-id-components",
|
||||
function_id="test-function",
|
||||
)
|
||||
|
||||
input_cost: Final = 0.00002
|
||||
output_cost: Final = 0.00004
|
||||
cache_read_cost: Final = 0.000005
|
||||
cache_creation_cost: Final = 0.00001
|
||||
reasoning_cost: Final = 0.000015
|
||||
tool_usage_cost: Final = 0.00003
|
||||
total_cost: Final = input_cost + output_cost + tool_usage_cost
|
||||
uncached_input_cost: Final = input_cost - cache_read_cost - cache_creation_cost
|
||||
|
||||
logging_obj.set_cost_breakdown(
|
||||
input_cost=input_cost,
|
||||
output_cost=output_cost,
|
||||
total_cost=total_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=tool_usage_cost,
|
||||
cache_read_cost=cache_read_cost,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
reasoning_cost=reasoning_cost,
|
||||
)
|
||||
|
||||
headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
call_id="test-call-id-components",
|
||||
response_cost=total_cost,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert "x-litellm-response-cost" in headers
|
||||
assert float(headers["x-litellm-response-cost"]) == pytest.approx(total_cost)
|
||||
|
||||
assert "x-litellm-response-cost-input" in headers
|
||||
assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(uncached_input_cost)
|
||||
|
||||
assert "x-litellm-response-cost-output" in headers
|
||||
assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(output_cost)
|
||||
|
||||
assert "x-litellm-response-cost-cache-read" in headers
|
||||
assert float(headers["x-litellm-response-cost-cache-read"]) == pytest.approx(cache_read_cost)
|
||||
|
||||
assert "x-litellm-response-cost-cache-creation" in headers
|
||||
assert float(headers["x-litellm-response-cost-cache-creation"]) == pytest.approx(cache_creation_cost)
|
||||
|
||||
assert "x-litellm-response-cost-reasoning" in headers
|
||||
assert float(headers["x-litellm-response-cost-reasoning"]) == pytest.approx(reasoning_cost)
|
||||
|
||||
assert "x-litellm-response-cost-tool-usage" in headers
|
||||
assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(tool_usage_cost)
|
||||
|
||||
component_sum: Final = (
|
||||
float(headers["x-litellm-response-cost-input"])
|
||||
+ float(headers["x-litellm-response-cost-cache-read"])
|
||||
+ float(headers["x-litellm-response-cost-cache-creation"])
|
||||
+ float(headers["x-litellm-response-cost-output"])
|
||||
+ float(headers["x-litellm-response-cost-tool-usage"])
|
||||
)
|
||||
assert component_sum == pytest.approx(float(headers["x-litellm-response-cost"]))
|
||||
assert float(headers["x-litellm-response-cost-reasoning"]) <= float(headers["x-litellm-response-cost-output"])
|
||||
|
||||
def test_get_custom_headers_without_cost_breakdown_omits_component_headers(self):
|
||||
"""Test that when litellm_logging_obj has no cost_breakdown, component headers are omitted."""
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObj,
|
||||
)
|
||||
|
||||
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
|
||||
mock_user_api_key_dict.tpm_limit = None
|
||||
mock_user_api_key_dict.rpm_limit = None
|
||||
mock_user_api_key_dict.max_budget = None
|
||||
mock_user_api_key_dict.spend = 0
|
||||
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="gpt-4",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="test-call-id-no-breakdown",
|
||||
function_id="test-function",
|
||||
)
|
||||
|
||||
headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
response_cost=0.0001,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert "x-litellm-response-cost" in headers
|
||||
assert "x-litellm-response-cost-input" not in headers
|
||||
assert "x-litellm-response-cost-output" not in headers
|
||||
assert "x-litellm-response-cost-cache-read" not in headers
|
||||
assert "x-litellm-response-cost-cache-creation" not in headers
|
||||
assert "x-litellm-response-cost-reasoning" not in headers
|
||||
assert "x-litellm-response-cost-tool-usage" not in headers
|
||||
|
||||
def test_get_custom_headers_per_component_with_discount_and_margin(self):
|
||||
"""Test that component headers co-exist accurately with discount and margin headers."""
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as LiteLLMLoggingObj,
|
||||
)
|
||||
|
||||
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
|
||||
mock_user_api_key_dict.tpm_limit = None
|
||||
mock_user_api_key_dict.rpm_limit = None
|
||||
mock_user_api_key_dict.max_budget = None
|
||||
mock_user_api_key_dict.spend = 0
|
||||
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="gpt-4",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="test-call-id-combined",
|
||||
function_id="test-function",
|
||||
)
|
||||
|
||||
logging_obj.set_cost_breakdown(
|
||||
input_cost=0.00006,
|
||||
output_cost=0.00004,
|
||||
total_cost=0.000105,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
original_cost=0.0001,
|
||||
discount_percent=0.05,
|
||||
discount_amount=0.000005,
|
||||
margin_percent=0.10,
|
||||
margin_total_amount=0.00001,
|
||||
)
|
||||
|
||||
headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
response_cost=0.000105,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert float(headers["x-litellm-response-cost"]) == pytest.approx(0.000105)
|
||||
assert float(headers["x-litellm-response-cost-original"]) == pytest.approx(0.0001)
|
||||
assert float(headers["x-litellm-response-cost-discount-amount"]) == pytest.approx(0.000005)
|
||||
assert float(headers["x-litellm-response-cost-margin-amount"]) == pytest.approx(0.00001)
|
||||
assert float(headers["x-litellm-response-cost-margin-percent"]) == pytest.approx(0.10)
|
||||
assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(0.00006)
|
||||
assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(0.00004)
|
||||
assert "x-litellm-response-cost-cache-read" not in headers
|
||||
assert "x-litellm-response-cost-cache-creation" not in headers
|
||||
assert "x-litellm-response-cost-reasoning" not in headers
|
||||
assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(0.0)
|
||||
|
||||
@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
|
||||
def test_get_custom_headers_classifier_cost_from_routing_decision(self, metadata_key):
|
||||
"""The auto-router's LLM classifier cost must surface as its own header.
|
||||
|
|
@ -919,16 +1093,14 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
discount_amount=0.000005,
|
||||
)
|
||||
|
||||
(
|
||||
original_cost,
|
||||
discount_amount,
|
||||
margin_total_amount,
|
||||
margin_percent,
|
||||
) = _get_cost_breakdown_from_logging_obj(logging_obj)
|
||||
assert original_cost == 0.0001
|
||||
assert discount_amount == 0.000005
|
||||
assert margin_total_amount is None
|
||||
assert margin_percent is None
|
||||
breakdown = _get_cost_breakdown_from_logging_obj(logging_obj)
|
||||
assert breakdown.original_cost == 0.0001
|
||||
assert breakdown.discount_amount == 0.000005
|
||||
assert breakdown.margin_total_amount is None
|
||||
assert breakdown.margin_percent is None
|
||||
assert breakdown.input_cost == 0.00005
|
||||
assert breakdown.output_cost == 0.00005
|
||||
assert breakdown.tool_usage_cost == 0.0
|
||||
|
||||
# Test with margin info
|
||||
logging_obj_with_margin = LiteLLMLoggingObj(
|
||||
|
|
@ -950,16 +1122,11 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
margin_total_amount=0.00001,
|
||||
)
|
||||
|
||||
(
|
||||
original_cost,
|
||||
discount_amount,
|
||||
margin_total_amount,
|
||||
margin_percent,
|
||||
) = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin)
|
||||
assert original_cost == 0.0001
|
||||
assert discount_amount is None
|
||||
assert margin_total_amount == 0.00001
|
||||
assert margin_percent == 0.10
|
||||
breakdown_with_margin = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin)
|
||||
assert breakdown_with_margin.original_cost == 0.0001
|
||||
assert breakdown_with_margin.discount_amount is None
|
||||
assert breakdown_with_margin.margin_total_amount == 0.00001
|
||||
assert breakdown_with_margin.margin_percent == 0.10
|
||||
|
||||
# Test with no discount or margin info
|
||||
logging_obj_no_discount = LiteLLMLoggingObj(
|
||||
|
|
@ -978,28 +1145,42 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
)
|
||||
|
||||
(
|
||||
original_cost,
|
||||
discount_amount,
|
||||
margin_total_amount,
|
||||
margin_percent,
|
||||
) = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount)
|
||||
assert original_cost is None
|
||||
assert discount_amount is None
|
||||
assert margin_total_amount is None
|
||||
assert margin_percent is None
|
||||
breakdown_no_discount = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount)
|
||||
assert breakdown_no_discount.original_cost is None
|
||||
assert breakdown_no_discount.discount_amount is None
|
||||
assert breakdown_no_discount.margin_total_amount is None
|
||||
assert breakdown_no_discount.margin_percent is None
|
||||
assert breakdown_no_discount.input_cost == 0.00005
|
||||
assert breakdown_no_discount.output_cost == 0.00005
|
||||
|
||||
# Test that cache components stored nested inside input_cost are subtracted out
|
||||
logging_obj_with_cache = LiteLLMLoggingObj(
|
||||
model="claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="test-call-id-cache",
|
||||
function_id="test-function-id-cache",
|
||||
)
|
||||
logging_obj_with_cache.set_cost_breakdown(
|
||||
input_cost=0.00008,
|
||||
output_cost=0.00002,
|
||||
total_cost=0.0001,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
cache_read_cost=0.00003,
|
||||
cache_creation_cost=0.00004,
|
||||
)
|
||||
|
||||
breakdown_with_cache = _get_cost_breakdown_from_logging_obj(logging_obj_with_cache)
|
||||
assert breakdown_with_cache.input_cost == pytest.approx(0.00001)
|
||||
assert breakdown_with_cache.cache_read_cost == 0.00003
|
||||
assert breakdown_with_cache.cache_creation_cost == 0.00004
|
||||
assert breakdown_with_cache.output_cost == 0.00002
|
||||
|
||||
# Test with None logging object
|
||||
(
|
||||
original_cost,
|
||||
discount_amount,
|
||||
margin_total_amount,
|
||||
margin_percent,
|
||||
) = _get_cost_breakdown_from_logging_obj(None)
|
||||
assert original_cost is None
|
||||
assert discount_amount is None
|
||||
assert margin_total_amount is None
|
||||
assert margin_percent is None
|
||||
breakdown_none = _get_cost_breakdown_from_logging_obj(None)
|
||||
assert all(value is None for value in breakdown_none)
|
||||
|
||||
def test_get_custom_headers_key_spend_includes_response_cost(self):
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue