diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a9773c22d96..891915eb357 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, Callable, Mapping from datetime import datetime from functools import lru_cache from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload +from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, overload import anyio import httpx @@ -928,28 +928,57 @@ def _override_openai_response_model( ) +class CostBreakdownHeaderValues(NamedTuple): + original_cost: float | None = None + discount_amount: float | None = None + margin_total_amount: float | None = None + margin_percent: float | None = None + input_cost: float | None = None + output_cost: float | None = None + cache_read_cost: float | None = None + cache_creation_cost: float | None = None + reasoning_cost: float | None = None + tool_usage_cost: float | None = None + + +def _uncached_input_cost( + input_cost: float | None, + cache_read_cost: float | None, + cache_creation_cost: float | None, +) -> float | None: + """The stored input cost nests the cache costs inside it; headers advertise the additive split instead.""" + if input_cost is None: + return None + return input_cost - (cache_read_cost or 0.0) - (cache_creation_cost or 0.0) + + def _get_cost_breakdown_from_logging_obj( litellm_logging_obj: LiteLLMLoggingObj | None, -) -> tuple[float | None, float | None, float | None, float | None]: - """ - Extract discount and margin information from logging object's cost breakdown. - - Returns: - Tuple of (original_cost, discount_amount, margin_total_amount, margin_percent) - """ +) -> CostBreakdownHeaderValues: + """Extract discount, margin, and per-component cost information from logging object's cost breakdown.""" if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"): - return None, None, None, None + return CostBreakdownHeaderValues() cost_breakdown: Final = litellm_logging_obj.cost_breakdown if not cost_breakdown: - return None, None, None, None + return CostBreakdownHeaderValues() - original_cost: Final = cost_breakdown.get("original_cost") - discount_amount: Final = cost_breakdown.get("discount_amount") - margin_total_amount: Final = cost_breakdown.get("margin_total_amount") - margin_percent: Final = cost_breakdown.get("margin_percent") - - return original_cost, discount_amount, margin_total_amount, margin_percent + return CostBreakdownHeaderValues( + original_cost=cost_breakdown.get("original_cost"), + discount_amount=cost_breakdown.get("discount_amount"), + margin_total_amount=cost_breakdown.get("margin_total_amount"), + margin_percent=cost_breakdown.get("margin_percent"), + input_cost=_uncached_input_cost( + input_cost=cost_breakdown.get("input_cost"), + cache_read_cost=cost_breakdown.get("cache_read_cost"), + cache_creation_cost=cost_breakdown.get("cache_creation_cost"), + ), + output_cost=cost_breakdown.get("output_cost"), + cache_read_cost=cost_breakdown.get("cache_read_cost"), + cache_creation_cost=cost_breakdown.get("cache_creation_cost"), + reasoning_cost=cost_breakdown.get("reasoning_cost"), + tool_usage_cost=cost_breakdown.get("tool_usage_cost"), + ) def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None) -> float | None: @@ -1075,13 +1104,7 @@ class ProxyBaseLLMRequestProcessing: exclude_values: Final = {"", None, "None"} hidden_params = hidden_params or {} - # Extract discount and margin info from cost_breakdown if available - ( - original_cost, - discount_amount, - margin_total_amount, - margin_percent, - ) = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj) + cost_breakdown: Final = _get_cost_breakdown_from_logging_obj(litellm_logging_obj=litellm_logging_obj) # Calculate updated spend for header (include current response_cost) current_spend: Final = user_api_key_dict.spend or 0.0 @@ -1110,12 +1133,36 @@ class ProxyBaseLLMRequestProcessing: "x-litellm-version": version, "x-litellm-model-region": model_region, "x-litellm-response-cost": str(response_cost), - "x-litellm-response-cost-original": (str(original_cost) if original_cost is not None else None), - "x-litellm-response-cost-discount-amount": (str(discount_amount) if discount_amount is not None else None), - "x-litellm-response-cost-margin-amount": ( - str(margin_total_amount) if margin_total_amount is not None else None + "x-litellm-response-cost-original": ( + str(cost_breakdown.original_cost) if cost_breakdown.original_cost is not None else None + ), + "x-litellm-response-cost-discount-amount": ( + str(cost_breakdown.discount_amount) if cost_breakdown.discount_amount is not None else None + ), + "x-litellm-response-cost-margin-amount": ( + str(cost_breakdown.margin_total_amount) if cost_breakdown.margin_total_amount is not None else None + ), + "x-litellm-response-cost-margin-percent": ( + str(cost_breakdown.margin_percent) if cost_breakdown.margin_percent is not None else None + ), + "x-litellm-response-cost-input": ( + str(cost_breakdown.input_cost) if cost_breakdown.input_cost is not None else None + ), + "x-litellm-response-cost-output": ( + str(cost_breakdown.output_cost) if cost_breakdown.output_cost is not None else None + ), + "x-litellm-response-cost-cache-read": ( + str(cost_breakdown.cache_read_cost) if cost_breakdown.cache_read_cost is not None else None + ), + "x-litellm-response-cost-cache-creation": ( + str(cost_breakdown.cache_creation_cost) if cost_breakdown.cache_creation_cost is not None else None + ), + "x-litellm-response-cost-reasoning": ( + str(cost_breakdown.reasoning_cost) if cost_breakdown.reasoning_cost is not None else None + ), + "x-litellm-response-cost-tool-usage": ( + str(cost_breakdown.tool_usage_cost) if cost_breakdown.tool_usage_cost is not None else None ), - "x-litellm-response-cost-margin-percent": (str(margin_percent) if margin_percent is not None else None), "x-litellm-classifier-cost": (str(classifier_cost) if classifier_cost is not None else None), "x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit), "x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit), diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index a3c0f0089fe..9ddd74a46a8 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -834,6 +834,180 @@ class TestProxyBaseLLMRequestProcessing: assert "x-litellm-response-cost-margin-amount" not in headers assert "x-litellm-response-cost-margin-percent" not in headers + def test_get_custom_headers_per_component_cost_breakdown(self): + """Test per-component cost headers against the stored production breakdown. + + cost_calculator stores full prompt cost (cache pricing included) as input_cost + and full completion cost (reasoning included) as output_cost. The input header + subtracts the cache components so the emitted contract is additive: + input + cache_read + cache_creation + output + tool_usage == total, with + reasoning remaining a subset of output. + """ + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + + logging_obj = LiteLLMLoggingObj( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="test-call-id-components", + function_id="test-function", + ) + + input_cost: Final = 0.00002 + output_cost: Final = 0.00004 + cache_read_cost: Final = 0.000005 + cache_creation_cost: Final = 0.00001 + reasoning_cost: Final = 0.000015 + tool_usage_cost: Final = 0.00003 + total_cost: Final = input_cost + output_cost + tool_usage_cost + uncached_input_cost: Final = input_cost - cache_read_cost - cache_creation_cost + + logging_obj.set_cost_breakdown( + input_cost=input_cost, + output_cost=output_cost, + total_cost=total_cost, + cost_for_built_in_tools_cost_usd_dollar=tool_usage_cost, + cache_read_cost=cache_read_cost, + cache_creation_cost=cache_creation_cost, + reasoning_cost=reasoning_cost, + ) + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + call_id="test-call-id-components", + response_cost=total_cost, + litellm_logging_obj=logging_obj, + ) + + assert "x-litellm-response-cost" in headers + assert float(headers["x-litellm-response-cost"]) == pytest.approx(total_cost) + + assert "x-litellm-response-cost-input" in headers + assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(uncached_input_cost) + + assert "x-litellm-response-cost-output" in headers + assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(output_cost) + + assert "x-litellm-response-cost-cache-read" in headers + assert float(headers["x-litellm-response-cost-cache-read"]) == pytest.approx(cache_read_cost) + + assert "x-litellm-response-cost-cache-creation" in headers + assert float(headers["x-litellm-response-cost-cache-creation"]) == pytest.approx(cache_creation_cost) + + assert "x-litellm-response-cost-reasoning" in headers + assert float(headers["x-litellm-response-cost-reasoning"]) == pytest.approx(reasoning_cost) + + assert "x-litellm-response-cost-tool-usage" in headers + assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(tool_usage_cost) + + component_sum: Final = ( + float(headers["x-litellm-response-cost-input"]) + + float(headers["x-litellm-response-cost-cache-read"]) + + float(headers["x-litellm-response-cost-cache-creation"]) + + float(headers["x-litellm-response-cost-output"]) + + float(headers["x-litellm-response-cost-tool-usage"]) + ) + assert component_sum == pytest.approx(float(headers["x-litellm-response-cost"])) + assert float(headers["x-litellm-response-cost-reasoning"]) <= float(headers["x-litellm-response-cost-output"]) + + def test_get_custom_headers_without_cost_breakdown_omits_component_headers(self): + """Test that when litellm_logging_obj has no cost_breakdown, component headers are omitted.""" + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + + logging_obj = LiteLLMLoggingObj( + model="gpt-4", + messages=[], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="test-call-id-no-breakdown", + function_id="test-function", + ) + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + response_cost=0.0001, + litellm_logging_obj=logging_obj, + ) + + assert "x-litellm-response-cost" in headers + assert "x-litellm-response-cost-input" not in headers + assert "x-litellm-response-cost-output" not in headers + assert "x-litellm-response-cost-cache-read" not in headers + assert "x-litellm-response-cost-cache-creation" not in headers + assert "x-litellm-response-cost-reasoning" not in headers + assert "x-litellm-response-cost-tool-usage" not in headers + + def test_get_custom_headers_per_component_with_discount_and_margin(self): + """Test that component headers co-exist accurately with discount and margin headers.""" + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + + mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) + mock_user_api_key_dict.tpm_limit = None + mock_user_api_key_dict.rpm_limit = None + mock_user_api_key_dict.max_budget = None + mock_user_api_key_dict.spend = 0 + + logging_obj = LiteLLMLoggingObj( + model="gpt-4", + messages=[], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="test-call-id-combined", + function_id="test-function", + ) + + logging_obj.set_cost_breakdown( + input_cost=0.00006, + output_cost=0.00004, + total_cost=0.000105, + cost_for_built_in_tools_cost_usd_dollar=0.0, + original_cost=0.0001, + discount_percent=0.05, + discount_amount=0.000005, + margin_percent=0.10, + margin_total_amount=0.00001, + ) + + headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=mock_user_api_key_dict, + response_cost=0.000105, + litellm_logging_obj=logging_obj, + ) + + assert float(headers["x-litellm-response-cost"]) == pytest.approx(0.000105) + assert float(headers["x-litellm-response-cost-original"]) == pytest.approx(0.0001) + assert float(headers["x-litellm-response-cost-discount-amount"]) == pytest.approx(0.000005) + assert float(headers["x-litellm-response-cost-margin-amount"]) == pytest.approx(0.00001) + assert float(headers["x-litellm-response-cost-margin-percent"]) == pytest.approx(0.10) + assert float(headers["x-litellm-response-cost-input"]) == pytest.approx(0.00006) + assert float(headers["x-litellm-response-cost-output"]) == pytest.approx(0.00004) + assert "x-litellm-response-cost-cache-read" not in headers + assert "x-litellm-response-cost-cache-creation" not in headers + assert "x-litellm-response-cost-reasoning" not in headers + assert float(headers["x-litellm-response-cost-tool-usage"]) == pytest.approx(0.0) + @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) def test_get_custom_headers_classifier_cost_from_routing_decision(self, metadata_key): """The auto-router's LLM classifier cost must surface as its own header. @@ -919,16 +1093,14 @@ class TestProxyBaseLLMRequestProcessing: discount_amount=0.000005, ) - ( - original_cost, - discount_amount, - margin_total_amount, - margin_percent, - ) = _get_cost_breakdown_from_logging_obj(logging_obj) - assert original_cost == 0.0001 - assert discount_amount == 0.000005 - assert margin_total_amount is None - assert margin_percent is None + breakdown = _get_cost_breakdown_from_logging_obj(logging_obj) + assert breakdown.original_cost == 0.0001 + assert breakdown.discount_amount == 0.000005 + assert breakdown.margin_total_amount is None + assert breakdown.margin_percent is None + assert breakdown.input_cost == 0.00005 + assert breakdown.output_cost == 0.00005 + assert breakdown.tool_usage_cost == 0.0 # Test with margin info logging_obj_with_margin = LiteLLMLoggingObj( @@ -950,16 +1122,11 @@ class TestProxyBaseLLMRequestProcessing: margin_total_amount=0.00001, ) - ( - original_cost, - discount_amount, - margin_total_amount, - margin_percent, - ) = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin) - assert original_cost == 0.0001 - assert discount_amount is None - assert margin_total_amount == 0.00001 - assert margin_percent == 0.10 + breakdown_with_margin = _get_cost_breakdown_from_logging_obj(logging_obj_with_margin) + assert breakdown_with_margin.original_cost == 0.0001 + assert breakdown_with_margin.discount_amount is None + assert breakdown_with_margin.margin_total_amount == 0.00001 + assert breakdown_with_margin.margin_percent == 0.10 # Test with no discount or margin info logging_obj_no_discount = LiteLLMLoggingObj( @@ -978,28 +1145,42 @@ class TestProxyBaseLLMRequestProcessing: cost_for_built_in_tools_cost_usd_dollar=0.0, ) - ( - original_cost, - discount_amount, - margin_total_amount, - margin_percent, - ) = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount) - assert original_cost is None - assert discount_amount is None - assert margin_total_amount is None - assert margin_percent is None + breakdown_no_discount = _get_cost_breakdown_from_logging_obj(logging_obj_no_discount) + assert breakdown_no_discount.original_cost is None + assert breakdown_no_discount.discount_amount is None + assert breakdown_no_discount.margin_total_amount is None + assert breakdown_no_discount.margin_percent is None + assert breakdown_no_discount.input_cost == 0.00005 + assert breakdown_no_discount.output_cost == 0.00005 + + # Test that cache components stored nested inside input_cost are subtracted out + logging_obj_with_cache = LiteLLMLoggingObj( + model="claude-haiku-4-5", + messages=[{"role": "user", "content": "test"}], + stream=False, + call_type="completion", + start_time=None, + litellm_call_id="test-call-id-cache", + function_id="test-function-id-cache", + ) + logging_obj_with_cache.set_cost_breakdown( + input_cost=0.00008, + output_cost=0.00002, + total_cost=0.0001, + cost_for_built_in_tools_cost_usd_dollar=0.0, + cache_read_cost=0.00003, + cache_creation_cost=0.00004, + ) + + breakdown_with_cache = _get_cost_breakdown_from_logging_obj(logging_obj_with_cache) + assert breakdown_with_cache.input_cost == pytest.approx(0.00001) + assert breakdown_with_cache.cache_read_cost == 0.00003 + assert breakdown_with_cache.cache_creation_cost == 0.00004 + assert breakdown_with_cache.output_cost == 0.00002 # Test with None logging object - ( - original_cost, - discount_amount, - margin_total_amount, - margin_percent, - ) = _get_cost_breakdown_from_logging_obj(None) - assert original_cost is None - assert discount_amount is None - assert margin_total_amount is None - assert margin_percent is None + breakdown_none = _get_cost_breakdown_from_logging_obj(None) + assert all(value is None for value in breakdown_none) def test_get_custom_headers_key_spend_includes_response_cost(self): """