scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

fix(proxy): keep the response-cost headers on calls priced at zero

Browse source 
Pricing responses reads and vector-store management routes at zero dropped the whole
x-litellm-response-cost family off those replies. The header build reads a falsy zero as
a cost this response never recorded and filters it out, and a call that returns before
pricing stores no cost breakdown for the component headers to read, so a client parsing
the cost off a read got a KeyError where it had previously been handed a number.

Those calls now advertise the family at zero. Retrieving a background response, and the
cost poller's read of one, still report their real cost.

The params-taking form of the predicate moves from opentelemetry into
internal_call_metadata so the proxy header build and the OTEL recorders share one copy.
This commit is contained in:
Yucheng Zhu 2026-08-26 16:05:56 -07:00
parent c656aa3253
commit 271fdbd22e
5 changed files with 202 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
litellm/integrations/opentelemetry.py
View file

@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
import litellm
from litellm._logging import verbose_logger
from litellm.constants import NON_INFERENCE_CALL_TYPES
from litellm.integrations._types.open_inference import (
OpenInferenceSpanKindValues,
SpanAttributes,
@ -23,7 +22,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
)
from litellm.integrations.otel.model.db_endpoint import db_span_attributes
from litellm.integrations.otel.model.semconv import Metric
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.service_tier_utils import (
@ -240,25 +239,6 @@ def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope:
return repr(value)
def _is_unbilled_non_inference(
call_type: str | None, litellm_params: Mapping[str, object] | None, response_obj: object
) -> bool:
"""Whether this call is a read or management route whose token counts describe an
earlier request rather than this one.
The call-type membership test runs first so that inference traffic, which is every
request in a normal workload, never pays for the metadata merge behind it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
metadata: Final = (
StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None
)
return is_unbilled_non_inference_call(call_type, metadata, response_obj)
def _shutdown_tracer_provider(provider: "_SDKTracerProvider") -> None:
"""Flush and stop a dropped provider so its exporter thread is reclaimed."""
try:
@ -1667,7 +1647,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if (
self._token_usage_histogram
and response_obj
and not _is_unbilled_non_inference(kwargs.get("call_type"), params, response_obj)
and not is_unbilled_non_inference_call_from_params(kwargs.get("call_type"), params, response_obj)
and (usage := response_obj.get("usage"))
):
in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
@ -1745,7 +1725,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if not self._time_per_output_token_histogram:
return
if _is_unbilled_non_inference(kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj):
if is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
):
return
# Get completion tokens from response_obj
@ -2500,7 +2482,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
usage: Final = (
response_obj.get("usage")
if response_obj
and not _is_unbilled_non_inference(kwargs.get("call_type"), litellm_params, response_obj)
and not is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), litellm_params, response_obj
)
else None
)
if usage:

4
litellm/integrations/otel/plumbing/metrics.py
View file

@ -21,7 +21,6 @@ from litellm.integrations.opentelemetry import (
METRIC_METADATA_KEYS,
TOKEN_TYPE_ATTRIBUTE,
_build_metric_attribute_filter,
_is_unbilled_non_inference,
_resolve_metric_attribute_filter,
)
from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
@ -33,6 +32,7 @@ from litellm.integrations.otel.model.semconv import (
resolve_provider,
)
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -199,7 +199,7 @@ class GenAIMetricRecorder:
) -> None:
common_attrs: Final = self._filter_attributes(self._bounded_attributes(kwargs))
duration_s: Final = (end_time - start_time).total_seconds()
usage_is_replayed: Final = _is_unbilled_non_inference(
usage_is_replayed: Final = is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
)

20
litellm/litellm_core_utils/internal_call_metadata.py
View file

@ -79,6 +79,26 @@ def is_unbilled_non_inference_call(
return metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
def is_unbilled_non_inference_call_from_params(
call_type: str | None,
litellm_params: Mapping[str, object] | None,
response: object,
) -> bool:
""":func:`is_unbilled_non_inference_call` for callers holding raw ``litellm_params``.
The call-type membership test runs first so that inference traffic, which is every
request in a normal workload, never pays for the metadata merge behind it.
"""
if call_type not in NON_INFERENCE_CALL_TYPES:
return False
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
metadata: Final = (
StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None
)
return is_unbilled_non_inference_call(call_type, metadata, response)
def sanitize_user_api_key_auth(auth: object) -> object:
"""Copy of the auth object with its budget reservation removed; the cost callback
falls back to reading the reservation from inside the auth object."""

31
litellm/proxy/common_request_processing.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
LITELLM_DETAILED_TIMING,
LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
NON_INFERENCE_CALL_TYPES,
RETURN_RAW_MODEL_NAME_METADATA_KEY,
ROUTER_MODEL_NAME_RESPONSE_FIELD,
STREAM_SSE_DATA_PREFIX,
@ -38,6 +39,7 @@ from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
@ -1293,15 +1295,35 @@ def _uncached_input_cost(
return input_cost - (cache_read_cost or 0.0) - (cache_creation_cost or 0.0)
_ZERO_COST_BREAKDOWN: Final = CostBreakdownHeaderValues(
original_cost=0.0,
discount_amount=0.0,
margin_total_amount=0.0,
margin_percent=0.0,
input_cost=0.0,
output_cost=0.0,
tool_usage_cost=0.0,
)
"""The component split a call priced at zero advertises, so a client reading the cost headers off a
read or management route still finds the whole family rather than a partially populated one."""
def _get_cost_breakdown_from_logging_obj(
litellm_logging_obj: LiteLLMLoggingObj | None,
) -> CostBreakdownHeaderValues:
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown."""
"""Extract discount, margin, and per-component cost information from logging object's cost breakdown.
A non-inference call that priced at zero never records a breakdown, so its components are
reported as zero here. Any such call that did price normally (retrieving a background response,
and the cost poller's read of one) has a stored breakdown and takes the branch below instead.
"""
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
return CostBreakdownHeaderValues()
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
if not cost_breakdown:
if litellm_logging_obj.call_type in NON_INFERENCE_CALL_TYPES:
return _ZERO_COST_BREAKDOWN
return CostBreakdownHeaderValues()
return CostBreakdownHeaderValues(
@ -2577,11 +2599,16 @@ class ProxyBaseLLMRequestProcessing:
additional_headers = hidden_params.get("additional_headers", {}) or {}
recover_response_cost: Final = not response_cost and hidden_params.get("response_cost") is None
llm_cost_for_headers: Final = (
computed_cost_for_headers: Final = (
self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj) or ""
if recover_response_cost
else response_cost
)
llm_cost_for_headers: Final = (
0.0
if is_unbilled_non_inference_call_from_params(logging_obj.call_type, logging_obj.litellm_params, response)
else computed_cost_for_headers
)
_, request_metadata_bucket = get_or_create_metadata_bucket(self.data)
guardrail_cost_for_headers: Final = guardrail_information_cost(
request_metadata_bucket.get("standard_logging_guardrail_information")

143
tests/test_litellm/proxy/test_common_request_processing.py
View file

@ -30,6 +30,7 @@ from litellm.proxy.common_request_processing import (
_ClientDisconnectedBeforeFirstChunk,
_extract_error_from_sse_chunk,
_get_cost_breakdown_from_logging_obj,
CostBreakdownHeaderValues,
_has_attribute_error_in_chain,
_is_azure_model_router_request,
open_sse_before_first_byte,
@ -4974,6 +4975,148 @@ class TestResponseCostHeaderForTypedDictResponses:
assert fastapi_response.headers["x-litellm-response-cost"] == "0.00123"
class TestCostHeadersForCallsPricedAtZero:
"""
Regression for LIT-5602. Pricing responses reads and vector-store management routes at
zero dropped the entire x-litellm-response-cost family off those replies: the header
build reads a falsy zero as "this response never recorded a cost" and filters it out,
and a call that returns before pricing stores no cost breakdown for the component
headers to read. A client parsing the cost off a read got a KeyError where it had
previously been handed a number. Those calls now advertise the whole family at zero.
"""
@staticmethod
def _responses_read(*, background=False):
from litellm.types.llms.openai import ResponsesAPIResponse
return ResponsesAPIResponse(
id="resp_lit5602",
created_at=0,
model="gpt-4.1-mini",
object="response",
output=[],
status="completed",
background=background,
usage={"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
)
@staticmethod
def _logging_obj(*, call_type, recovered_cost=0.0):
logging_obj = MagicMock()
logging_obj.litellm_call_id = "call-lit5602"
logging_obj.call_type = call_type
logging_obj.litellm_params = {}
logging_obj.cost_breakdown = None
logging_obj.model_call_details = {"response_cost": recovered_cost}
logging_obj._response_cost_calculator = MagicMock(return_value=recovered_cost)
logging_obj._enqueue_deferred_logging = None
logging_obj._on_deferred_stream_complete = None
return logging_obj
async def _drive(self, *, monkeypatch, response, logging_obj, route_type):
import litellm.proxy.common_request_processing as crp
from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth
async def fake_route_request(**kwargs):
async def _llm_call():
return response
return _llm_call()
monkeypatch.setattr(crp, "route_request", fake_route_request)
async def fake_post_call_success_hook(data, user_api_key_dict, response):
return response
proxy_logging_obj = MagicMock(spec=ProxyLogging)
proxy_logging_obj.during_call_hook = AsyncMock(return_value=None)
proxy_logging_obj.update_request_status = AsyncMock(return_value=None)
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
proxy_logging_obj.post_call_success_hook = fake_post_call_success_hook
fastapi_response = Response()
processing_obj = ProxyBaseLLMRequestProcessing(data={"litellm_logging_obj": logging_obj})
with patch.object(
ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails", return_value=False
):
await processing_obj.base_process_llm_request(
request=MagicMock(spec=Request, headers={}),
fastapi_response=fastapi_response,
user_api_key_dict=RealUserAPIKeyAuth(api_key="sk-test"),
route_type=route_type,
proxy_logging_obj=proxy_logging_obj,
general_settings={},
proxy_config=MagicMock(spec=ProxyConfig),
select_data_generator=None,
llm_router=None,
skip_pre_call_logic=True,
)
return fastapi_response
@pytest.mark.asyncio
async def test_responses_read_emits_the_cost_header_family_at_zero(self, monkeypatch):
fastapi_response = await self._drive(
monkeypatch=monkeypatch,
response=self._responses_read(),
logging_obj=self._logging_obj(call_type="aget_responses"),
route_type="aget_responses",
)
assert fastapi_response.headers["x-litellm-response-cost"] == "0.0"
for component in (
"original",
"discount-amount",
"margin-amount",
"margin-percent",
"input",
"output",
"tool-usage",
):
assert fastapi_response.headers[f"x-litellm-response-cost-{component}"] == "0.0"
@pytest.mark.asyncio
async def test_reading_a_background_response_keeps_its_real_cost(self, monkeypatch):
fastapi_response = await self._drive(
monkeypatch=monkeypatch,
response=self._responses_read(background=True),
logging_obj=self._logging_obj(call_type="aget_responses", recovered_cost=0.00042),
route_type="aget_responses",
)
assert float(fastapi_response.headers["x-litellm-response-cost"]) == pytest.approx(0.00042)
@pytest.mark.asyncio
async def test_an_inference_call_without_a_recorded_cost_still_omits_the_header(self, monkeypatch):
"""A chat completion has no zero-priced route, so a falsy cost there means the cost was
never recorded and the header stays absent rather than advertising a made-up zero."""
fastapi_response = await self._drive(
monkeypatch=monkeypatch,
response=SimpleNamespace(_hidden_params={}),
logging_obj=self._logging_obj(call_type="acompletion"),
route_type="acompletion",
)
assert "x-litellm-response-cost" not in fastapi_response.headers
def test_cost_breakdown_reports_zero_components_for_a_call_priced_at_zero(self):
breakdown = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=self._logging_obj(call_type="aget_responses")
)
assert breakdown.original_cost == 0.0
assert breakdown.input_cost == 0.0
assert breakdown.output_cost == 0.0
assert breakdown.tool_usage_cost == 0.0
def test_cost_breakdown_stays_empty_for_an_inference_call(self):
breakdown = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=self._logging_obj(call_type="acompletion")
)
assert breakdown == CostBreakdownHeaderValues()
class TestPreCallWithFallbacksOnLocalRateLimit:
@pytest.mark.asyncio