fix(logging): price a None hidden response_cost instead of logging it as 0

A present-but-None `_hidden_params["response_cost"]` was copied verbatim into model_call_details by `_process_hidden_params_and_response_cost`, so callbacks saw StandardLoggingPayload.response_cost == 0 on sync `responses` calls while the SpendLogs row for the same request id was billed. The guard now falls through to `_response_cost_calculator` on None, matching the calculator's own "already calculated" check. Explicit 0 and numeric hidden costs are unchanged.

Adds the unit regression from #42015 plus an explicit-zero test, and two live e2e tests (config-registered CustomLogger and prometheus) asserting the callback-visible cost equals the SpendLogs row for a /v1/responses request.

Fixes #41976

Co-authored-by: onatozmenn <onatozmen44@gmail.com>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-22 05:34:06 +00:00
parent 19c13ac971
commit a217bcf2a9
8 changed files with 315 additions and 5 deletions

View file

@ -125,6 +125,7 @@ SERVER_ENV=(
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
"PYTHONPATH=${REPO_ROOT}"
"E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl"
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
"JWT_AUDIENCE=litellm-e2e"
@ -210,6 +211,7 @@ E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
E2E_KEYCLOAK_ADMIN_USER=admin
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl
DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
EOF

View file

@ -2317,7 +2317,7 @@ class Logging(LiteLLMLoggingBaseClass):
if self.model_call_details.get("cache_hit") is True:
self.model_call_details["response_cost"] = 0.0
elif "response_cost" in hidden_params:
elif hidden_params.get("response_cost") is not None:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:

View file

@ -7,6 +7,8 @@
- {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"}
- {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"}
- {id: logging.prometheus.success.records_queue_time, module: logging, tier: P1, event: success, assertions: [records_queue_time], exercised_on: [chat_completions], source: "integrations/prometheus.py / LIT-2034", fail_before_fix: proven, rationale: "Queue time feeds saturation alerting; the family stayed registered while no observation was ever recorded, so presence alone is not the contract"}
- {id: logging.prometheus.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/prometheus.py / GitHub issue #41976", rationale: "The callback-visible StandardLogging response_cost must equal the SpendLogs row for the same request id; a present-but-None hidden cost used to reach callbacks as 0 while the spend row was billed"}
- {id: logging.custom_callback.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/custom_logger.py / GitHub issue #41976", rationale: "kwargs[standard_logging_object][response_cost] handed to a config-registered CustomLogger must equal the SpendLogs row for the same request id; the proxy routes /v1/responses through aresponses, which stayed correct on unfixed main, so this pins the reporter surface rather than reproducing the sync responses defect"}
- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
- {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"}
- {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"}

View file

@ -32,7 +32,7 @@ litellm_settings:
- host: 127.0.0.1
port: 6379
ssl: true
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"]
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "tests.e2e.logging.callback_cost_capture.capture"]
require_auth_for_metrics_endpoint: false
router_settings:

View file

@ -0,0 +1,65 @@
"""Config-registered CustomLogger that records the cost each success callback is handed.
Registered on the e2e gateway as ``callbacks: ["tests.e2e.logging.callback_cost_capture.capture"]``.
Every success event appends one JSON line to ``E2E_CALLBACK_COST_CAPTURE_FILE`` with the request id
and ``standard_logging_object.response_cost`` exactly as a customer's Langfuse, Datadog, or custom
CustomLogger would see it. Tests read the file back and compare against the SpendLogs row.
"""
from __future__ import annotations
import json
import os
from datetime import datetime
from typing import Final
from pydantic import BaseModel, ConfigDict
from litellm.integrations.custom_logger import CustomLogger
CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE"
class CapturedCost(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
request_id: str
call_type: str | None
hook: str
response_cost: float | None
class _StandardLoggingCostFields(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
id: str
call_type: str | None = None
response_cost: float | None = None
def _append(hook: str, kwargs: dict[str, object]) -> None:
path: Final = os.environ.get(CAPTURE_FILE_ENV)
payload: Final = kwargs.get("standard_logging_object")
if path is None or not isinstance(payload, dict):
return
fields: Final = _StandardLoggingCostFields.model_validate(payload)
row: Final = CapturedCost(
request_id=fields.id, call_type=fields.call_type, hook=hook, response_cost=fields.response_cost
)
with open(path, "a", encoding="utf-8") as handle:
handle.write(json.dumps(row.model_dump()) + "\n")
class CallbackCostCapture(CustomLogger):
def log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
_append("log_success_event", kwargs)
async def async_log_success_event(
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
_append("async_log_success_event", kwargs)
capture: Final = CallbackCostCapture()

View file

@ -0,0 +1,84 @@
"""The response_cost a config-registered CustomLogger receives must be the cost the spend row bills.
GitHub issue #41976: a successful sync, non-streaming ``/v1/responses`` call reached callbacks with
``kwargs["standard_logging_object"]["response_cost"] == 0`` while ``LiteLLM_SpendLogs.spend`` for the
same request id was positive. The gateway registers ``callback_cost_capture.capture`` from this
directory as a plain ``callbacks:`` entry, so this is the exact payload a Langfuse, Datadog, or
customer CustomLogger is handed. The capture file path comes from ``E2E_CALLBACK_COST_CAPTURE_FILE``.
"""
from __future__ import annotations
import json
import math
import os
import time
from pathlib import Path
import pytest
from callback_cost_capture import CAPTURE_FILE_ENV, CapturedCost
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
from lifecycle import ResourceManager
from logging_client import LoggingClient, first_ok
from pydantic import BaseModel, ConfigDict
pytestmark = pytest.mark.e2e
class _ResponsesBody(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
def _capture_file() -> Path:
path = os.environ.get(CAPTURE_FILE_ENV)
assert path, f"{CAPTURE_FILE_ENV} is unset: the gateway must register callback_cost_capture.capture"
return Path(path)
def _captured_rows(path: Path, request_id: str) -> list[CapturedCost]:
if not path.exists():
return []
rows = [CapturedCost.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
return [row for row in rows if row.request_id == request_id]
def _poll_captured(client: LoggingClient, path: Path, request_id: str) -> list[CapturedCost]:
deadline = time.monotonic() + client.proxy.poll_timeout
while True:
rows = _captured_rows(path, request_id)
if rows or time.monotonic() >= deadline:
return rows
time.sleep(client.proxy.poll_interval)
class TestCustomCallbackResponsesCost:
@pytest.mark.covers("logging.custom_callback.success.logs_spend", exercised_on=["responses"])
def test_sync_responses_callback_payload_cost_equals_spend_log_row(
self, client: LoggingClient, resources: ResourceManager
) -> None:
capture_file = _capture_file()
alias = f"e2e-callback-cost-{unique_marker()}"
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
resources.defer(lambda: client.delete_key(key))
outcome = first_ok(
client,
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
)
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
)
captured = _poll_captured(client, capture_file, response_id)
assert captured, f"CustomLogger saw no success event for request_id={response_id} in {capture_file}"
for row in captured:
assert row.response_cost is not None and math.isclose(row.response_cost, spend_row.spend, rel_tol=1e-9), (
f"{row.hook} ({row.call_type}) saw standard_logging_object.response_cost={row.response_cost} "
f"but SpendLogs billed {spend_row.spend} for request_id={response_id}"
)

View file

@ -0,0 +1,83 @@
"""The cost a logging callback sees must be the cost the spend row bills.
GitHub issue #41976: a successful sync ``/v1/responses`` call reached callbacks with
``StandardLoggingPayload.response_cost == 0`` while ``LiteLLM_SpendLogs.spend`` for the
same request id was positive. Prometheus is the callback under test here because its
``litellm_spend_metric`` counter is fed straight from ``standard_logging_object.response_cost``
and is readable from the proxy itself, so no external sink is needed. A fresh key with one
request makes the per-alias counter equal that request's callback-visible cost.
"""
from __future__ import annotations
import json
import math
import time
import pytest
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
from lifecycle import ResourceManager
from logging_client import LoggingClient, first_ok
from prometheus_client.parser import text_string_to_metric_families
from pydantic import BaseModel, ConfigDict
pytestmark = pytest.mark.e2e
SPEND_METRIC = "litellm_spend_metric"
ALIAS_LABEL = "api_key_alias"
class _ResponsesBody(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
def _callback_spend(exposition: str, alias: str) -> float | None:
values = [
sample.value
for family in text_string_to_metric_families(exposition)
for sample in family.samples
if sample.name == f"{SPEND_METRIC}_total" and sample.labels.get(ALIAS_LABEL) == alias
]
return sum(values) if values else None
def _poll_callback_spend(client: LoggingClient, alias: str) -> float | None:
deadline = time.monotonic() + client.proxy.poll_timeout
while True:
spend = _callback_spend(client.scrape_metrics(), alias)
if spend is not None or time.monotonic() >= deadline:
return spend
time.sleep(client.proxy.poll_interval)
class TestResponsesCallbackCost:
@pytest.mark.covers("logging.prometheus.success.logs_spend", exercised_on=["responses"])
def test_responses_callback_cost_equals_spend_log_row(
self, client: LoggingClient, resources: ResourceManager
) -> None:
alias = f"e2e-responses-cost-{unique_marker()}"
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
resources.defer(lambda: client.delete_key(key))
outcome = first_ok(
client,
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
)
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
)
callback_spend = _poll_callback_spend(client, alias)
assert callback_spend is not None, (
f"{SPEND_METRIC} has no series for {ALIAS_LABEL}={alias} although the spend row billed {spend_row.spend}"
)
assert math.isclose(callback_spend, spend_row.spend, rel_tol=1e-9), (
f"callback saw response_cost={callback_spend} but SpendLogs billed {spend_row.spend} "
f"for request_id={response_id}"
)

View file

@ -472,7 +472,7 @@ class TestZeroCostDiagnostic:
self._assert_flagged(logging_obj, caplog)
assert payload["zero_cost_diagnostic"] == logging_obj.model_call_details["zero_cost_diagnostic"]
def test_uncomputed_hidden_cost_is_not_a_zero_cost(
def test_uncomputed_hidden_cost_is_priced_like_an_absent_one(
self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture
) -> None:
usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
@ -484,8 +484,13 @@ class TestZeroCostDiagnostic:
response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
)
assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None
assert self._zero_cost_warnings(caplog) == []
payload: Final = logging_obj.model_call_details["standard_logging_object"]
assert payload["response_cost"] == 0.0
if deployment_pricing is self.FREE_PRICING:
assert payload["zero_cost_diagnostic"] is None
assert self._zero_cost_warnings(caplog) == []
return
self._assert_flagged(logging_obj, caplog)
def test_unbilled_read_route_with_usage_stays_silent(
self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture
@ -3822,6 +3827,75 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer
assert slo.get("response_cost") == passthrough_cost
def test_process_hidden_params_falls_through_on_none_response_cost():
"""A present-but-None hidden response_cost is priced by the calculator, not logged as 0 (issue #41976)."""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponse, Usage
logging_obj = LiteLLMLoggingObj(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="responses",
start_time=datetime.now(),
litellm_call_id="test-none-hidden-cost",
function_id="test-none-hidden-cost",
)
logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"}
logging_obj.optional_params = {}
result = ModelResponse(
id="success",
choices=[{"message": {"role": "assistant", "content": "ok"}}],
usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110),
)
result._hidden_params = {"response_cost": None}
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
calculated_cost = logging_obj._response_cost_calculator(result=result)
assert calculated_cost is not None and calculated_cost > 0
assert logging_obj.model_call_details.get("response_cost") == calculated_cost
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
assert slo.get("response_cost") == calculated_cost
def test_process_hidden_params_explicit_zero_cost_overrides_earlier_cost():
"""An explicit hidden response_cost of 0 wins over a cost already on model_call_details; only None falls through."""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponse, Usage
logging_obj = LiteLLMLoggingObj(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="responses",
start_time=datetime.now(),
litellm_call_id="test-zero-hidden-cost-overrides",
function_id="test-zero-hidden-cost-overrides",
)
logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"}
logging_obj.optional_params = {}
logging_obj.model_call_details["response_cost"] = 0.5
result = ModelResponse(
id="success",
choices=[{"message": {"role": "assistant", "content": "ok"}}],
usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110),
)
result._hidden_params = {"response_cost": 0.0}
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
assert logging_obj.model_call_details.get("response_cost") == 0.0
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
assert slo.get("response_cost") == 0.0
def test_function_setup_litellm_metadata_populates_metadata():
"""
Test that function_setup() properly handles litellm_metadata (used by /v1/messages,