mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(logging): price a None hidden response_cost instead of logging it as 0
A present-but-None `_hidden_params["response_cost"]` was copied verbatim into model_call_details by `_process_hidden_params_and_response_cost`, so callbacks saw StandardLoggingPayload.response_cost == 0 on sync `responses` calls while the SpendLogs row for the same request id was billed. The guard now falls through to `_response_cost_calculator` on None, matching the calculator's own "already calculated" check. Explicit 0 and numeric hidden costs are unchanged. Adds the unit regression from #42015 plus an explicit-zero test, and two live e2e tests (config-registered CustomLogger and prometheus) asserting the callback-visible cost equals the SpendLogs row for a /v1/responses request. Fixes #41976 Co-authored-by: onatozmenn <onatozmen44@gmail.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
19c13ac971
commit
a217bcf2a9
8 changed files with 315 additions and 5 deletions
2
.github/e2e-stack/up.sh
vendored
2
.github/e2e-stack/up.sh
vendored
|
|
@ -125,6 +125,7 @@ SERVER_ENV=(
|
|||
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
|
||||
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
|
||||
"PYTHONPATH=${REPO_ROOT}"
|
||||
"E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl"
|
||||
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
|
||||
"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
|
||||
"JWT_AUDIENCE=litellm-e2e"
|
||||
|
|
@ -210,6 +211,7 @@ E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
|
|||
E2E_KEYCLOAK_ADMIN_USER=admin
|
||||
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
|
||||
SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
|
||||
E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl
|
||||
DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
|
||||
EOF
|
||||
|
||||
|
|
|
|||
|
|
@ -2317,7 +2317,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
if self.model_call_details.get("cache_hit") is True:
|
||||
self.model_call_details["response_cost"] = 0.0
|
||||
elif "response_cost" in hidden_params:
|
||||
elif hidden_params.get("response_cost") is not None:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
- {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"}
|
||||
- {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"}
|
||||
- {id: logging.prometheus.success.records_queue_time, module: logging, tier: P1, event: success, assertions: [records_queue_time], exercised_on: [chat_completions], source: "integrations/prometheus.py / LIT-2034", fail_before_fix: proven, rationale: "Queue time feeds saturation alerting; the family stayed registered while no observation was ever recorded, so presence alone is not the contract"}
|
||||
- {id: logging.prometheus.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/prometheus.py / GitHub issue #41976", rationale: "The callback-visible StandardLogging response_cost must equal the SpendLogs row for the same request id; a present-but-None hidden cost used to reach callbacks as 0 while the spend row was billed"}
|
||||
- {id: logging.custom_callback.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/custom_logger.py / GitHub issue #41976", rationale: "kwargs[standard_logging_object][response_cost] handed to a config-registered CustomLogger must equal the SpendLogs row for the same request id; the proxy routes /v1/responses through aresponses, which stayed correct on unfixed main, so this pins the reporter surface rather than reproducing the sync responses defect"}
|
||||
- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
|
||||
- {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"}
|
||||
- {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ litellm_settings:
|
|||
- host: 127.0.0.1
|
||||
port: 6379
|
||||
ssl: true
|
||||
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"]
|
||||
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "tests.e2e.logging.callback_cost_capture.capture"]
|
||||
require_auth_for_metrics_endpoint: false
|
||||
|
||||
router_settings:
|
||||
|
|
|
|||
65
tests/e2e/logging/callback_cost_capture.py
Normal file
65
tests/e2e/logging/callback_cost_capture.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""Config-registered CustomLogger that records the cost each success callback is handed.
|
||||
|
||||
Registered on the e2e gateway as ``callbacks: ["tests.e2e.logging.callback_cost_capture.capture"]``.
|
||||
Every success event appends one JSON line to ``E2E_CALLBACK_COST_CAPTURE_FILE`` with the request id
|
||||
and ``standard_logging_object.response_cost`` exactly as a customer's Langfuse, Datadog, or custom
|
||||
CustomLogger would see it. Tests read the file back and compare against the SpendLogs row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE"
|
||||
|
||||
|
||||
class CapturedCost(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
request_id: str
|
||||
call_type: str | None
|
||||
hook: str
|
||||
response_cost: float | None
|
||||
|
||||
|
||||
class _StandardLoggingCostFields(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
id: str
|
||||
call_type: str | None = None
|
||||
response_cost: float | None = None
|
||||
|
||||
|
||||
def _append(hook: str, kwargs: dict[str, object]) -> None:
|
||||
path: Final = os.environ.get(CAPTURE_FILE_ENV)
|
||||
payload: Final = kwargs.get("standard_logging_object")
|
||||
if path is None or not isinstance(payload, dict):
|
||||
return
|
||||
fields: Final = _StandardLoggingCostFields.model_validate(payload)
|
||||
row: Final = CapturedCost(
|
||||
request_id=fields.id, call_type=fields.call_type, hook=hook, response_cost=fields.response_cost
|
||||
)
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(row.model_dump()) + "\n")
|
||||
|
||||
|
||||
class CallbackCostCapture(CustomLogger):
|
||||
def log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
_append("log_success_event", kwargs)
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
_append("async_log_success_event", kwargs)
|
||||
|
||||
|
||||
capture: Final = CallbackCostCapture()
|
||||
84
tests/e2e/logging/test_custom_callback_responses_cost_e2e.py
Normal file
84
tests/e2e/logging/test_custom_callback_responses_cost_e2e.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""The response_cost a config-registered CustomLogger receives must be the cost the spend row bills.
|
||||
|
||||
GitHub issue #41976: a successful sync, non-streaming ``/v1/responses`` call reached callbacks with
|
||||
``kwargs["standard_logging_object"]["response_cost"] == 0`` while ``LiteLLM_SpendLogs.spend`` for the
|
||||
same request id was positive. The gateway registers ``callback_cost_capture.capture`` from this
|
||||
directory as a plain ``callbacks:`` entry, so this is the exact payload a Langfuse, Datadog, or
|
||||
customer CustomLogger is handed. The capture file path comes from ``E2E_CALLBACK_COST_CAPTURE_FILE``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from callback_cost_capture import CAPTURE_FILE_ENV, CapturedCost
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import LoggingClient, first_ok
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
class _ResponsesBody(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
def _capture_file() -> Path:
|
||||
path = os.environ.get(CAPTURE_FILE_ENV)
|
||||
assert path, f"{CAPTURE_FILE_ENV} is unset: the gateway must register callback_cost_capture.capture"
|
||||
return Path(path)
|
||||
|
||||
|
||||
def _captured_rows(path: Path, request_id: str) -> list[CapturedCost]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows = [CapturedCost.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
|
||||
return [row for row in rows if row.request_id == request_id]
|
||||
|
||||
|
||||
def _poll_captured(client: LoggingClient, path: Path, request_id: str) -> list[CapturedCost]:
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while True:
|
||||
rows = _captured_rows(path, request_id)
|
||||
if rows or time.monotonic() >= deadline:
|
||||
return rows
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
|
||||
|
||||
class TestCustomCallbackResponsesCost:
|
||||
@pytest.mark.covers("logging.custom_callback.success.logs_spend", exercised_on=["responses"])
|
||||
def test_sync_responses_callback_payload_cost_equals_spend_log_row(
|
||||
self, client: LoggingClient, resources: ResourceManager
|
||||
) -> None:
|
||||
capture_file = _capture_file()
|
||||
alias = f"e2e-callback-cost-{unique_marker()}"
|
||||
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
outcome = first_ok(
|
||||
client,
|
||||
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
|
||||
)
|
||||
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
|
||||
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
|
||||
|
||||
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
|
||||
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
|
||||
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
|
||||
)
|
||||
|
||||
captured = _poll_captured(client, capture_file, response_id)
|
||||
assert captured, f"CustomLogger saw no success event for request_id={response_id} in {capture_file}"
|
||||
for row in captured:
|
||||
assert row.response_cost is not None and math.isclose(row.response_cost, spend_row.spend, rel_tol=1e-9), (
|
||||
f"{row.hook} ({row.call_type}) saw standard_logging_object.response_cost={row.response_cost} "
|
||||
f"but SpendLogs billed {spend_row.spend} for request_id={response_id}"
|
||||
)
|
||||
83
tests/e2e/logging/test_prometheus_responses_cost_e2e.py
Normal file
83
tests/e2e/logging/test_prometheus_responses_cost_e2e.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
"""The cost a logging callback sees must be the cost the spend row bills.
|
||||
|
||||
GitHub issue #41976: a successful sync ``/v1/responses`` call reached callbacks with
|
||||
``StandardLoggingPayload.response_cost == 0`` while ``LiteLLM_SpendLogs.spend`` for the
|
||||
same request id was positive. Prometheus is the callback under test here because its
|
||||
``litellm_spend_metric`` counter is fed straight from ``standard_logging_object.response_cost``
|
||||
and is readable from the proxy itself, so no external sink is needed. A fresh key with one
|
||||
request makes the per-alias counter equal that request's callback-visible cost.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import LoggingClient, first_ok
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
SPEND_METRIC = "litellm_spend_metric"
|
||||
ALIAS_LABEL = "api_key_alias"
|
||||
|
||||
|
||||
class _ResponsesBody(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
def _callback_spend(exposition: str, alias: str) -> float | None:
|
||||
values = [
|
||||
sample.value
|
||||
for family in text_string_to_metric_families(exposition)
|
||||
for sample in family.samples
|
||||
if sample.name == f"{SPEND_METRIC}_total" and sample.labels.get(ALIAS_LABEL) == alias
|
||||
]
|
||||
return sum(values) if values else None
|
||||
|
||||
|
||||
def _poll_callback_spend(client: LoggingClient, alias: str) -> float | None:
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while True:
|
||||
spend = _callback_spend(client.scrape_metrics(), alias)
|
||||
if spend is not None or time.monotonic() >= deadline:
|
||||
return spend
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
|
||||
|
||||
class TestResponsesCallbackCost:
|
||||
@pytest.mark.covers("logging.prometheus.success.logs_spend", exercised_on=["responses"])
|
||||
def test_responses_callback_cost_equals_spend_log_row(
|
||||
self, client: LoggingClient, resources: ResourceManager
|
||||
) -> None:
|
||||
alias = f"e2e-responses-cost-{unique_marker()}"
|
||||
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
outcome = first_ok(
|
||||
client,
|
||||
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
|
||||
)
|
||||
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
|
||||
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
|
||||
|
||||
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
|
||||
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
|
||||
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
|
||||
)
|
||||
|
||||
callback_spend = _poll_callback_spend(client, alias)
|
||||
assert callback_spend is not None, (
|
||||
f"{SPEND_METRIC} has no series for {ALIAS_LABEL}={alias} although the spend row billed {spend_row.spend}"
|
||||
)
|
||||
assert math.isclose(callback_spend, spend_row.spend, rel_tol=1e-9), (
|
||||
f"callback saw response_cost={callback_spend} but SpendLogs billed {spend_row.spend} "
|
||||
f"for request_id={response_id}"
|
||||
)
|
||||
|
|
@ -472,7 +472,7 @@ class TestZeroCostDiagnostic:
|
|||
self._assert_flagged(logging_obj, caplog)
|
||||
assert payload["zero_cost_diagnostic"] == logging_obj.model_call_details["zero_cost_diagnostic"]
|
||||
|
||||
def test_uncomputed_hidden_cost_is_not_a_zero_cost(
|
||||
def test_uncomputed_hidden_cost_is_priced_like_an_absent_one(
|
||||
self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
|
||||
|
|
@ -484,8 +484,13 @@ class TestZeroCostDiagnostic:
|
|||
response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now()
|
||||
)
|
||||
|
||||
assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None
|
||||
assert self._zero_cost_warnings(caplog) == []
|
||||
payload: Final = logging_obj.model_call_details["standard_logging_object"]
|
||||
assert payload["response_cost"] == 0.0
|
||||
if deployment_pricing is self.FREE_PRICING:
|
||||
assert payload["zero_cost_diagnostic"] is None
|
||||
assert self._zero_cost_warnings(caplog) == []
|
||||
return
|
||||
self._assert_flagged(logging_obj, caplog)
|
||||
|
||||
def test_unbilled_read_route_with_usage_stays_silent(
|
||||
self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture
|
||||
|
|
@ -3822,6 +3827,75 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer
|
|||
assert slo.get("response_cost") == passthrough_cost
|
||||
|
||||
|
||||
def test_process_hidden_params_falls_through_on_none_response_cost():
|
||||
"""A present-but-None hidden response_cost is priced by the calculator, not logged as 0 (issue #41976)."""
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="responses",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-none-hidden-cost",
|
||||
function_id="test-none-hidden-cost",
|
||||
)
|
||||
logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"}
|
||||
logging_obj.optional_params = {}
|
||||
|
||||
result = ModelResponse(
|
||||
id="success",
|
||||
choices=[{"message": {"role": "assistant", "content": "ok"}}],
|
||||
usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110),
|
||||
)
|
||||
result._hidden_params = {"response_cost": None}
|
||||
|
||||
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
|
||||
|
||||
calculated_cost = logging_obj._response_cost_calculator(result=result)
|
||||
assert calculated_cost is not None and calculated_cost > 0
|
||||
assert logging_obj.model_call_details.get("response_cost") == calculated_cost
|
||||
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
|
||||
assert slo.get("response_cost") == calculated_cost
|
||||
|
||||
|
||||
def test_process_hidden_params_explicit_zero_cost_overrides_earlier_cost():
|
||||
"""An explicit hidden response_cost of 0 wins over a cost already on model_call_details; only None falls through."""
|
||||
from datetime import datetime
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="responses",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-zero-hidden-cost-overrides",
|
||||
function_id="test-zero-hidden-cost-overrides",
|
||||
)
|
||||
logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"}
|
||||
logging_obj.optional_params = {}
|
||||
logging_obj.model_call_details["response_cost"] = 0.5
|
||||
|
||||
result = ModelResponse(
|
||||
id="success",
|
||||
choices=[{"message": {"role": "assistant", "content": "ok"}}],
|
||||
usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110),
|
||||
)
|
||||
result._hidden_params = {"response_cost": 0.0}
|
||||
|
||||
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
|
||||
|
||||
assert logging_obj.model_call_details.get("response_cost") == 0.0
|
||||
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
|
||||
assert slo.get("response_cost") == 0.0
|
||||
|
||||
|
||||
def test_function_setup_litellm_metadata_populates_metadata():
|
||||
"""
|
||||
Test that function_setup() properly handles litellm_metadata (used by /v1/messages,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue