diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 928b58e93bb..174c85f9825 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -125,6 +125,7 @@ SERVER_ENV=( "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" + "E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl" "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" "JWT_AUDIENCE=litellm-e2e" @@ -210,6 +211,7 @@ E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem +E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME} EOF diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f5967185af0..4249cd4573d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2317,7 +2317,7 @@ class Logging(LiteLLMLoggingBaseClass): if self.model_call_details.get("cache_hit") is True: self.model_call_details["response_cost"] = 0.0 - elif "response_cost" in hidden_params: + elif hidden_params.get("response_cost") is not None: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0: diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 1f2f1d64711..d21a89d9aa2 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -7,6 +7,8 @@ - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} - {id: logging.prometheus.success.records_queue_time, module: logging, tier: P1, event: success, assertions: [records_queue_time], exercised_on: [chat_completions], source: "integrations/prometheus.py / LIT-2034", fail_before_fix: proven, rationale: "Queue time feeds saturation alerting; the family stayed registered while no observation was ever recorded, so presence alone is not the contract"} +- {id: logging.prometheus.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/prometheus.py / GitHub issue #41976", rationale: "The callback-visible StandardLogging response_cost must equal the SpendLogs row for the same request id; a present-but-None hidden cost used to reach callbacks as 0 while the spend row was billed"} +- {id: logging.custom_callback.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/custom_logger.py / GitHub issue #41976", rationale: "kwargs[standard_logging_object][response_cost] handed to a config-registered CustomLogger must equal the SpendLogs row for the same request id; the proxy routes /v1/responses through aresponses, which stayed correct on unfixed main, so this pins the reporter surface rather than reproducing the sync responses defect"} - {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} - {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"} - {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"} diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 2d02fedddae..b5a3d6ff7cf 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -32,7 +32,7 @@ litellm_settings: - host: 127.0.0.1 port: 6379 ssl: true - callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "tests.e2e.logging.callback_cost_capture.capture"] require_auth_for_metrics_endpoint: false router_settings: diff --git a/tests/e2e/logging/callback_cost_capture.py b/tests/e2e/logging/callback_cost_capture.py new file mode 100644 index 00000000000..efc1a8edfbb --- /dev/null +++ b/tests/e2e/logging/callback_cost_capture.py @@ -0,0 +1,65 @@ +"""Config-registered CustomLogger that records the cost each success callback is handed. + +Registered on the e2e gateway as ``callbacks: ["tests.e2e.logging.callback_cost_capture.capture"]``. +Every success event appends one JSON line to ``E2E_CALLBACK_COST_CAPTURE_FILE`` with the request id +and ``standard_logging_object.response_cost`` exactly as a customer's Langfuse, Datadog, or custom +CustomLogger would see it. Tests read the file back and compare against the SpendLogs row. +""" + +from __future__ import annotations + +import json +import os +from datetime import datetime +from typing import Final + +from pydantic import BaseModel, ConfigDict + +from litellm.integrations.custom_logger import CustomLogger + +CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" + + +class CapturedCost(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + request_id: str + call_type: str | None + hook: str + response_cost: float | None + + +class _StandardLoggingCostFields(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + id: str + call_type: str | None = None + response_cost: float | None = None + + +def _append(hook: str, kwargs: dict[str, object]) -> None: + path: Final = os.environ.get(CAPTURE_FILE_ENV) + payload: Final = kwargs.get("standard_logging_object") + if path is None or not isinstance(payload, dict): + return + fields: Final = _StandardLoggingCostFields.model_validate(payload) + row: Final = CapturedCost( + request_id=fields.id, call_type=fields.call_type, hook=hook, response_cost=fields.response_cost + ) + with open(path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(row.model_dump()) + "\n") + + +class CallbackCostCapture(CustomLogger): + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _append("log_success_event", kwargs) + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _append("async_log_success_event", kwargs) + + +capture: Final = CallbackCostCapture() diff --git a/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py b/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py new file mode 100644 index 00000000000..13024e7ed31 --- /dev/null +++ b/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py @@ -0,0 +1,84 @@ +"""The response_cost a config-registered CustomLogger receives must be the cost the spend row bills. + +GitHub issue #41976: a successful sync, non-streaming ``/v1/responses`` call reached callbacks with +``kwargs["standard_logging_object"]["response_cost"] == 0`` while ``LiteLLM_SpendLogs.spend`` for the +same request id was positive. The gateway registers ``callback_cost_capture.capture`` from this +directory as a plain ``callbacks:`` entry, so this is the exact payload a Langfuse, Datadog, or +customer CustomLogger is handed. The capture file path comes from ``E2E_CALLBACK_COST_CAPTURE_FILE``. +""" + +from __future__ import annotations + +import json +import math +import os +import time +from pathlib import Path + +import pytest +from callback_cost_capture import CAPTURE_FILE_ENV, CapturedCost +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from lifecycle import ResourceManager +from logging_client import LoggingClient, first_ok +from pydantic import BaseModel, ConfigDict + +pytestmark = pytest.mark.e2e + + +class _ResponsesBody(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: str + + +def _capture_file() -> Path: + path = os.environ.get(CAPTURE_FILE_ENV) + assert path, f"{CAPTURE_FILE_ENV} is unset: the gateway must register callback_cost_capture.capture" + return Path(path) + + +def _captured_rows(path: Path, request_id: str) -> list[CapturedCost]: + if not path.exists(): + return [] + rows = [CapturedCost.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + return [row for row in rows if row.request_id == request_id] + + +def _poll_captured(client: LoggingClient, path: Path, request_id: str) -> list[CapturedCost]: + deadline = time.monotonic() + client.proxy.poll_timeout + while True: + rows = _captured_rows(path, request_id) + if rows or time.monotonic() >= deadline: + return rows + time.sleep(client.proxy.poll_interval) + + +class TestCustomCallbackResponsesCost: + @pytest.mark.covers("logging.custom_callback.success.logs_spend", exercised_on=["responses"]) + def test_sync_responses_callback_payload_cost_equals_spend_log_row( + self, client: LoggingClient, resources: ResourceManager + ) -> None: + capture_file = _capture_file() + alias = f"e2e-callback-cost-{unique_marker()}" + key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + outcome = first_ok( + client, + lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"), + ) + response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id + assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}" + + spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id) + assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( + f"no positive-spend row for request_id={response_id}, got {spend_row!r}" + ) + + captured = _poll_captured(client, capture_file, response_id) + assert captured, f"CustomLogger saw no success event for request_id={response_id} in {capture_file}" + for row in captured: + assert row.response_cost is not None and math.isclose(row.response_cost, spend_row.spend, rel_tol=1e-9), ( + f"{row.hook} ({row.call_type}) saw standard_logging_object.response_cost={row.response_cost} " + f"but SpendLogs billed {spend_row.spend} for request_id={response_id}" + ) diff --git a/tests/e2e/logging/test_prometheus_responses_cost_e2e.py b/tests/e2e/logging/test_prometheus_responses_cost_e2e.py new file mode 100644 index 00000000000..7bb63a02549 --- /dev/null +++ b/tests/e2e/logging/test_prometheus_responses_cost_e2e.py @@ -0,0 +1,83 @@ +"""The cost a logging callback sees must be the cost the spend row bills. + +GitHub issue #41976: a successful sync ``/v1/responses`` call reached callbacks with +``StandardLoggingPayload.response_cost == 0`` while ``LiteLLM_SpendLogs.spend`` for the +same request id was positive. Prometheus is the callback under test here because its +``litellm_spend_metric`` counter is fed straight from ``standard_logging_object.response_cost`` +and is readable from the proxy itself, so no external sink is needed. A fresh key with one +request makes the per-alias counter equal that request's callback-visible cost. +""" + +from __future__ import annotations + +import json +import math +import time + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from lifecycle import ResourceManager +from logging_client import LoggingClient, first_ok +from prometheus_client.parser import text_string_to_metric_families +from pydantic import BaseModel, ConfigDict + +pytestmark = pytest.mark.e2e + +SPEND_METRIC = "litellm_spend_metric" +ALIAS_LABEL = "api_key_alias" + + +class _ResponsesBody(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: str + + +def _callback_spend(exposition: str, alias: str) -> float | None: + values = [ + sample.value + for family in text_string_to_metric_families(exposition) + for sample in family.samples + if sample.name == f"{SPEND_METRIC}_total" and sample.labels.get(ALIAS_LABEL) == alias + ] + return sum(values) if values else None + + +def _poll_callback_spend(client: LoggingClient, alias: str) -> float | None: + deadline = time.monotonic() + client.proxy.poll_timeout + while True: + spend = _callback_spend(client.scrape_metrics(), alias) + if spend is not None or time.monotonic() >= deadline: + return spend + time.sleep(client.proxy.poll_interval) + + +class TestResponsesCallbackCost: + @pytest.mark.covers("logging.prometheus.success.logs_spend", exercised_on=["responses"]) + def test_responses_callback_cost_equals_spend_log_row( + self, client: LoggingClient, resources: ResourceManager + ) -> None: + alias = f"e2e-responses-cost-{unique_marker()}" + key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + outcome = first_ok( + client, + lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"), + ) + response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id + assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}" + + spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id) + assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( + f"no positive-spend row for request_id={response_id}, got {spend_row!r}" + ) + + callback_spend = _poll_callback_spend(client, alias) + assert callback_spend is not None, ( + f"{SPEND_METRIC} has no series for {ALIAS_LABEL}={alias} although the spend row billed {spend_row.spend}" + ) + assert math.isclose(callback_spend, spend_row.spend, rel_tol=1e-9), ( + f"callback saw response_cost={callback_spend} but SpendLogs billed {spend_row.spend} " + f"for request_id={response_id}" + ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 1bfc55d1486..527a786fbc3 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -472,7 +472,7 @@ class TestZeroCostDiagnostic: self._assert_flagged(logging_obj, caplog) assert payload["zero_cost_diagnostic"] == logging_obj.model_call_details["zero_cost_diagnostic"] - def test_uncomputed_hidden_cost_is_not_a_zero_cost( + def test_uncomputed_hidden_cost_is_priced_like_an_absent_one( self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture ) -> None: usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) @@ -484,8 +484,13 @@ class TestZeroCostDiagnostic: response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() ) - assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None - assert self._zero_cost_warnings(caplog) == [] + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["response_cost"] == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert payload["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) def test_unbilled_read_route_with_usage_stays_silent( self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture @@ -3822,6 +3827,75 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer assert slo.get("response_cost") == passthrough_cost +def test_process_hidden_params_falls_through_on_none_response_cost(): + """A present-but-None hidden response_cost is priced by the calculator, not logged as 0 (issue #41976).""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import ModelResponse, Usage + + logging_obj = LiteLLMLoggingObj( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="responses", + start_time=datetime.now(), + litellm_call_id="test-none-hidden-cost", + function_id="test-none-hidden-cost", + ) + logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"} + logging_obj.optional_params = {} + + result = ModelResponse( + id="success", + choices=[{"message": {"role": "assistant", "content": "ok"}}], + usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110), + ) + result._hidden_params = {"response_cost": None} + + logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now()) + + calculated_cost = logging_obj._response_cost_calculator(result=result) + assert calculated_cost is not None and calculated_cost > 0 + assert logging_obj.model_call_details.get("response_cost") == calculated_cost + slo = logging_obj.model_call_details.get("standard_logging_object") or {} + assert slo.get("response_cost") == calculated_cost + + +def test_process_hidden_params_explicit_zero_cost_overrides_earlier_cost(): + """An explicit hidden response_cost of 0 wins over a cost already on model_call_details; only None falls through.""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.utils import ModelResponse, Usage + + logging_obj = LiteLLMLoggingObj( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="responses", + start_time=datetime.now(), + litellm_call_id="test-zero-hidden-cost-overrides", + function_id="test-zero-hidden-cost-overrides", + ) + logging_obj.model_call_details["litellm_params"] = {"model": "openai/gpt-4o-mini"} + logging_obj.optional_params = {} + logging_obj.model_call_details["response_cost"] = 0.5 + + result = ModelResponse( + id="success", + choices=[{"message": {"role": "assistant", "content": "ok"}}], + usage=Usage(prompt_tokens=100, completion_tokens=10, total_tokens=110), + ) + result._hidden_params = {"response_cost": 0.0} + + logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now()) + + assert logging_obj.model_call_details.get("response_cost") == 0.0 + slo = logging_obj.model_call_details.get("standard_logging_object") or {} + assert slo.get("response_cost") == 0.0 + + def test_function_setup_litellm_metadata_populates_metadata(): """ Test that function_setup() properly handles litellm_metadata (used by /v1/messages,