From 8d7ab9ec0e9803ed240a853d162b56d4fc7d9d9f Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 22 Sep 2026 18:47:07 +0000 Subject: [PATCH] test(observability): move the responses callback cost repro to tests/integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/e2e-stack/up.sh | 2 - tests/e2e/conftest.py | 8 -- tests/e2e/coverage_registry/logging.yaml | 2 - tests/e2e/e2e_config.py | 1 - tests/e2e/gateway/stage_mirror_ci_config.yml | 2 +- tests/e2e/logging/callback_cost_capture.py | 47 ------- tests/e2e/logging/callback_cost_record.py | 16 --- ...test_custom_callback_responses_cost_e2e.py | 75 ----------- .../test_prometheus_responses_cost_e2e.py | 73 ----------- tests/e2e/pytest.ini | 1 - tests/integration/contracts.json | 3 + .../test_responses_callback_cost.py | 116 ++++++++++++++++++ 12 files changed, 120 insertions(+), 226 deletions(-) delete mode 100644 tests/e2e/logging/callback_cost_capture.py delete mode 100644 tests/e2e/logging/callback_cost_record.py delete mode 100644 tests/e2e/logging/test_custom_callback_responses_cost_e2e.py delete mode 100644 tests/e2e/logging/test_prometheus_responses_cost_e2e.py create mode 100644 tests/integration/observability/test_responses_callback_cost.py diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 174c85f9825..928b58e93bb 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -125,7 +125,6 @@ SERVER_ENV=( "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" - "E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl" "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" "JWT_AUDIENCE=litellm-e2e" @@ -211,7 +210,6 @@ E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem -E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME} EOF diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 214fe594f00..ca0fbd84c35 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -24,7 +24,6 @@ from typing import Final import pytest import requests from e2e_config import ( - CALLBACK_COST_CAPTURE_OPT_IN_ENV, CLI_DETERMINISM_OPT_IN_ENV, CONTROL_PLANE_BASE_URL, FIXTURE_DIR, @@ -63,7 +62,6 @@ OPT_IN_MARKERS: Final = MappingProxyType( "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, - "callback_cost_capture": CALLBACK_COST_CAPTURE_OPT_IN_ENV, "otel_v2": OTEL_V2_OPT_IN_ENV, } ) @@ -154,12 +152,6 @@ def pytest_configure(config: pytest.Config) -> None: "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " "gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set", ) - config.addinivalue_line( - "markers", - "callback_cost_capture: reads the JSONL written by the gateway's registered " - "logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; " - "deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set", - ) config.addinivalue_line( "markers", "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index d21a89d9aa2..1f2f1d64711 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -7,8 +7,6 @@ - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} - {id: logging.prometheus.success.records_queue_time, module: logging, tier: P1, event: success, assertions: [records_queue_time], exercised_on: [chat_completions], source: "integrations/prometheus.py / LIT-2034", fail_before_fix: proven, rationale: "Queue time feeds saturation alerting; the family stayed registered while no observation was ever recorded, so presence alone is not the contract"} -- {id: logging.prometheus.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/prometheus.py / GitHub issue #41976", rationale: "The callback-visible StandardLogging response_cost must equal the SpendLogs row for the same request id; a present-but-None hidden cost used to reach callbacks as 0 while the spend row was billed"} -- {id: logging.custom_callback.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/custom_logger.py / GitHub issue #41976", rationale: "kwargs[standard_logging_object][response_cost] handed to a config-registered CustomLogger must equal the SpendLogs row for the same request id; the proxy routes /v1/responses through aresponses, which stayed correct on unfixed main, so this pins the reporter surface rather than reproducing the sync responses defect"} - {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} - {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"} - {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 5055d36c0dc..311c944eeb4 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -147,7 +147,6 @@ REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" -CALLBACK_COST_CAPTURE_OPT_IN_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index b5a3d6ff7cf..2d02fedddae 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -32,7 +32,7 @@ litellm_settings: - host: 127.0.0.1 port: 6379 ssl: true - callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "tests.e2e.logging.callback_cost_capture.capture"] + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] require_auth_for_metrics_endpoint: false router_settings: diff --git a/tests/e2e/logging/callback_cost_capture.py b/tests/e2e/logging/callback_cost_capture.py deleted file mode 100644 index 95cd434f0bb..00000000000 --- a/tests/e2e/logging/callback_cost_capture.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -import json -import os -from datetime import datetime -from typing import Final - -from pydantic import BaseModel, ConfigDict - -from litellm.integrations.custom_logger import CustomLogger -from tests.e2e.logging.callback_cost_record import CAPTURE_FILE_ENV, CapturedCost - - -class _StandardLoggingCostFields(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - - id: str - call_type: str | None = None - response_cost: float | None = None - - -def _append(hook: str, kwargs: dict[str, object]) -> None: - path: Final = os.environ.get(CAPTURE_FILE_ENV) - payload: Final = kwargs.get("standard_logging_object") - if path is None or not isinstance(payload, dict): - return - fields: Final = _StandardLoggingCostFields.model_validate(payload) - row: Final = CapturedCost( - request_id=fields.id, call_type=fields.call_type, hook=hook, response_cost=fields.response_cost - ) - with open(path, "a", encoding="utf-8") as handle: - handle.write(json.dumps(row.model_dump()) + "\n") - - -class CallbackCostCapture(CustomLogger): - def log_success_event( - self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime - ) -> None: - _append("log_success_event", kwargs) - - async def async_log_success_event( - self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime - ) -> None: - _append("async_log_success_event", kwargs) - - -capture: Final = CallbackCostCapture() diff --git a/tests/e2e/logging/callback_cost_record.py b/tests/e2e/logging/callback_cost_record.py deleted file mode 100644 index 300f235662b..00000000000 --- a/tests/e2e/logging/callback_cost_record.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from pydantic import BaseModel, ConfigDict - -CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE" - - -class CapturedCost(BaseModel): - model_config = ConfigDict(extra="ignore", frozen=True) - - request_id: str - call_type: str | None - hook: str - response_cost: float | None diff --git a/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py b/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py deleted file mode 100644 index f70726f50c8..00000000000 --- a/tests/e2e/logging/test_custom_callback_responses_cost_e2e.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import annotations - -import json -import math -import os -import time -from pathlib import Path - -import pytest -from callback_cost_record import CAPTURE_FILE_ENV, CapturedCost -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker -from lifecycle import ResourceManager -from logging_client import LoggingClient, first_ok -from pydantic import BaseModel, ConfigDict - -pytestmark = [pytest.mark.e2e, pytest.mark.callback_cost_capture] - - -class _ResponsesBody(BaseModel): - model_config = ConfigDict(extra="ignore") - - id: str - - -def _capture_file() -> Path: - path = os.environ.get(CAPTURE_FILE_ENV) - assert path, f"{CAPTURE_FILE_ENV} is unset: the gateway must register callback_cost_capture.capture" - return Path(path) - - -def _captured_rows(path: Path, request_id: str) -> list[CapturedCost]: - if not path.exists(): - return [] - rows = [CapturedCost.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line] - return [row for row in rows if row.request_id == request_id] - - -def _poll_captured(client: LoggingClient, path: Path, request_id: str) -> list[CapturedCost]: - deadline = time.monotonic() + client.proxy.poll_timeout - while True: - rows = _captured_rows(path, request_id) - if rows or time.monotonic() >= deadline: - return rows - time.sleep(client.proxy.poll_interval) - - -class TestCustomCallbackResponsesCost: - @pytest.mark.covers("logging.custom_callback.success.logs_spend", exercised_on=["responses"]) - def test_sync_responses_callback_payload_cost_equals_spend_log_row( - self, client: LoggingClient, resources: ResourceManager - ) -> None: - capture_file = _capture_file() - alias = f"e2e-callback-cost-{unique_marker()}" - key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL]) - resources.defer(lambda: client.delete_key(key)) - - outcome = first_ok( - client, - lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"), - ) - response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id - assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}" - - spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id) - assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( - f"no positive-spend row for request_id={response_id}, got {spend_row!r}" - ) - - captured = _poll_captured(client, capture_file, response_id) - assert captured, f"CustomLogger saw no success event for request_id={response_id} in {capture_file}" - for row in captured: - assert row.response_cost is not None and math.isclose(row.response_cost, spend_row.spend, rel_tol=1e-9), ( - f"{row.hook} ({row.call_type}) saw standard_logging_object.response_cost={row.response_cost} " - f"but SpendLogs billed {spend_row.spend} for request_id={response_id}" - ) diff --git a/tests/e2e/logging/test_prometheus_responses_cost_e2e.py b/tests/e2e/logging/test_prometheus_responses_cost_e2e.py deleted file mode 100644 index 8d293d8f112..00000000000 --- a/tests/e2e/logging/test_prometheus_responses_cost_e2e.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import json -import math -import time - -import pytest -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker -from lifecycle import ResourceManager -from logging_client import LoggingClient, first_ok -from prometheus_client.parser import text_string_to_metric_families -from pydantic import BaseModel, ConfigDict - -pytestmark = pytest.mark.e2e - -SPEND_METRIC = "litellm_spend_metric" -ALIAS_LABEL = "api_key_alias" - - -class _ResponsesBody(BaseModel): - model_config = ConfigDict(extra="ignore") - - id: str - - -def _callback_spend(exposition: str, alias: str) -> float | None: - values = [ - sample.value - for family in text_string_to_metric_families(exposition) - for sample in family.samples - if sample.name == f"{SPEND_METRIC}_total" and sample.labels.get(ALIAS_LABEL) == alias - ] - return sum(values) if values else None - - -def _poll_callback_spend(client: LoggingClient, alias: str) -> float | None: - deadline = time.monotonic() + client.proxy.poll_timeout - while True: - spend = _callback_spend(client.scrape_metrics(), alias) - if spend is not None or time.monotonic() >= deadline: - return spend - time.sleep(client.proxy.poll_interval) - - -class TestResponsesCallbackCost: - @pytest.mark.covers("logging.prometheus.success.logs_spend", exercised_on=["responses"]) - def test_responses_callback_cost_equals_spend_log_row( - self, client: LoggingClient, resources: ResourceManager - ) -> None: - alias = f"e2e-responses-cost-{unique_marker()}" - key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL]) - resources.defer(lambda: client.delete_key(key)) - - outcome = first_ok( - client, - lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"), - ) - response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id - assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}" - - spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id) - assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, ( - f"no positive-spend row for request_id={response_id}, got {spend_row!r}" - ) - - callback_spend = _poll_callback_spend(client, alias) - assert callback_spend is not None, ( - f"{SPEND_METRIC} has no series for {ALIAS_LABEL}={alias} although the spend row billed {spend_row.spend}" - ) - assert math.isclose(callback_spend, spend_row.spend, rel_tol=1e-9), ( - f"callback saw response_cost={callback_spend} but SpendLogs billed {spend_row.spend} " - f"for request_id={response_id}" - ) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index a84562e04d0..6f9f57d333e 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -14,5 +14,4 @@ markers = redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set - callback_cost_capture: reads the JSONL written by the gateway's registered logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e1b5940935f..376389a973c 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -221,6 +221,9 @@ "other.observability.callbacks.credentials_stay_out_of_event_bodies", "other.observability.callbacks.concurrent_results_join_complete_events_and_rows" ], + "tests/integration/observability/test_responses_callback_cost.py::test_responses_success_callback_cost_is_priced_and_matches_spend_row": [ + "other.observability.callbacks.responses_success_cost_matches_spend_row" + ], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" ], diff --git a/tests/integration/observability/test_responses_callback_cost.py b/tests/integration/observability/test_responses_callback_cost.py new file mode 100644 index 00000000000..2604153ed84 --- /dev/null +++ b/tests/integration/observability/test_responses_callback_cost.py @@ -0,0 +1,116 @@ +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + + +@pytest.mark.covers("other.observability.callbacks.responses_success_cost_matches_spend_row") +def test_responses_success_callback_cost_is_priced_and_matches_spend_row(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "responses-cost-" + uuid.uuid4().hex + secret: Final = "synthetic-provider-secret-" + marker + sink_secret: Final = "synthetic-sink-secret-" + marker + + def upstream(request: Request) -> Reply: + assert request.target == "/v1/responses" + assert request.headers["authorization"] == f"Bearer {secret}" + body: Final = json.loads(request.body) + assert body["input"] == marker + return Reply( + body=json.dumps( + { + "id": "resp_" + marker, + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_" + marker, + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": { + "input_tokens": 11, + "output_tokens": 4, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + } + ).encode() + ) + + def sink(request: Request) -> Reply: + assert request.headers["authorization"] == f"Bearer {sink_secret}" + return Reply() + + with wire_server(upstream) as provider, wire_server(sink) as endpoint: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"].update({"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}) + path: Final = tmp_path / "callbacks.yaml" + path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy( + gateway, + tmp_path, + { + "GENERIC_LOGGER_ENDPOINT": endpoint.url, + "GENERIC_LOGGER_HEADERS": f"Authorization=Bearer {sink_secret}", + }, + config=path, + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model( + api_base=provider.url + "/v1", api_key=secret, input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + key: Final = scenario.key(models=[model]) + response: Final = candidate.request( + "POST", + "/v1/responses", + {"model": model, "input": marker, "metadata": {"tags": [marker]}}, + key=key, + ) + assert response.status_code == 200, response.text + assert len(provider.drain()) == 1 + response_id: Final = response.json()["id"] + batches = [] + + def delivered() -> tuple[dict, ...]: + batches.extend(endpoint.drain()) + return tuple( + event + for batch in batches + for event in json.loads(batch.body) + if marker in event.get("request_tags", []) + ) + + events: Final = eventually(delivered, lambda values: len(values) == 1, seconds=10) + event: Final = events[0] + assert event["status"] == "success" + assert event["call_type"] in ("responses", "aresponses") + assert event["prompt_tokens"] == 11 and event["completion_tokens"] == 4 + assert event["response_cost"] == pytest.approx(11 * 0.001 + 4 * 0.002) + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, call_type FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (response_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(rows[0]["spend"]) == pytest.approx(event["response_cost"]) + assert rows[0]["call_type"] == event["call_type"]