mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(observability): move the responses callback cost repro to tests/integration
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
3587090c5c
commit
8d7ab9ec0e
12 changed files with 120 additions and 226 deletions
2
.github/e2e-stack/up.sh
vendored
2
.github/e2e-stack/up.sh
vendored
|
|
@ -125,7 +125,6 @@ SERVER_ENV=(
|
|||
"OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
|
||||
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
|
||||
"PYTHONPATH=${REPO_ROOT}"
|
||||
"E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl"
|
||||
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
|
||||
"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
|
||||
"JWT_AUDIENCE=litellm-e2e"
|
||||
|
|
@ -211,7 +210,6 @@ E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
|
|||
E2E_KEYCLOAK_ADMIN_USER=admin
|
||||
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
|
||||
SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
|
||||
E2E_CALLBACK_COST_CAPTURE_FILE=${STACK_DIR}/callback-cost-capture.jsonl
|
||||
DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
|
||||
EOF
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ from typing import Final
|
|||
import pytest
|
||||
import requests
|
||||
from e2e_config import (
|
||||
CALLBACK_COST_CAPTURE_OPT_IN_ENV,
|
||||
CLI_DETERMINISM_OPT_IN_ENV,
|
||||
CONTROL_PLANE_BASE_URL,
|
||||
FIXTURE_DIR,
|
||||
|
|
@ -63,7 +62,6 @@ OPT_IN_MARKERS: Final = MappingProxyType(
|
|||
"cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
|
||||
"mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV,
|
||||
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
"callback_cost_capture": CALLBACK_COST_CAPTURE_OPT_IN_ENV,
|
||||
"otel_v2": OTEL_V2_OPT_IN_ENV,
|
||||
}
|
||||
)
|
||||
|
|
@ -154,12 +152,6 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the "
|
||||
"gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"callback_cost_capture: reads the JSONL written by the gateway's registered "
|
||||
"logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; "
|
||||
"deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set",
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@
|
|||
- {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"}
|
||||
- {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"}
|
||||
- {id: logging.prometheus.success.records_queue_time, module: logging, tier: P1, event: success, assertions: [records_queue_time], exercised_on: [chat_completions], source: "integrations/prometheus.py / LIT-2034", fail_before_fix: proven, rationale: "Queue time feeds saturation alerting; the family stayed registered while no observation was ever recorded, so presence alone is not the contract"}
|
||||
- {id: logging.prometheus.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/prometheus.py / GitHub issue #41976", rationale: "The callback-visible StandardLogging response_cost must equal the SpendLogs row for the same request id; a present-but-None hidden cost used to reach callbacks as 0 while the spend row was billed"}
|
||||
- {id: logging.custom_callback.success.logs_spend, module: logging, tier: P0, event: success, assertions: [logs_spend], exercised_on: [responses], source: "integrations/custom_logger.py / GitHub issue #41976", rationale: "kwargs[standard_logging_object][response_cost] handed to a config-registered CustomLogger must equal the SpendLogs row for the same request id; the proxy routes /v1/responses through aresponses, which stayed correct on unfixed main, so this pins the reporter surface rather than reproducing the sync responses defect"}
|
||||
- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
|
||||
- {id: logging.otel.stream.exports_metric, module: logging, tier: P0, event: stream, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/logger.py", rationale: "Streaming closes the LLM span from the stream path; historically prone to duplicate/orphaned spans"}
|
||||
- {id: logging.otel.stream.records_ttft, module: logging, tier: P1, event: stream, assertions: [records_ttft], exercised_on: [chat_completions, messages, responses], source: "integrations/otel/mappers/genai.py", rationale: "TTFT is the streaming latency SLI; a zero or span-length value silently corrupts dashboards"}
|
||||
|
|
|
|||
|
|
@ -147,7 +147,6 @@ REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS"
|
|||
CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
|
||||
MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
|
||||
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
|
||||
CALLBACK_COST_CAPTURE_OPT_IN_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE"
|
||||
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"
|
||||
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
|
||||
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ litellm_settings:
|
|||
- host: 127.0.0.1
|
||||
port: 6379
|
||||
ssl: true
|
||||
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "tests.e2e.logging.callback_cost_capture.capture"]
|
||||
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"]
|
||||
require_auth_for_metrics_endpoint: false
|
||||
|
||||
router_settings:
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from tests.e2e.logging.callback_cost_record import CAPTURE_FILE_ENV, CapturedCost
|
||||
|
||||
|
||||
class _StandardLoggingCostFields(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
id: str
|
||||
call_type: str | None = None
|
||||
response_cost: float | None = None
|
||||
|
||||
|
||||
def _append(hook: str, kwargs: dict[str, object]) -> None:
|
||||
path: Final = os.environ.get(CAPTURE_FILE_ENV)
|
||||
payload: Final = kwargs.get("standard_logging_object")
|
||||
if path is None or not isinstance(payload, dict):
|
||||
return
|
||||
fields: Final = _StandardLoggingCostFields.model_validate(payload)
|
||||
row: Final = CapturedCost(
|
||||
request_id=fields.id, call_type=fields.call_type, hook=hook, response_cost=fields.response_cost
|
||||
)
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(row.model_dump()) + "\n")
|
||||
|
||||
|
||||
class CallbackCostCapture(CustomLogger):
|
||||
def log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
_append("log_success_event", kwargs)
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
_append("async_log_success_event", kwargs)
|
||||
|
||||
|
||||
capture: Final = CallbackCostCapture()
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
CAPTURE_FILE_ENV: Final = "E2E_CALLBACK_COST_CAPTURE_FILE"
|
||||
|
||||
|
||||
class CapturedCost(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
request_id: str
|
||||
call_type: str | None
|
||||
hook: str
|
||||
response_cost: float | None
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from callback_cost_record import CAPTURE_FILE_ENV, CapturedCost
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import LoggingClient, first_ok
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.callback_cost_capture]
|
||||
|
||||
|
||||
class _ResponsesBody(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
def _capture_file() -> Path:
|
||||
path = os.environ.get(CAPTURE_FILE_ENV)
|
||||
assert path, f"{CAPTURE_FILE_ENV} is unset: the gateway must register callback_cost_capture.capture"
|
||||
return Path(path)
|
||||
|
||||
|
||||
def _captured_rows(path: Path, request_id: str) -> list[CapturedCost]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows = [CapturedCost.model_validate_json(line) for line in path.read_text(encoding="utf-8").splitlines() if line]
|
||||
return [row for row in rows if row.request_id == request_id]
|
||||
|
||||
|
||||
def _poll_captured(client: LoggingClient, path: Path, request_id: str) -> list[CapturedCost]:
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while True:
|
||||
rows = _captured_rows(path, request_id)
|
||||
if rows or time.monotonic() >= deadline:
|
||||
return rows
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
|
||||
|
||||
class TestCustomCallbackResponsesCost:
|
||||
@pytest.mark.covers("logging.custom_callback.success.logs_spend", exercised_on=["responses"])
|
||||
def test_sync_responses_callback_payload_cost_equals_spend_log_row(
|
||||
self, client: LoggingClient, resources: ResourceManager
|
||||
) -> None:
|
||||
capture_file = _capture_file()
|
||||
alias = f"e2e-callback-cost-{unique_marker()}"
|
||||
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
outcome = first_ok(
|
||||
client,
|
||||
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
|
||||
)
|
||||
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
|
||||
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
|
||||
|
||||
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
|
||||
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
|
||||
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
|
||||
)
|
||||
|
||||
captured = _poll_captured(client, capture_file, response_id)
|
||||
assert captured, f"CustomLogger saw no success event for request_id={response_id} in {capture_file}"
|
||||
for row in captured:
|
||||
assert row.response_cost is not None and math.isclose(row.response_cost, spend_row.spend, rel_tol=1e-9), (
|
||||
f"{row.hook} ({row.call_type}) saw standard_logging_object.response_cost={row.response_cost} "
|
||||
f"but SpendLogs billed {spend_row.spend} for request_id={response_id}"
|
||||
)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import LoggingClient, first_ok
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
SPEND_METRIC = "litellm_spend_metric"
|
||||
ALIAS_LABEL = "api_key_alias"
|
||||
|
||||
|
||||
class _ResponsesBody(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
def _callback_spend(exposition: str, alias: str) -> float | None:
|
||||
values = [
|
||||
sample.value
|
||||
for family in text_string_to_metric_families(exposition)
|
||||
for sample in family.samples
|
||||
if sample.name == f"{SPEND_METRIC}_total" and sample.labels.get(ALIAS_LABEL) == alias
|
||||
]
|
||||
return sum(values) if values else None
|
||||
|
||||
|
||||
def _poll_callback_spend(client: LoggingClient, alias: str) -> float | None:
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while True:
|
||||
spend = _callback_spend(client.scrape_metrics(), alias)
|
||||
if spend is not None or time.monotonic() >= deadline:
|
||||
return spend
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
|
||||
|
||||
class TestResponsesCallbackCost:
|
||||
@pytest.mark.covers("logging.prometheus.success.logs_spend", exercised_on=["responses"])
|
||||
def test_responses_callback_cost_equals_spend_log_row(
|
||||
self, client: LoggingClient, resources: ResourceManager
|
||||
) -> None:
|
||||
alias = f"e2e-responses-cost-{unique_marker()}"
|
||||
key = client.key_with_alias(alias, models=[CHEAP_OPENAI_MODEL])
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
outcome = first_ok(
|
||||
client,
|
||||
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {unique_marker()}"),
|
||||
)
|
||||
response_id = _ResponsesBody.model_validate(json.loads(outcome.body)).id
|
||||
assert response_id, f"/v1/responses returned no id: {outcome.body[:200]}"
|
||||
|
||||
spend_row = client.poll_proxy_spend_for_key(key, response_id=response_id)
|
||||
assert spend_row is not None and spend_row.spend is not None and spend_row.spend > 0, (
|
||||
f"no positive-spend row for request_id={response_id}, got {spend_row!r}"
|
||||
)
|
||||
|
||||
callback_spend = _poll_callback_spend(client, alias)
|
||||
assert callback_spend is not None, (
|
||||
f"{SPEND_METRIC} has no series for {ALIAS_LABEL}={alias} although the spend row billed {spend_row.spend}"
|
||||
)
|
||||
assert math.isclose(callback_spend, spend_row.spend, rel_tol=1e-9), (
|
||||
f"callback saw response_cost={callback_spend} but SpendLogs billed {spend_row.spend} "
|
||||
f"for request_id={response_id}"
|
||||
)
|
||||
|
|
@ -14,5 +14,4 @@ markers =
|
|||
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set
|
||||
mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set
|
||||
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
|
||||
callback_cost_capture: reads the JSONL written by the gateway's registered logging.callback_cost_capture.capture CustomLogger, so the gateway and pytest must share a filesystem; deselected unless E2E_CALLBACK_COST_CAPTURE_FILE is set
|
||||
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set
|
||||
|
|
|
|||
|
|
@ -221,6 +221,9 @@
|
|||
"other.observability.callbacks.credentials_stay_out_of_event_bodies",
|
||||
"other.observability.callbacks.concurrent_results_join_complete_events_and_rows"
|
||||
],
|
||||
"tests/integration/observability/test_responses_callback_cost.py::test_responses_success_callback_cost_is_priced_and_matches_spend_row": [
|
||||
"other.observability.callbacks.responses_success_cost_matches_spend_row"
|
||||
],
|
||||
"tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [
|
||||
"other.observability.guardrails.rewrite_reaches_correct_anthropic_positions"
|
||||
],
|
||||
|
|
|
|||
116
tests/integration/observability/test_responses_callback_cost.py
Normal file
116
tests/integration/observability/test_responses_callback_cost.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.callbacks.responses_success_cost_matches_spend_row")
|
||||
def test_responses_success_callback_cost_is_priced_and_matches_spend_row(gateway: Gateway, tmp_path: Path) -> None:
|
||||
marker: Final = "responses-cost-" + uuid.uuid4().hex
|
||||
secret: Final = "synthetic-provider-secret-" + marker
|
||||
sink_secret: Final = "synthetic-sink-secret-" + marker
|
||||
|
||||
def upstream(request: Request) -> Reply:
|
||||
assert request.target == "/v1/responses"
|
||||
assert request.headers["authorization"] == f"Bearer {secret}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["input"] == marker
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "resp_" + marker,
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_" + marker,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": marker, "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 11,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 15,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
def sink(request: Request) -> Reply:
|
||||
assert request.headers["authorization"] == f"Bearer {sink_secret}"
|
||||
return Reply()
|
||||
|
||||
with wire_server(upstream) as provider, wire_server(sink) as endpoint:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update({"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1})
|
||||
path: Final = tmp_path / "callbacks.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{
|
||||
"GENERIC_LOGGER_ENDPOINT": endpoint.url,
|
||||
"GENERIC_LOGGER_HEADERS": f"Authorization=Bearer {sink_secret}",
|
||||
},
|
||||
config=path,
|
||||
) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
api_base=provider.url + "/v1", api_key=secret, input_cost_per_token=0.001, output_cost_per_token=0.002
|
||||
)
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{"model": model, "input": marker, "metadata": {"tags": [marker]}},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(provider.drain()) == 1
|
||||
response_id: Final = response.json()["id"]
|
||||
batches = []
|
||||
|
||||
def delivered() -> tuple[dict, ...]:
|
||||
batches.extend(endpoint.drain())
|
||||
return tuple(
|
||||
event
|
||||
for batch in batches
|
||||
for event in json.loads(batch.body)
|
||||
if marker in event.get("request_tags", [])
|
||||
)
|
||||
|
||||
events: Final = eventually(delivered, lambda values: len(values) == 1, seconds=10)
|
||||
event: Final = events[0]
|
||||
assert event["status"] == "success"
|
||||
assert event["call_type"] in ("responses", "aresponses")
|
||||
assert event["prompt_tokens"] == 11 and event["completion_tokens"] == 4
|
||||
assert event["response_cost"] == pytest.approx(11 * 0.001 + 4 * 0.002)
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT spend, call_type FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
|
||||
(response_id,),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert float(rows[0]["spend"]) == pytest.approx(event["response_cost"])
|
||||
assert rows[0]["call_type"] == event["call_type"]
|
||||
Loading…
Add table
Reference in a new issue