mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
edc30ea515
* test(e2e): datadog log delivery for successful chat, messages, and responses Covers logging.datadog.success.exports_metric on all three routes: one successful non-streaming call must reach the DataDog logs intake as exactly one log event whose StandardLoggingPayload message carries the model group, real token counts, and a response cost equal to the x-litellm-response-cost header of the same response. Delivery is judged at the intake: the compose stack gains a dd-sink service recording every batch the datadog callback ships via the DD_BASE_URL testing override, and a typed reader replays it. Writing these caught a live product bug: /v1/messages double-logs every success (two byte-identical events per call), filed as LIT-4447; the messages test tolerates byte-identical duplicates of the one event until it lands, while a second differing event still fails * test(e2e): address review findings on the datadog delivery suite Consolidates the fresh-key first_ok helper into logging_client now that the otel PR it mirrored has merged (both test files use the shared copy), moves intake batch parsing into a helper so no path can leave the batch unbound, and gives the sink's /health endpoint a truthful text/plain content type * test(e2e): tolerate same-logical-event duplicates by call id, not byte identity A clean LIT-4447 repro showed the duplicated payload is built twice and can mint a fresh synthetic completion id per emission, arriving as two separate intake POSTs with the same litellm_call_id and identical substantive fields. Byte-identity was therefore a flaky criterion; duplicates now qualify only when they share the call id, call type, model group, tokens, and cost, and a second differing event still fails * test(e2e): assert the scenario strictly; the messages test is the LIT-4447 regression pin Per review direction the tests now assert exactly what the scenario promises: exactly one DataDog log event per successful call, on every route. The /v1/messages test therefore fails on current code against the known double-log (LIT-4447) and is its regression pin; it goes green when the fix lands. The duplicate-tolerance machinery is removed * Simplify docstrings for DataDog log tests Removed redundant phrasing about cost cross-checking in docstrings. * Update test_datadog_log_e2e.py
57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
"""Fixtures for the logging e2e suite.
|
|
|
|
Missing proxy, provider keys, or integration credentials are hard failures.
|
|
Never pytest.skip from this suite for environment gaps.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds
|
|
from datadog_sink import DdSinkReader, build_dd_sink_reader
|
|
from otel_client import OtelReader, build_otel_reader
|
|
|
|
|
|
def pytest_configure(config: pytest.Config) -> None:
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"covers: registry cell a test covers, e.g. logging.langfuse.success.logs_spend",
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def client() -> LoggingClient:
|
|
"""The logging suite's client: holds the shared Gateway so `resources` /
|
|
`scoped_key` clean up keys and teams, and adds `/metrics` scraping plus
|
|
Langfuse read-back."""
|
|
return build_logging_client()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def otel_reader() -> OtelReader:
|
|
"""Read-back client for the compose stack's Jaeger trace destination."""
|
|
return build_otel_reader()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def dd_sink() -> DdSinkReader:
|
|
"""Read-back client for the compose stack's DataDog logs-intake sink."""
|
|
return build_dd_sink_reader()
|
|
|
|
|
|
@pytest.fixture
|
|
def datadog_creds() -> None:
|
|
"""Require Datadog shipping credentials. Hard-fail when absent; never skip."""
|
|
if not (os.getenv("DD_API_KEY") and os.getenv("DD_SITE")):
|
|
pytest.fail(
|
|
"Datadog e2e requires DD_API_KEY and DD_SITE; missing credentials is a hard failure, not a skip"
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def langfuse_creds() -> LangfuseCreds:
|
|
"""Require real Langfuse cloud credentials for team callback + trace poll."""
|
|
return load_langfuse_creds()
|