mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
* test(e2e): read datadog log delivery back from the real datadog api (#33604) * test(e2e): read datadog log delivery back from the real datadog api * test(e2e): compare datadog-read cost with math.isclose, not bit-equality The response_cost now round-trips through DataDog's attribute indexing pipeline, whose float serialization is not guaranteed to preserve the exact bit pattern the proxy shipped. rel_tol=1e-9 (equal to 9 significant digits) still fails on any real cost discrepancy while tolerating representation drift. Addresses the Greptile P2 on this PR. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test(e2e): widen the duplicate-settle window to 30s for real DataDog Against the local sink one poll interval (5s) after the first hit was enough to catch a same-call duplicate, because both events arrived in the same flush batch. Against real DataDog, ingestion jitter can make one call's two events searchable tens of seconds apart, so a 5s settle could let the LIT-4447 duplicate slip past the exactly-one assertion. The reader now keeps re-reading for DD_SETTLE_SECONDS (default 30s, env-overridable via E2E_DD_SETTLE_SECONDS) after the first event appears, returning early only when a duplicate is already visible - more waiting cannot clear it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(e2e): point UI tests at dashboard service; register complexity router Stage gateway 404s /ui; the Next.js dashboard is litellm-ui:3000. Drive playwright against E2E_UI_BASE_URL and wait on login placeholders after client render. Register complexity-smart-router via /model/new when the proxy does not already list it so stage matches compose config * docs(e2e): clarify E2E_UI_BASE_URL should be ALB when ingress splits UI * docs(e2e): prefer single path-routing host for control plane and UI CONTROL_PLANE and UI already default to PROXY_BASE_URL; clarify that stage should set one ALB host rather than three endpoints * fix(e2e): always capture complexity router model_id for teardown Split /model/new from the data-plane wait so a propagation timeout still deletes the control-plane registration (greptile orphan-model concern) * fix(e2e): click exact Login button so SSO control is not matched Playwright strict mode matched both Login and Login with SSO * fix(router): score complexity by difficulty not request length The LLM classifier prompt treated short wording as SIMPLE, so probes like "Is P equal to NP?" stayed on the SIMPLE backend even though the classifier ran. Judge intellectual difficulty so short hard questions route higher * fix(e2e): open key edit via Key ID and wait for team models Key Alias text is not the row open control on the virtual keys table; KeyInfoView opens from the Key ID button in that row. Also wait for a real team model in the edit Models dropdown so we do not race the async availableModels fetch that only has All Team Models on first paint * fix(e2e): keep settled DD events on empty search; bump mcp for OSV Do not let a transient empty DataDog search wipe events already seen in the settle window (Greptile P1). Make the logs-search from window env-overridable via E2E_DD_SEARCH_FROM (Greptile P2). Prefer the mono Key ID button when opening key edit. Bump mcp 1.26.0 -> 1.28.1 so OSV clears the three high GHSA findings on the staging PR * revert: drop mcp lock bump from e2e staging PR OSV mcp upgrade is unrelated to the e2e fixes; leave the dep pin alone --------- Co-authored-by: yucheng-berri <yucheng@berri.ai> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
58 lines
1.8 KiB
Python
58 lines
1.8 KiB
Python
"""Fixtures for the logging e2e suite.
|
|
|
|
Missing proxy, provider keys, or integration credentials are hard failures.
|
|
Never pytest.skip from this suite for environment gaps.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from logging_client import LangfuseCreds, LoggingClient, build_logging_client, load_langfuse_creds
|
|
from datadog_reader import DdLogsReader, build_dd_logs_reader
|
|
from otel_client import OtelReader, build_otel_reader
|
|
|
|
|
|
def pytest_configure(config: pytest.Config) -> None:
|
|
config.addinivalue_line(
|
|
"markers",
|
|
"covers: registry cell a test covers, e.g. logging.langfuse.success.logs_spend",
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def client() -> LoggingClient:
|
|
"""The logging suite's client: holds the shared Gateway so `resources` /
|
|
`scoped_key` clean up keys and teams, and adds `/metrics` scraping plus
|
|
Langfuse read-back."""
|
|
return build_logging_client()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def otel_reader() -> OtelReader:
|
|
"""Read-back client for the compose stack's Jaeger trace destination."""
|
|
return build_otel_reader()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def dd_logs() -> DdLogsReader:
|
|
"""Read-back client for the real DataDog Logs Search API (keys from the
|
|
secret manager on the cluster, tests/e2e/.env locally)."""
|
|
return build_dd_logs_reader()
|
|
|
|
|
|
@pytest.fixture
|
|
def datadog_creds() -> None:
|
|
"""Require Datadog shipping credentials. Hard-fail when absent; never skip."""
|
|
if not (os.getenv("DD_API_KEY") and os.getenv("DD_SITE")):
|
|
pytest.fail(
|
|
"Datadog e2e requires DD_API_KEY and DD_SITE; missing credentials is a hard failure, not a skip"
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def langfuse_creds() -> LangfuseCreds:
|
|
"""Require real Langfuse cloud credentials for team callback + trace poll."""
|
|
return load_langfuse_creds()
|