mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
daf22ec871
* test(e2e): harden harness and tests against data-plane pod churn A stage autoscaler scale-down produced a 2s window of ALB 502s that killed six budget tests on their first management call, and a freshly scaled-up pod that had not run its 30s DB object sync yet failed two MCP tests and one prometheus cardinality test. Retry transient gateway errors (502/503/504, connection errors) once at the shared e2e_http dispatch seam, poll MCP server registration to the poll deadline instead of asserting a single-shot listing, anchor the MCP guardrail full-sync wait to the later of the guardrail and server writes, and turn the prometheus alias poll into a drive-and-scrape convergence loop that re-sends traffic for missing aliases and unions results across scrapes * test(e2e): drain request body in retry stub handler so keep-alive reuse cannot misparse leftovers as requests * revert(e2e): drop the transient-502 retry seam A raw 502 during a pod scale-down is what a real client sees, so the suite retrying past it hides an availability gap instead of flagging it. The gateway-side fix is graceful drain on the deployment; until then the failures are signal * test(e2e): cap per-alias driver re-drives in the prometheus cardinality poll Bounds worst-case provider spend to 4 completions per alias while scrapes keep polling to the deadline; counters persist on whichever pod served them, so the cap costs no convergence unless that pod dies * test(e2e): drop driver re-drives from the prometheus cardinality poll The per-key cardinality contract is process-local and counters persist on whichever pod served the driver call, so unioning aliases across free scrape polls converges without re-sending billable traffic. The residual gap, a pod dying inside the poll window, is deferred to direct per-pod scraping
74 lines
3.1 KiB
Python
74 lines
3.1 KiB
Python
"""Live e2e: Prometheus request metrics grow one series per virtual key.
|
|
|
|
The proxy exposes ``/metrics`` (prometheus is in the callbacks and
|
|
``require_auth_for_metrics_endpoint`` is off in the e2e config). The counter
|
|
``litellm_requests_metric_total`` carries an ``api_key_alias`` label, so driving
|
|
traffic through keys with distinct aliases must produce a distinct labeled series
|
|
per alias. This is the per-key cardinality contract: a regression that stops
|
|
stamping ``api_key_alias`` (or collapses every key onto one series) would drop
|
|
the aliases and fail here.
|
|
|
|
Scraping goes through ``transport.probe`` (raw text) and is parsed with
|
|
prometheus_client. ``/metrics`` is per-pod behind a round-robin LB and the metric
|
|
is eventually consistent (it increments on the success-logging callback), so the
|
|
poll unions the aliases seen across scrapes until the deadline: counters persist
|
|
on whichever pod served the driver call, so repeated scrapes converge without
|
|
re-sending any billable traffic.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
|
|
import pytest
|
|
from prometheus_client.parser import text_string_to_metric_families
|
|
|
|
from e2e_config import unique_marker
|
|
from lifecycle import ResourceManager
|
|
from logging_client import LoggingClient
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
DRIVER_MODEL = "gemini-2.5-flash"
|
|
REQUESTS_METRIC = "litellm_requests_metric_total"
|
|
ALIAS_LABEL = "api_key_alias"
|
|
DISTINCT_KEYS = 3
|
|
|
|
|
|
def _aliases_in_metric(exposition: str, metric: str, label: str) -> frozenset[str]:
|
|
"""The set of ``label`` values present on ``metric`` samples in a scrape."""
|
|
return frozenset(
|
|
sample.labels[label]
|
|
for family in text_string_to_metric_families(exposition)
|
|
for sample in family.samples
|
|
if sample.name == metric and label in sample.labels
|
|
)
|
|
|
|
|
|
class TestPrometheusPerKeyCardinality:
|
|
@pytest.mark.covers("logging.prometheus.success.exports_metric", exercised_on=[])
|
|
def test_distinct_key_aliases_produce_distinct_series(
|
|
self, client: LoggingClient, resources: ResourceManager
|
|
) -> None:
|
|
aliases = tuple(f"e2e-prom-{unique_marker()}" for _ in range(DISTINCT_KEYS))
|
|
for alias in aliases:
|
|
key = client.key_with_alias(alias, models=[DRIVER_MODEL])
|
|
resources.defer(lambda k=key: client.delete_key(k))
|
|
response = client.chat(key, DRIVER_MODEL, f"reply with one word {alias}")
|
|
assert response.model, f"driver call for {alias} returned no model: {response}"
|
|
|
|
wanted = frozenset(aliases)
|
|
deadline = time.monotonic() + client.proxy.poll_timeout
|
|
seen: frozenset[str] = frozenset()
|
|
while time.monotonic() < deadline:
|
|
seen = seen | _aliases_in_metric(client.scrape_metrics(), REQUESTS_METRIC, ALIAS_LABEL)
|
|
if wanted <= seen:
|
|
break
|
|
time.sleep(client.proxy.poll_interval)
|
|
|
|
missing = wanted - seen
|
|
assert not missing, (
|
|
f"{REQUESTS_METRIC} never exposed a per-key series for aliases {sorted(missing)} "
|
|
f"on any scraped pod within the deadline; "
|
|
f"each distinct {ALIAS_LABEL} must grow its own series"
|
|
)
|