mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
31c1ffc5a4
* fix(e2e): define SpendTagsResponse/TagSpend so spend suite collects spend_tracking/spend_e2e_client.py imported SpendTagsResponse and TagSpend from models, but neither was ever defined, so importing the client raised ImportError and pytest aborted collection for the whole e2e session. The tag-spend tests had never run. Model /spend/tags as it actually answers: a bare array of per-tag aggregates, so SpendTagsResponse is a RootModel[list[TagSpend]] like the existing SpendLogs. spend_by_tags read a nonexistent spend_per_tag field that also wouldn't match the array shape; it now reads .root, matching how spend_logs consumes its RootModel. * test(e2e): close coverage gaps across chat/responses, provider features, batches, prometheus, and langfuse eviction Adds regression nets and gap-surfacing tests: A1 (llm_translation/test_deepseek_reasoning_e2e.py): control case proves the DeepSeek reasoner returns reasoning_content; two xfail(strict) cases document that reasoning_effort='none' and thinking type='disabled' are silently dropped (LIT-3686 / GH #27453) A2 (llm_translation/test_chat_completions_regression_e2e.py and test_responses_e2e.py): parametrized regression net asserting real completion content, not just a 200, across the configured providers for /chat/completions and /responses (GH #28991) A3 (llm_translation/test_provider_features_e2e.py): asserts service_tier is honored and prompt-cache read tokens grow on a repeated cacheable prefix A4 (batches/test_batches_e2e.py): mints a rate-limited key so the batch pre-call rate limiter runs, then asserts no unattributed spend row is left behind by the internal input-file retrieval (LIT-3266) A5 (logging/test_prometheus_cardinality_e2e.py): drives one chat per distinct key_alias and asserts each alias gets its own labeled series on /metrics A6 (test_litellm/.../specialty_caches/test_dynamic_logging_cache.py): xfail(strict) regression proving eviction must not close an httpx client still held by an in-flight caller (LIT-3221 / GH #13034) Extends tests/e2e/models.py with the typed request and response fields these tests read (reasoning_effort, thinking, service_tier, key_alias, cache usage fields, spend-log api_key) Co-authored-by: Cursor <cursoragent@cursor.com> * test(e2e): drop unused litellm-regression-tests submodule The e2e suite migrated the regression cases into this repo; nothing imports the submodule at runtime (only a provenance comment references it), so the .gitmodules entry and gitlink pointing at a personal repo would just make upstream CI init a submodule it never uses. Remove both to keep the change test-only. * test(e2e): drop A6 langfuse-eviction xfail; keep PR to live e2e coverage The dynamic_logging_cache strict-xfail documented an unfixed shared-httpx-client close-on-eviction bug (LIT-3221 / GH #13034). That is a non-trivial fix (thread cleanup vs shared client teardown) and belongs in its own PR, not this e2e coverage PR, so revert the file to its base state. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
70 lines
2.8 KiB
Python
70 lines
2.8 KiB
Python
"""Live e2e: Prometheus request metrics grow one series per virtual key.
|
|
|
|
The proxy exposes ``/metrics`` (prometheus is in the callbacks and
|
|
``require_auth_for_metrics_endpoint`` is off in the e2e config). The counter
|
|
``litellm_requests_metric_total`` carries an ``api_key_alias`` label, so driving
|
|
traffic through keys with distinct aliases must produce a distinct labeled series
|
|
per alias. This is the per-key cardinality contract: a regression that stops
|
|
stamping ``api_key_alias`` (or collapses every key onto one series) would drop
|
|
the aliases and fail here.
|
|
|
|
Scraping goes through ``transport.probe`` (raw text) and is parsed with
|
|
prometheus_client; the metric is eventually consistent (it increments on the
|
|
success-logging callback), so the scrape polls to a deadline.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
|
|
import pytest
|
|
from prometheus_client.parser import text_string_to_metric_families
|
|
|
|
from e2e_config import unique_marker
|
|
from lifecycle import ResourceManager
|
|
from logging_client import LoggingClient
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
DRIVER_MODEL = "gemini-2.5-flash"
|
|
REQUESTS_METRIC = "litellm_requests_metric_total"
|
|
ALIAS_LABEL = "api_key_alias"
|
|
DISTINCT_KEYS = 3
|
|
|
|
|
|
def _aliases_in_metric(exposition: str, metric: str, label: str) -> frozenset[str]:
|
|
"""The set of ``label`` values present on ``metric`` samples in a scrape."""
|
|
return frozenset(
|
|
sample.labels[label]
|
|
for family in text_string_to_metric_families(exposition)
|
|
for sample in family.samples
|
|
if sample.name == metric and label in sample.labels
|
|
)
|
|
|
|
|
|
class TestPrometheusPerKeyCardinality:
|
|
@pytest.mark.covers("logging.prometheus.success.exports_metric", exercised_on=[])
|
|
def test_distinct_key_aliases_produce_distinct_series(
|
|
self, client: LoggingClient, resources: ResourceManager
|
|
) -> None:
|
|
aliases = tuple(f"e2e-prom-{unique_marker()}" for _ in range(DISTINCT_KEYS))
|
|
for alias in aliases:
|
|
key = client.key_with_alias(alias, models=[DRIVER_MODEL])
|
|
resources.defer(lambda k=key: client.delete_key(k))
|
|
response = client.chat(key, DRIVER_MODEL, f"reply with one word {alias}")
|
|
assert response.model, f"driver call for {alias} returned no model: {response}"
|
|
|
|
wanted = frozenset(aliases)
|
|
deadline = time.monotonic() + client.gateway.poll_timeout
|
|
seen: frozenset[str] = frozenset()
|
|
while time.monotonic() < deadline:
|
|
seen = _aliases_in_metric(client.scrape_metrics(), REQUESTS_METRIC, ALIAS_LABEL)
|
|
if wanted <= seen:
|
|
break
|
|
time.sleep(client.gateway.poll_interval)
|
|
|
|
missing = wanted - seen
|
|
assert not missing, (
|
|
f"{REQUESTS_METRIC} is missing a per-key series for aliases {sorted(missing)}; "
|
|
f"each distinct {ALIAS_LABEL} must grow its own series"
|
|
)
|