mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
7aef79b774
* test(e2e): send no-cache on every cacheable request body, opt in only where a hit is the assertion
The e2e proxy runs with the response cache on, so any test that re-sends an
identical chat, messages, responses, completions, embeddings or rerank body
reads back a redis copy of an earlier call instead of reaching the provider.
Five tests in the last week failed that way. Default cache: {"no-cache": true}
on those request models and pass cache=None only in the two tests whose
assertion is the cache hit itself.
* test(e2e): give image edits and OCR a 180s client timeout
Both routes wait on providers that can legitimately take longer than the
60s transport-wide request timeout (gpt-image edits, Azure Document
Intelligence), and a client-side read timeout there fails a green request.
post/upload now accept a per-call timeout like get already does; only those
two call sites use it.
* test(e2e): rerun once on network errors and upstream 5xx only
Assertion failures still fail on the first attempt; only an outcome whose
error string carries the e2e_http network kind or a 5xx status gets one
more try. Test Engine records every attempt, so the flake rate stays
visible while a single provider blip no longer reds the rc run.
* test(e2e): let the reseed burst survive one upstream failure and print why
The burst is the precondition, not the property: one 5xx among six
concurrent calls still leaves five workers racing the cold counter, which
is what the reseed assertion measures. Two or more failures still abort,
and the failing bodies are now in the message instead of only the status
codes.
* test(e2e): keep polling Jaeger through a transient query failure
poll_traces_for_call already waits up to POLL_TIMEOUT for spans to land,
but a single refused connection to the query API failed the test on the
spot. Jaeger restarted twice during today's gate runs (19:05 and 19:41
UTC, each under a minute) and took ten and three otel tests with it while
the same tests passed on the rc build minutes later. A network failure
now counts as not-yet inside the same deadline; if Jaeger is still
unreachable when the deadline passes the test fails with that error, and
any non-network failure still fails immediately.
214 lines
11 KiB
Python
214 lines
11 KiB
Python
"""Generic configuration for live e2e tests against a running LiteLLM proxy.
|
|
|
|
Shared by every e2e suite under tests/e2e/. Values come from the
|
|
environment so the same tests run against localhost or a deployed proxy.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
from fixture_mode import deterministic_marker, parse_fixture_mode
|
|
from provider_edge import provider_edge_api_base
|
|
|
|
# Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md).
|
|
# Compose injects them into the proxy container, but pytest on the host does not
|
|
# inherit that file unless we load it. override=False so a real shell export wins.
|
|
load_dotenv(Path(__file__).resolve().parent / ".env", override=False)
|
|
|
|
PROXY_BASE_URL = os.environ.get("LITELLM_PROXY_URL", "http://localhost:4000").rstrip("/")
|
|
MASTER_KEY = os.environ.get("LITELLM_MASTER_KEY", "sk-1234")
|
|
|
|
# Control-plane (management/admin) base URL. Defaults to PROXY_BASE_URL so a
|
|
# single path-routing host (stage ALB, compose monolith) works for both planes.
|
|
# Set LITELLM_CONTROL_PLANE_URL only when management is a different base than
|
|
# the LLM host and you are not going through an ingress that path-routes.
|
|
CONTROL_PLANE_BASE_URL = os.environ.get(
|
|
"LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL
|
|
).rstrip("/")
|
|
|
|
UI_USERNAME = os.environ.get("E2E_UI_USERNAME", "admin")
|
|
UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY)
|
|
|
|
# Dashboard base for playwright. Defaults to PROXY_BASE_URL so one ALB/monolith
|
|
# host covers /ui as well. Override E2E_UI_BASE_URL only if the UI is elsewhere.
|
|
UI_BASE_URL = os.environ.get("E2E_UI_BASE_URL", PROXY_BASE_URL).rstrip("/")
|
|
|
|
CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5")
|
|
CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5")
|
|
|
|
LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp")
|
|
LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "")
|
|
|
|
# Jaeger query API of the compose stack's OTEL trace destination (the `jaeger`
|
|
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
|
|
# read exported spans back through it.
|
|
OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/")
|
|
|
|
# Real-DataDog read-back (no local sink - destination fakes cannot be deployed
|
|
# on the cluster): the proxy delivers with DD_API_KEY as in production, and the
|
|
# tests read ingested events back through the DataDog Logs Search API, which
|
|
# additionally needs an application key. On the cluster the secret manager
|
|
# injects both; locally tests/e2e/.env provides them.
|
|
DD_SITE = os.environ.get("DD_SITE", "datadoghq.com").strip()
|
|
DD_API_KEY = os.environ.get("DD_API_KEY", "").strip()
|
|
DD_APP_KEY = os.environ.get("DD_APP_KEY", "").strip()
|
|
|
|
# After the first event is searchable, keep watching this long for a late
|
|
# duplicate before the exactly-one assertion: real-DataDog ingestion jitter can
|
|
# make one call's two events searchable tens of seconds apart, and a duplicate
|
|
# that surfaces late IS the bug (LIT-4447), so one poll interval is not enough.
|
|
DD_SETTLE_SECONDS = float(os.environ.get("E2E_DD_SETTLE_SECONDS", "30"))
|
|
# DataDog Logs Search `from` window (relative to now). Wide enough for a suite
|
|
# run plus ingestion lag; override if a long CI queue needs a wider lookback.
|
|
DD_SEARCH_FROM = os.environ.get("E2E_DD_SEARCH_FROM", "now-30m").strip() or "now-30m"
|
|
# The Logs Search API budget is tight - 2 requests per 10s org-wide
|
|
# (x-ratelimit-name logs_public_search_api) - so read-backs pace their search
|
|
# calls at this interval instead of POLL_INTERVAL, and back off when a 429
|
|
# still slips through (the budget is shared with anything else searching).
|
|
DD_SEARCH_INTERVAL = float(os.environ.get("E2E_DD_SEARCH_INTERVAL", "10"))
|
|
|
|
# Writes on the proxy are eventually consistent (e.g. spend rows flush on
|
|
# proxy_batch_write_at, ~60s). Read-backs poll to this deadline, never sleep-once.
|
|
POLL_TIMEOUT = float(os.environ.get("E2E_POLL_TIMEOUT", "120"))
|
|
POLL_INTERVAL = float(os.environ.get("E2E_POLL_INTERVAL", "5"))
|
|
REQUEST_TIMEOUT = float(os.environ.get("E2E_REQUEST_TIMEOUT", "60"))
|
|
SLOW_PROVIDER_TIMEOUT_SECONDS = float(os.environ.get("E2E_SLOW_PROVIDER_TIMEOUT", "180"))
|
|
|
|
# How long a control-plane write (/model/new, /guardrails, /v1/agents) may take to
|
|
# reach EVERY replica. Distinct from POLL_TIMEOUT, which is sized for spend-row
|
|
# flush; this one is sized for the proxy's config reload
|
|
# (`proxy_config_reload_interval_seconds`, 30s by default and 7s on the e2e stack)
|
|
# plus margin.
|
|
#
|
|
# The barriers below wait this out instead of returning on first sight, because a
|
|
# single successful read only proves ONE replica converged: every request opens a
|
|
# fresh connection, so a load-balanced Service routes each one independently and
|
|
# the next call re-rolls. See ProxyClient._await_model_servable.
|
|
PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15"))
|
|
|
|
EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes")
|
|
|
|
# Record/replay fixture selection (see fixture_mode.py and provider_edge.py).
|
|
# The raw mode value is parsed and validated there; "live" (the default, also
|
|
# for empty values) means the harness behaves exactly as before this knob
|
|
# existed.
|
|
FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live")
|
|
FIXTURE_DIR = Path(
|
|
os.environ.get("E2E_FIXTURE_DIR", "").strip()
|
|
or str(Path(__file__).resolve().parent / ".fixtures")
|
|
)
|
|
|
|
# Where the provider-edge server binds, and the host name edge api_base URLs
|
|
# advertise to the proxy. They differ when the proxy runs in a container and
|
|
# reaches the pytest host via a gateway name like host.docker.internal.
|
|
PROVIDER_EDGE_BIND_HOST = os.environ.get("E2E_PROVIDER_EDGE_BIND_HOST", "").strip() or "127.0.0.1"
|
|
PROVIDER_EDGE_ADVERTISE_HOST = (
|
|
os.environ.get("E2E_PROVIDER_EDGE_ADVERTISE_HOST", "").strip() or PROVIDER_EDGE_BIND_HOST
|
|
)
|
|
|
|
# Deliberately modest concurrency. The suite shares its proxy with every other
|
|
# suite in the run, and 750 users at spawn rate 50 saturated the request path hard
|
|
# enough to distort latency-sensitive neighbours (and to spend real provider money
|
|
# fast).
|
|
LOAD_USERS = int(os.environ.get("E2E_LOAD_USERS", "200"))
|
|
LOAD_SPAWN_RATE = float(os.environ.get("E2E_LOAD_SPAWN_RATE", "20"))
|
|
LOAD_DURATION_SECONDS = float(os.environ.get("E2E_LOAD_DURATION_SECONDS", "60"))
|
|
LOAD_MAX_FAILURE_RATIO = float(os.environ.get("E2E_LOAD_MAX_FAILURE_RATIO", "0.01"))
|
|
|
|
# The throughput floor is derived per replica instead of being an absolute fleet
|
|
# number, so the verdict does not depend on how many replicas happen to be warm.
|
|
# One closed-loop user only ever occupies one replica at a time, so a short serial
|
|
# pass measures a single replica's request path: its throughput is 1/latency, and
|
|
# the concurrent phase then has to reach at least that much no matter how large
|
|
# the fleet is. An absolute floor instead asserted replicas x per-replica rate,
|
|
# which reactive autoscaling decides rather than the request path.
|
|
LOAD_BASELINE_SECONDS = float(os.environ.get("E2E_LOAD_BASELINE_SECONDS", "15"))
|
|
LOAD_MAX_SERIAL_LATENCY_SECONDS = float(os.environ.get("E2E_LOAD_MAX_SERIAL_LATENCY_SECONDS", "0.5"))
|
|
LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY_EFFICIENCY", "0.8"))
|
|
|
|
WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
|
|
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
|
|
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
|
|
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
|
|
ANOMALY_MAX_ERROR_RATIO = float(os.environ.get("E2E_ANOMALY_MAX_ERROR_RATIO", "0.05"))
|
|
ANOMALY_MIN_WARM_CACHE_READ_SHARE = float(
|
|
os.environ.get("E2E_ANOMALY_MIN_WARM_CACHE_READ_SHARE", "0.65")
|
|
)
|
|
ANOMALY_MAX_P95_TURN_SECONDS = float(
|
|
os.environ.get("E2E_ANOMALY_MAX_P95_TURN_SECONDS", "30")
|
|
)
|
|
ANOMALY_MAX_KEY_SPEND_USD = float(
|
|
os.environ.get("E2E_ANOMALY_MAX_KEY_SPEND_USD", "0.60")
|
|
)
|
|
ANOMALY_SPEND_SETTLE_SECONDS = float(
|
|
os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75")
|
|
)
|
|
|
|
|
|
def ws_base_url() -> str:
|
|
"""PROXY_BASE_URL with its scheme swapped for the websocket one, so a suite
|
|
opening a socket points at the same proxy every HTTP suite uses."""
|
|
for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
|
|
if PROXY_BASE_URL.startswith(scheme):
|
|
return ws_scheme + PROXY_BASE_URL[len(scheme) :]
|
|
return PROXY_BASE_URL
|
|
|
|
|
|
def datadog_mcp_url(*, toolsets: str = "core") -> str:
|
|
"""Regional Datadog remote MCP endpoint for this process's DD_SITE.
|
|
|
|
US1 is mcp.datadoghq.com; every other site is mcp.<site> (e.g. us5 ->
|
|
mcp.us5.datadoghq.com). A fixed mcp.datadoghq.com URL 403s when the keys
|
|
belong to a non-US1 org.
|
|
"""
|
|
site = (
|
|
os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com"
|
|
).strip().removeprefix("https://").removeprefix("http://").rstrip("/")
|
|
if site.startswith("app."):
|
|
site = site[len("app.") :]
|
|
host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}"
|
|
base = f"https://{host}/v1/mcp"
|
|
return f"{base}?toolsets={toolsets}" if toolsets else base
|
|
|
|
|
|
def provider_edge_base(mount: str) -> str | None:
|
|
"""The api_base an edge-wired deployment should register with, using this
|
|
process's fixture-mode and edge-host configuration: None in live mode, the
|
|
shared edge server's mount URL in record and replay."""
|
|
return provider_edge_api_base(
|
|
mount,
|
|
mode_raw=FIXTURE_MODE_RAW,
|
|
bundle_dir=FIXTURE_DIR,
|
|
bind_host=PROVIDER_EDGE_BIND_HOST,
|
|
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
|
|
forward_timeout=REQUEST_TIMEOUT,
|
|
)
|
|
|
|
|
|
def unique_marker() -> str:
|
|
"""A short unique token per call/run, so concurrent runs and the shared
|
|
response cache never collide on prompts, tags, or customer ids. In record
|
|
and replay modes the token is deterministic per test instead, so a replay
|
|
run regenerates the exact requests the record run sent."""
|
|
if parse_fixture_mode(FIXTURE_MODE_RAW) in ("record", "replay"):
|
|
return deterministic_marker()
|
|
return uuid.uuid4().hex[:12]
|
|
|
|
|
|
def settle_propagation(written_at: float) -> None:
|
|
"""Block until PROPAGATION_TIMEOUT has elapsed since `written_at`, a
|
|
`time.monotonic()` stamp taken the moment a control-plane write returned.
|
|
|
|
Callers that already polled for the object still need this: the poll proves one
|
|
replica has it, not all of them. Waiting out the config-reload budget is what
|
|
makes the object safe to use on whichever replica the next request lands on.
|
|
"""
|
|
remaining = PROPAGATION_TIMEOUT - (time.monotonic() - written_at)
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|