mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
a2614b1239
CodSpeed benchmarks the SDK with no IO, so it can't catch regressions that only appear under real concurrent load through the full proxy stack (auth, routing, logging, spend, Postgres, Redis). This adds a Locust load test under tests/e2e/load that drives concurrent POST /chat/completions traffic against a mock deployment (litellm_params.mock_response), so the measured throughput reflects proxy overhead rather than a provider's latency, and asserts an aggregate RPS SLO with a failure-ratio guard. The test is marked load and the parent conftest sorts load-marked items last so it never perturbs latency-sensitive suites. Covers reliability.perf.throughput.under_slo.
27 lines
709 B
Python
27 lines
709 B
Python
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from locust import FastHttpUser, constant, task
|
|
|
|
_MODEL = os.environ["LOAD_MODEL"]
|
|
_HEADERS = {"Authorization": f"Bearer {os.environ['LOAD_API_KEY']}"}
|
|
_PAYLOAD = {
|
|
"model": _MODEL,
|
|
"messages": [{"role": "user", "content": "load test ping"}],
|
|
"temperature": 0,
|
|
"max_tokens": 16,
|
|
}
|
|
|
|
|
|
class ChatUser(FastHttpUser):
|
|
wait_time = constant(0)
|
|
|
|
@task
|
|
def chat(self) -> None:
|
|
self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any
|
|
"/chat/completions",
|
|
json=_PAYLOAD,
|
|
headers=_HEADERS,
|
|
name="/chat/completions",
|
|
)
|