mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
66980bbb87
Pad the locust payload to tens of KB so per-request bookkeeping cost scales with body size instead of hiding behind a 40-byte prompt. Turn on use_redis_transaction_buffer in the chaos config and JSON_LOGS in the workflow so the spend buffer, pod lock, and JSON-encoded breaker tracebacks are all part of the measured chaos cost. Add a log-bytes-per-request budget alongside latency, RSS, and CPU, reading the proxy's log file size at each phase split; its ceiling is uncalibrated since no chaos run has measured it yet. Co-Authored-By: Claude Code <noreply@anthropic.com>
52 lines
1.8 KiB
Python
52 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import random
|
|
import uuid
|
|
from itertools import cycle
|
|
from typing import Final
|
|
|
|
from locust import FastHttpUser, constant, task
|
|
|
|
_MODEL: Final = os.environ["LOAD_MODEL"]
|
|
_API_KEYS: Final = tuple(os.environ["LOAD_API_KEYS"].split(","))
|
|
_NEXT_ENDPOINT: Final = cycle(os.environ["LOAD_ENDPOINTS"].split(","))
|
|
_FILLER: Final = "x" * 40_000
|
|
|
|
|
|
def _payload() -> dict[str, object]:
|
|
"""A prompt no other request sent, so the response cache never answers for the deployment.
|
|
|
|
Both endpoints take the same body: /v1/messages requires max_tokens, which /chat/completions
|
|
also accepts, so one payload serves the whole round robin. Padded to tens of KB so a
|
|
per-request bookkeeping cost that scales with body size (string formatting, hashing) shows
|
|
up in the CPU and log-size budgets instead of hiding behind a 40-byte prompt.
|
|
"""
|
|
return {
|
|
"model": _MODEL,
|
|
"messages": [{"role": "user", "content": f"load test ping {uuid.uuid4().hex} {_FILLER}"}],
|
|
"max_tokens": 16,
|
|
}
|
|
|
|
|
|
class GatewayUser(FastHttpUser):
|
|
"""One simulated user, pinned to one endpoint for its lifetime.
|
|
|
|
Endpoints are handed out round robin as users spawn, so a run spreads evenly over them
|
|
while each user's traffic stays on a single route, the way a real client behaves.
|
|
"""
|
|
|
|
wait_time = constant(0)
|
|
|
|
def on_start(self) -> None:
|
|
self.headers = {"Authorization": f"Bearer {random.choice(_API_KEYS)}"}
|
|
self.endpoint = next(_NEXT_ENDPOINT)
|
|
|
|
@task
|
|
def call(self) -> None:
|
|
self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any
|
|
self.endpoint,
|
|
json=_payload(),
|
|
headers=self.headers,
|
|
name=self.endpoint,
|
|
)
|