mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
d93353bcb6
Adds litellm-rust/loadtest: a mock LLM (OpenAI chat, Anthropic messages, Bedrock invoke with event-stream framing) that takes its timing from the prompt, an open-loop load generator with workloads covering 10k to 1M token contexts, 6 minute non-streaming calls and 30 minute streams, a config and virtual key generator, a summarizer and a /proc sampler, plus Terraform for an AWS rig with load generators, gateway and mocks on separate machines The gateway sets TCP_NODELAY on accepted sockets, removing a ~40 ms stall per streamed response, and its Docker build takes LTO and CODEGEN_UNITS build args for faster iteration builds
24 lines
1.1 KiB
JSON
24 lines
1.1 KiB
JSON
{
|
|
"name": "500k to 1M token bodies at rising rates: request parsing CPU, memory and bandwidth",
|
|
"stages": [
|
|
{"duration_s": 300, "rps": 0.5},
|
|
{"duration_s": 300, "rps": 1},
|
|
{"duration_s": 300, "rps": 2},
|
|
{"duration_s": 300, "rps": 4},
|
|
{"duration_s": 300, "rps": 8}
|
|
],
|
|
"nonstream_max_s": 360,
|
|
"warmup_s": 30,
|
|
"keys": {"file": "out/keys.txt", "zipf_s": 0},
|
|
"models": {"chat": {"prefix": "gpt-6-astra-", "count": 10}, "messages": {"prefix": "claude-opus-5-5-", "count": 10}},
|
|
"classes": [
|
|
{"name": "huge-messages", "weight": 1, "endpoint": "messages", "stream_fraction": 0.5,
|
|
"context_tokens": {"dist": "uniform", "min": 500000, "max": 1000000},
|
|
"ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000},
|
|
"duration_s": {"dist": "uniform", "min": 5, "max": 60}},
|
|
{"name": "huge-chat", "weight": 1, "endpoint": "chat", "stream_fraction": 0.5,
|
|
"context_tokens": {"dist": "uniform", "min": 500000, "max": 900000},
|
|
"ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000},
|
|
"duration_s": {"dist": "uniform", "min": 5, "max": 60}}
|
|
]
|
|
}
|