scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 206

feat(loadtest): stream-1m workload, all-streamed near-1M-token bodies at rising rates

Browse source
This commit is contained in:
narmstrong-berry 2026-11-04 02:25:20 +00:00
parent 5990e10ff8
commit 5816ceabdf
No known key found for this signature in database
1 changed files with 24 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm-rust/loadtest/workloads/stream-1m.json Normal file
View file

@ -0,0 +1,24 @@
{
"name": "1M-token streams at rising rates (0.5 to 8 rps, 3 minutes each): parsing CPU, memory per open stream and first-byte time at the largest context",
"stages": [
{"duration_s": 180, "rps": 0.5},
{"duration_s": 180, "rps": 1},
{"duration_s": 180, "rps": 2},
{"duration_s": 180, "rps": 4},
{"duration_s": 180, "rps": 8}
],
"nonstream_max_s": 360,
"warmup_s": 30,
"keys": {"file": "out/keys.txt", "zipf_s": 0},
"models": {"chat": {"prefix": "gpt-6-astra-", "count": 10}, "messages": {"prefix": "claude-opus-5-5-", "count": 10}},
"classes": [
{"name": "stream-1m-messages", "weight": 1, "endpoint": "messages", "stream_fraction": 1,
"context_tokens": {"dist": "uniform", "min": 950000, "max": 1000000},
"ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000},
"duration_s": {"dist": "uniform", "min": 5, "max": 60}},
{"name": "stream-1m-chat", "weight": 1, "endpoint": "chat", "stream_fraction": 1,
"context_tokens": {"dist": "uniform", "min": 950000, "max": 1000000},
"ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000},
"duration_s": {"dist": "uniform", "min": 5, "max": 60}}
]
}