scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

feat(loadtest): sustained 10, 25, 50 and 75 rps workloads

Browse source
This commit is contained in:
narmstrong-berry 2026-11-03 17:52:59 +00:00
parent 23a2e861a0
commit c8edea05f0
No known key found for this signature in database
5 changed files with 285 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm-rust/loadtest/README.md
View file

@ -16,7 +16,7 @@ machines.
| `s3stub` | In-memory path-style S3 (PUT, GET, ListObjectsV2) for the `s3_v2` logs of a full-feature run. Objects are stored compressed and listed from a sorted index, so millions of logs fit. |
Workloads live in `workloads/`: `smoke`, `realistic-ramp`, `long-streams`, `large-context`,
`key-churn` and `soak`. `sustained-100`, `sustained-250`, `sustained-500` and `sustained-1000` hold one rate for
`key-churn` and `soak`. `sustained-10`, `-25`, `-50`, `-75`, `-100`, `-250`, `-500` and `-1000` hold one rate for
15 minutes with evenly spaced arrivals and smoke's request mix over every model group;
`sustained-check` runs the same mix at 100, 500 and 1,000 rps for 2 minutes each, to check that the
client, mock and stack hold 1,000 rps before the long runs.

71
litellm-rust/loadtest/workloads/sustained-10.json Normal file
View file

@ -0,0 +1,71 @@
{
"name": "sustained 10 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group",
"stages": [
{
"duration_s": 900,
"rps": 10
}
],
"arrivals": "constant",
"nonstream_max_s": 360,
"warmup_s": 60,
"max_in_flight": 30000,
"keys": {
"file": "out/keys.txt",
"zipf_s": 0
},
"models": {
"chat": {
"prefix": "gpt-6-astra-",
"count": 10
},
"messages": {
"prefix": "claude-opus-5-5-",
"count": 10
}
},
"classes": [
{
"name": "chat",
"weight": 1,
"endpoint": "chat",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
},
{
"name": "messages",
"weight": 1,
"endpoint": "messages",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
}
]
}

71
litellm-rust/loadtest/workloads/sustained-25.json Normal file
View file

@ -0,0 +1,71 @@
{
"name": "sustained 25 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group",
"stages": [
{
"duration_s": 900,
"rps": 25
}
],
"arrivals": "constant",
"nonstream_max_s": 360,
"warmup_s": 60,
"max_in_flight": 30000,
"keys": {
"file": "out/keys.txt",
"zipf_s": 0
},
"models": {
"chat": {
"prefix": "gpt-6-astra-",
"count": 10
},
"messages": {
"prefix": "claude-opus-5-5-",
"count": 10
}
},
"classes": [
{
"name": "chat",
"weight": 1,
"endpoint": "chat",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
},
{
"name": "messages",
"weight": 1,
"endpoint": "messages",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
}
]
}

71
litellm-rust/loadtest/workloads/sustained-50.json Normal file
View file

@ -0,0 +1,71 @@
{
"name": "sustained 50 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group",
"stages": [
{
"duration_s": 900,
"rps": 50
}
],
"arrivals": "constant",
"nonstream_max_s": 360,
"warmup_s": 60,
"max_in_flight": 30000,
"keys": {
"file": "out/keys.txt",
"zipf_s": 0
},
"models": {
"chat": {
"prefix": "gpt-6-astra-",
"count": 10
},
"messages": {
"prefix": "claude-opus-5-5-",
"count": 10
}
},
"classes": [
{
"name": "chat",
"weight": 1,
"endpoint": "chat",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
},
{
"name": "messages",
"weight": 1,
"endpoint": "messages",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
}
]
}

71
litellm-rust/loadtest/workloads/sustained-75.json Normal file
View file

@ -0,0 +1,71 @@
{
"name": "sustained 75 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group",
"stages": [
{
"duration_s": 900,
"rps": 75
}
],
"arrivals": "constant",
"nonstream_max_s": 360,
"warmup_s": 60,
"max_in_flight": 30000,
"keys": {
"file": "out/keys.txt",
"zipf_s": 0
},
"models": {
"chat": {
"prefix": "gpt-6-astra-",
"count": 10
},
"messages": {
"prefix": "claude-opus-5-5-",
"count": 10
}
},
"classes": [
{
"name": "chat",
"weight": 1,
"endpoint": "chat",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
},
{
"name": "messages",
"weight": 1,
"endpoint": "messages",
"stream_fraction": 0.5,
"context_tokens": {
"dist": "loguniform",
"min": 10000,
"max": 50000
},
"ttft_ms": {
"dist": "uniform",
"min": 200,
"max": 800
},
"duration_s": {
"dist": "uniform",
"min": 1,
"max": 5
}
}
]
}