diff --git a/litellm-rust/loadtest/README.md b/litellm-rust/loadtest/README.md index d9b0255a778..5b6b09d9d21 100644 --- a/litellm-rust/loadtest/README.md +++ b/litellm-rust/loadtest/README.md @@ -16,7 +16,7 @@ machines. | `s3stub` | In-memory path-style S3 (PUT, GET, ListObjectsV2) for the `s3_v2` logs of a full-feature run. Objects are stored compressed and listed from a sorted index, so millions of logs fit. | Workloads live in `workloads/`: `smoke`, `realistic-ramp`, `long-streams`, `large-context`, -`key-churn` and `soak`. `sustained-100`, `sustained-250`, `sustained-500` and `sustained-1000` hold one rate for +`key-churn` and `soak`. `sustained-10`, `-25`, `-50`, `-75`, `-100`, `-250`, `-500` and `-1000` hold one rate for 15 minutes with evenly spaced arrivals and smoke's request mix over every model group; `sustained-check` runs the same mix at 100, 500 and 1,000 rps for 2 minutes each, to check that the client, mock and stack hold 1,000 rps before the long runs. diff --git a/litellm-rust/loadtest/workloads/sustained-10.json b/litellm-rust/loadtest/workloads/sustained-10.json new file mode 100644 index 00000000000..b8612a9d554 --- /dev/null +++ b/litellm-rust/loadtest/workloads/sustained-10.json @@ -0,0 +1,71 @@ +{ + "name": "sustained 10 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [ + { + "duration_s": 900, + "rps": 10 + } + ], + "arrivals": "constant", + "nonstream_max_s": 360, + "warmup_s": 60, + "max_in_flight": 30000, + "keys": { + "file": "out/keys.txt", + "zipf_s": 0 + }, + "models": { + "chat": { + "prefix": "gpt-6-astra-", + "count": 10 + }, + "messages": { + "prefix": "claude-opus-5-5-", + "count": 10 + } + }, + "classes": [ + { + "name": "chat", + "weight": 1, + "endpoint": "chat", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + }, + { + "name": "messages", + "weight": 1, + "endpoint": "messages", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + } + ] +} diff --git a/litellm-rust/loadtest/workloads/sustained-25.json b/litellm-rust/loadtest/workloads/sustained-25.json new file mode 100644 index 00000000000..9f29f543bd5 --- /dev/null +++ b/litellm-rust/loadtest/workloads/sustained-25.json @@ -0,0 +1,71 @@ +{ + "name": "sustained 25 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [ + { + "duration_s": 900, + "rps": 25 + } + ], + "arrivals": "constant", + "nonstream_max_s": 360, + "warmup_s": 60, + "max_in_flight": 30000, + "keys": { + "file": "out/keys.txt", + "zipf_s": 0 + }, + "models": { + "chat": { + "prefix": "gpt-6-astra-", + "count": 10 + }, + "messages": { + "prefix": "claude-opus-5-5-", + "count": 10 + } + }, + "classes": [ + { + "name": "chat", + "weight": 1, + "endpoint": "chat", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + }, + { + "name": "messages", + "weight": 1, + "endpoint": "messages", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + } + ] +} diff --git a/litellm-rust/loadtest/workloads/sustained-50.json b/litellm-rust/loadtest/workloads/sustained-50.json new file mode 100644 index 00000000000..de1bd9e1091 --- /dev/null +++ b/litellm-rust/loadtest/workloads/sustained-50.json @@ -0,0 +1,71 @@ +{ + "name": "sustained 50 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [ + { + "duration_s": 900, + "rps": 50 + } + ], + "arrivals": "constant", + "nonstream_max_s": 360, + "warmup_s": 60, + "max_in_flight": 30000, + "keys": { + "file": "out/keys.txt", + "zipf_s": 0 + }, + "models": { + "chat": { + "prefix": "gpt-6-astra-", + "count": 10 + }, + "messages": { + "prefix": "claude-opus-5-5-", + "count": 10 + } + }, + "classes": [ + { + "name": "chat", + "weight": 1, + "endpoint": "chat", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + }, + { + "name": "messages", + "weight": 1, + "endpoint": "messages", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + } + ] +} diff --git a/litellm-rust/loadtest/workloads/sustained-75.json b/litellm-rust/loadtest/workloads/sustained-75.json new file mode 100644 index 00000000000..907ba5e2477 --- /dev/null +++ b/litellm-rust/loadtest/workloads/sustained-75.json @@ -0,0 +1,71 @@ +{ + "name": "sustained 75 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [ + { + "duration_s": 900, + "rps": 75 + } + ], + "arrivals": "constant", + "nonstream_max_s": 360, + "warmup_s": 60, + "max_in_flight": 30000, + "keys": { + "file": "out/keys.txt", + "zipf_s": 0 + }, + "models": { + "chat": { + "prefix": "gpt-6-astra-", + "count": 10 + }, + "messages": { + "prefix": "claude-opus-5-5-", + "count": 10 + } + }, + "classes": [ + { + "name": "chat", + "weight": 1, + "endpoint": "chat", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + }, + { + "name": "messages", + "weight": 1, + "endpoint": "messages", + "stream_fraction": 0.5, + "context_tokens": { + "dist": "loguniform", + "min": 10000, + "max": 50000 + }, + "ttft_ms": { + "dist": "uniform", + "min": 200, + "max": 800 + }, + "duration_s": { + "dist": "uniform", + "min": 1, + "max": 5 + } + } + ] +}