From d9b22d61c2a04ed8cb0301f9a9078bfc2a8584b7 Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Wed, 4 Nov 2026 02:25:36 +0000 Subject: [PATCH] fix(loadtest): stream-1m chat bodies stay under the chat model's 922k window --- litellm-rust/loadtest/README.md | 2 +- litellm-rust/loadtest/workloads/stream-1m.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm-rust/loadtest/README.md b/litellm-rust/loadtest/README.md index 25394c53ed3..1261378c494 100644 --- a/litellm-rust/loadtest/README.md +++ b/litellm-rust/loadtest/README.md @@ -15,7 +15,7 @@ machines. | `chatparity` | Sends the same chat completions requests through the Python proxy and the gateway, both pointed at its recording mock provider, and reports every difference in what the provider was sent and what the client got. See [Chat parity](#chat-parity). | | `s3stub` | In-memory path-style S3 (PUT, GET, ListObjectsV2) for the `s3_v2` logs of a full-feature run. Objects are stored compressed and listed from a sorted index, so millions of logs fit. | -Workloads live in `workloads/`: `smoke`, `realistic-ramp`, `long-streams`, `large-context`, +Workloads live in `workloads/`: `smoke`, `realistic-ramp`, `long-streams`, `large-context`, `stream-1m` (all streamed, near-1M-token bodies), `key-churn` and `soak`. `sustained-100`, `sustained-250`, `sustained-500` and `sustained-1000` hold one rate for 15 minutes with evenly spaced arrivals and smoke's request mix over every model group (`sweep-10`, `-25`, `-50`, `-75` and `-100` the same for 5 minutes each, for a low-rate sweep); diff --git a/litellm-rust/loadtest/workloads/stream-1m.json b/litellm-rust/loadtest/workloads/stream-1m.json index 0efdd348a8c..08a837462c7 100644 --- a/litellm-rust/loadtest/workloads/stream-1m.json +++ b/litellm-rust/loadtest/workloads/stream-1m.json @@ -1,5 +1,5 @@ { - "name": "1M-token streams at rising rates (0.5 to 8 rps, 3 minutes each): parsing CPU, memory per open stream and first-byte time at the largest context", + "name": "1M-token streams (chat up to 920k, under its 922k window) at rising rates (0.5 to 8 rps, 3 minutes each): parsing CPU, memory per open stream and first-byte time at the largest context", "stages": [ {"duration_s": 180, "rps": 0.5}, {"duration_s": 180, "rps": 1}, @@ -17,7 +17,7 @@ "ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000}, "duration_s": {"dist": "uniform", "min": 5, "max": 60}}, {"name": "stream-1m-chat", "weight": 1, "endpoint": "chat", "stream_fraction": 1, - "context_tokens": {"dist": "uniform", "min": 950000, "max": 1000000}, + "context_tokens": {"dist": "uniform", "min": 870000, "max": 920000}, "ttft_ms": {"dist": "uniform", "min": 1000, "max": 3000}, "duration_s": {"dist": "uniform", "min": 5, "max": 60}} ]