From 5990e10ff8adda1a69bec9c7412083daf8ed4b9b Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Tue, 3 Nov 2026 17:55:38 +0000 Subject: [PATCH] feat(loadtest): 5-minute low-rate sweep workloads; portkey routing with features full --- litellm-rust/loadtest/README.md | 7 ++++--- litellm-rust/loadtest/cmd/genconfig/main.go | 7 +++---- .../{sustained-25.json => sweep-10.json} | 4 ++-- .../{sustained-10.json => sweep-100.json} | 4 ++-- .../{sustained-50.json => sweep-25.json} | 4 ++-- .../{sustained-75.json => sweep-50.json} | 4 ++-- litellm-rust/loadtest/workloads/sweep-75.json | 20 +++++++++++++++++++ 7 files changed, 35 insertions(+), 15 deletions(-) rename litellm-rust/loadtest/workloads/{sustained-25.json => sweep-10.json} (84%) rename litellm-rust/loadtest/workloads/{sustained-10.json => sweep-100.json} (88%) rename litellm-rust/loadtest/workloads/{sustained-50.json => sweep-25.json} (84%) rename litellm-rust/loadtest/workloads/{sustained-75.json => sweep-50.json} (84%) create mode 100644 litellm-rust/loadtest/workloads/sweep-75.json diff --git a/litellm-rust/loadtest/README.md b/litellm-rust/loadtest/README.md index 5b6b09d9d21..25394c53ed3 100644 --- a/litellm-rust/loadtest/README.md +++ b/litellm-rust/loadtest/README.md @@ -16,8 +16,9 @@ machines. | `s3stub` | In-memory path-style S3 (PUT, GET, ListObjectsV2) for the `s3_v2` logs of a full-feature run. Objects are stored compressed and listed from a sorted index, so millions of logs fit. | Workloads live in `workloads/`: `smoke`, `realistic-ramp`, `long-streams`, `large-context`, -`key-churn` and `soak`. `sustained-10`, `-25`, `-50`, `-75`, `-100`, `-250`, `-500` and `-1000` hold one rate for -15 minutes with evenly spaced arrivals and smoke's request mix over every model group; +`key-churn` and `soak`. `sustained-100`, `sustained-250`, `sustained-500` and `sustained-1000` hold one rate for +15 minutes with evenly spaced arrivals and smoke's request mix over every model group (`sweep-10`, `-25`, `-50`, +`-75` and `-100` the same for 5 minutes each, for a low-rate sweep); `sustained-check` runs the same mix at 100, 500 and 1,000 rps for 2 minutes each, to check that the client, mock and stack hold 1,000 rps before the long runs. @@ -183,7 +184,7 @@ are only compared between the LiteLLM sides, since Bifrost's schema differs thro ## Portkey -A key set named `portkey` (`-key-sets ,portkey`, basic features only) is written as routing for the +A key set named `portkey` (`-key-sets ,portkey`; with `-features full` it is unchanged, since Portkey has none of those features) is written as routing for the [open-source Portkey gateway](https://github.com/Portkey-AI/gateway) instead of key rows. That gateway keeps no model list, keys or database: every request carries its routing in an `x-portkey-config` header. So genconfig writes `portkey-headers.json`, one header per model group, diff --git a/litellm-rust/loadtest/cmd/genconfig/main.go b/litellm-rust/loadtest/cmd/genconfig/main.go index 42461f3b0e1..4409fc13389 100644 --- a/litellm-rust/loadtest/cmd/genconfig/main.go +++ b/litellm-rust/loadtest/cmd/genconfig/main.go @@ -16,7 +16,7 @@ // on. S3 settings come from LOADTEST_S3_* environment variables. // // -key-sets ,portkey adds Portkey gateway routing (portkey.go): one x-portkey-config header per -// model group, in portkey-headers.json, for loadgen -model-headers. Basic features only. +// model group, in portkey-headers.json, for loadgen -model-headers. Features never change it. // // -key-sets python,rust writes one key set, with its own users and teams, per name, so each // gateway's spend can be told apart in a shared database. @@ -142,9 +142,8 @@ general_settings: sw.WriteString("-- Load-test virtual keys. Hash = sha256 hex of the key, as the proxy stores it.\n") for _, set := range sets { if set == portkeySet { - if full { - log.Fatal("the portkey key set needs -features basic: the open-source Portkey gateway has no keys, limits, budgets or logs to turn on") - } + // The same routing with -features full: the open-source Portkey gateway has no keys, + // limits, budgets or logs to turn on, so the other sides' features have no Portkey form. writePortkey(bifrostDeps, portkeyKeys(*keys, *keySeed), *outDir) continue } diff --git a/litellm-rust/loadtest/workloads/sustained-25.json b/litellm-rust/loadtest/workloads/sweep-10.json similarity index 84% rename from litellm-rust/loadtest/workloads/sustained-25.json rename to litellm-rust/loadtest/workloads/sweep-10.json index 103361af107..01dcabc3119 100644 --- a/litellm-rust/loadtest/workloads/sustained-25.json +++ b/litellm-rust/loadtest/workloads/sweep-10.json @@ -1,6 +1,6 @@ { - "name": "sustained 25 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", - "stages": [{"duration_s": 900, "rps": 25}], + "name": "sustained 10 rps for 5 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [{"duration_s": 300, "rps": 10}], "arrivals": "constant", "nonstream_max_s": 360, "warmup_s": 60, diff --git a/litellm-rust/loadtest/workloads/sustained-10.json b/litellm-rust/loadtest/workloads/sweep-100.json similarity index 88% rename from litellm-rust/loadtest/workloads/sustained-10.json rename to litellm-rust/loadtest/workloads/sweep-100.json index c25a31a1d2e..cbb0ac533db 100644 --- a/litellm-rust/loadtest/workloads/sustained-10.json +++ b/litellm-rust/loadtest/workloads/sweep-100.json @@ -1,6 +1,6 @@ { - "name": "sustained 10 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", - "stages": [{"duration_s": 900, "rps": 10}], + "name": "sustained 100 rps for 5 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [{"duration_s": 300, "rps": 100}], "arrivals": "constant", "nonstream_max_s": 360, "warmup_s": 60, diff --git a/litellm-rust/loadtest/workloads/sustained-50.json b/litellm-rust/loadtest/workloads/sweep-25.json similarity index 84% rename from litellm-rust/loadtest/workloads/sustained-50.json rename to litellm-rust/loadtest/workloads/sweep-25.json index c72e0a9332d..bf06a4f237a 100644 --- a/litellm-rust/loadtest/workloads/sustained-50.json +++ b/litellm-rust/loadtest/workloads/sweep-25.json @@ -1,6 +1,6 @@ { - "name": "sustained 50 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", - "stages": [{"duration_s": 900, "rps": 50}], + "name": "sustained 25 rps for 5 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [{"duration_s": 300, "rps": 25}], "arrivals": "constant", "nonstream_max_s": 360, "warmup_s": 60, diff --git a/litellm-rust/loadtest/workloads/sustained-75.json b/litellm-rust/loadtest/workloads/sweep-50.json similarity index 84% rename from litellm-rust/loadtest/workloads/sustained-75.json rename to litellm-rust/loadtest/workloads/sweep-50.json index 74f85df9aed..eb6cc322f7a 100644 --- a/litellm-rust/loadtest/workloads/sustained-75.json +++ b/litellm-rust/loadtest/workloads/sweep-50.json @@ -1,6 +1,6 @@ { - "name": "sustained 75 rps for 15 minutes, evenly spaced arrivals, smoke's request mix over every model group", - "stages": [{"duration_s": 900, "rps": 75}], + "name": "sustained 50 rps for 5 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [{"duration_s": 300, "rps": 50}], "arrivals": "constant", "nonstream_max_s": 360, "warmup_s": 60, diff --git a/litellm-rust/loadtest/workloads/sweep-75.json b/litellm-rust/loadtest/workloads/sweep-75.json new file mode 100644 index 00000000000..7753e5f9924 --- /dev/null +++ b/litellm-rust/loadtest/workloads/sweep-75.json @@ -0,0 +1,20 @@ +{ + "name": "sustained 75 rps for 5 minutes, evenly spaced arrivals, smoke's request mix over every model group", + "stages": [{"duration_s": 300, "rps": 75}], + "arrivals": "constant", + "nonstream_max_s": 360, + "warmup_s": 60, + "max_in_flight": 30000, + "keys": {"file": "out/keys.txt", "zipf_s": 0}, + "models": {"chat": {"prefix": "gpt-6-astra-", "count": 10}, "messages": {"prefix": "claude-opus-5-5-", "count": 10}}, + "classes": [ + {"name": "chat", "weight": 1, "endpoint": "chat", "stream_fraction": 0.5, + "context_tokens": {"dist": "loguniform", "min": 10000, "max": 50000}, + "ttft_ms": {"dist": "uniform", "min": 200, "max": 800}, + "duration_s": {"dist": "uniform", "min": 1, "max": 5}}, + {"name": "messages", "weight": 1, "endpoint": "messages", "stream_fraction": 0.5, + "context_tokens": {"dist": "loguniform", "min": 10000, "max": 50000}, + "ttft_ms": {"dist": "uniform", "min": 200, "max": 800}, + "duration_s": {"dist": "uniform", "min": 1, "max": 5}} + ] +}