test: add locust benchmark comparison tooling for perf analysis

Adds: - loadtest_config_perf.yaml: proxy config with spend_logs enabled - locustfile_perf.py: locust scenario for perf comparison - compare_perf_results.py: CSV parser + comparison report generator - run_perf_comparison.sh: automated baseline vs optimized runner Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com>
2026-09-12 23:01:41 +00:00 · 2026-03-08 01:00:05 +00:00 · 2026-03-08 01:00:05 +00:00 · 9e0bf1eec3
commit 9e0bf1eec3
parent a7b7a31b26
4 changed files with 313 additions and 0 deletions
--- a/tests/load_tests/compare_perf_results.py
+++ b/tests/load_tests/compare_perf_results.py
@ -0,0 +1,135 @@
+"""Compare baseline vs optimized locust load test results."""
+import csv
+import sys
+import os
+
+
+def parse_stats_csv(filepath):
+    """Parse a locust stats CSV file."""
+    if not os.path.exists(filepath):
+        return None
+
+    with open(filepath) as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            if row.get("Name") == "Aggregated":
+                return {
+                    "requests": int(row.get("Request Count", 0)),
+                    "failures": int(row.get("Failure Count", 0)),
+                    "median": float(row.get("Median Response Time", 0)),
+                    "avg": float(row.get("Average Response Time", 0)),
+                    "min": float(row.get("Min Response Time", 0)),
+                    "max": float(row.get("Max Response Time", 0)),
+                    "p50": float(row.get("50%", 0)),
+                    "p66": float(row.get("66%", 0)),
+                    "p75": float(row.get("75%", 0)),
+                    "p80": float(row.get("80%", 0)),
+                    "p90": float(row.get("90%", 0)),
+                    "p95": float(row.get("95%", 0)),
+                    "p98": float(row.get("98%", 0)),
+                    "p99": float(row.get("99%", 0)),
+                    "p999": float(row.get("99.9%", 0)),
+                    "p9999": float(row.get("99.99%", 0)),
+                    "rps": float(row.get("Requests/s", 0)),
+                }
+    return None
+
+
+def print_comparison(label, baseline, optimized):
+    print()
+    print(f"  {label}")
+    print("=" * 74)
+    print(f"{'Metric':<25} {'Baseline':>12} {'Optimized':>12} {'Change':>12}  {'':>2}")
+    print("=" * 74)
+
+    metrics = [
+        ("Total Requests", "requests", "", False),
+        ("Failures", "failures", "", False),
+        ("Failure %", None, "%", False),
+        ("Requests/sec", "rps", " rps", True),
+        ("Median (ms)", "median", " ms", False),
+        ("Average (ms)", "avg", " ms", False),
+        ("P50 (ms)", "p50", " ms", False),
+        ("P75 (ms)", "p75", " ms", False),
+        ("P90 (ms)", "p90", " ms", False),
+        ("P95 (ms)", "p95", " ms", False),
+        ("P98 (ms)", "p98", " ms", False),
+        ("P99 (ms)", "p99", " ms", False),
+        ("P99.9 (ms)", "p999", " ms", False),
+        ("Max (ms)", "max", " ms", False),
+    ]
+
+    for label_m, key, unit, higher_is_better in metrics:
+        if key is None:
+            b_val = (baseline["failures"] / baseline["requests"] * 100) if baseline["requests"] else 0
+            o_val = (optimized["failures"] / optimized["requests"] * 100) if optimized["requests"] else 0
+        else:
+            b_val = baseline[key]
+            o_val = optimized[key]
+
+        if b_val > 0:
+            change = ((o_val - b_val) / b_val) * 100
+            change_str = f"{change:+.1f}%"
+            if higher_is_better:
+                indicator = "+" if change > 2 else ("~" if change > -2 else "-")
+            else:
+                if key in ("requests", "failures", None):
+                    indicator = ""
+                else:
+                    indicator = "+" if change < -2 else ("~" if change < 2 else "-")
+        elif o_val == 0 and b_val == 0:
+            change_str = "0.0%"
+            indicator = "~"
+        else:
+            change_str = "N/A"
+            indicator = ""
+        print(
+            f"{label_m:<25} {b_val:>10.1f}{unit:>2} {o_val:>10.1f}{unit:>2} {change_str:>10}  {indicator}"
+        )
+
+    print("=" * 74)
+
+
+def main():
+    results_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/load_tests/results"
+
+    # 5000 user comparison
+    baseline_5k = parse_stats_csv(os.path.join(results_dir, "baseline_stats.csv"))
+    optimized_5k = parse_stats_csv(os.path.join(results_dir, "optimized_stats.csv"))
+
+    # 2000 user comparison
+    baseline_2k = parse_stats_csv(os.path.join(results_dir, "baseline_2k_stats.csv"))
+    optimized_2k = parse_stats_csv(os.path.join(results_dir, "optimized_2k_stats.csv"))
+
+    print()
+    print("=" * 74)
+    print("  LOCUST BENCHMARK: BASELINE vs OPTIMIZED (GIL perf fixes)")
+    print("  60s run time per test, warmed up proxy, mock LLM backend")
+
+    if baseline_5k and optimized_5k:
+        print_comparison("5,000 CONCURRENT USERS (saturation test)", baseline_5k, optimized_5k)
+
+    if baseline_2k and optimized_2k:
+        print_comparison("2,000 CONCURRENT USERS (sub-saturation test)", baseline_2k, optimized_2k)
+
+    if baseline_5k and optimized_5k:
+        rps_5k = ((optimized_5k["rps"] - baseline_5k["rps"]) / baseline_5k["rps"]) * 100
+        p50_5k = ((optimized_5k["p50"] - baseline_5k["p50"]) / baseline_5k["p50"]) * 100 if baseline_5k["p50"] else 0
+
+        print()
+        print("SUMMARY")
+        print("-" * 74)
+        print(f"  5k users: RPS {rps_5k:+.1f}% | P50 {p50_5k:+.1f}%")
+
+    if baseline_2k and optimized_2k:
+        rps_2k = ((optimized_2k["rps"] - baseline_2k["rps"]) / baseline_2k["rps"]) * 100
+        p50_2k = ((optimized_2k["p50"] - baseline_2k["p50"]) / baseline_2k["p50"]) * 100 if baseline_2k["p50"] else 0
+        p98_2k = ((optimized_2k["p98"] - baseline_2k["p98"]) / baseline_2k["p98"]) * 100 if baseline_2k["p98"] else 0
+        print(f"  2k users: RPS {rps_2k:+.1f}% | P50 {p50_2k:+.1f}% | P98 {p98_2k:+.1f}%")
+
+    print()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main() or 0)
--- a/tests/load_tests/loadtest_config_perf.yaml
+++ b/tests/load_tests/loadtest_config_perf.yaml
@ -0,0 +1,17 @@
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: http://127.0.0.1:18888/
+
+general_settings:
+  master_key: sk-1234
+  disable_spend_logs: False
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+  num_retries: 0
+  request_timeout: 30
+  callbacks: []
--- a/tests/load_tests/locustfile_perf.py
+++ b/tests/load_tests/locustfile_perf.py
@ -0,0 +1,23 @@
+"""
+Locust load test for performance comparison.
+Uses a custom shape: 10s ramp to 5000 users, hold for 60s, then stop.
+Results from the steady-state period are what matters.
+"""
+from locust import HttpUser, task, between
+
+
+class ChatCompletionUser(HttpUser):
+    wait_time = between(0.01, 0.02)
+
+    @task
+    def post_chat_completions(self):
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer sk-1234",
+        }
+        data = {
+            "model": "fake-openai-endpoint",
+            "max_tokens": 10,
+            "messages": [{"role": "user", "content": "Hello"}],
+        }
+        self.client.post("/chat/completions", json=data, headers=headers)
--- a/tests/load_tests/run_perf_comparison.sh
+++ b/tests/load_tests/run_perf_comparison.sh
@ -0,0 +1,138 @@
+#!/bin/bash
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORKSPACE="$(cd "$SCRIPT_DIR/../.." && pwd)"
+RESULTS_DIR="$SCRIPT_DIR/results"
+mkdir -p "$RESULTS_DIR"
+
+USERS=${1:-5000}
+SPAWN_RATE=${2:-500}
+DURATION=${3:-60s}
+
+echo "================================================================"
+echo "  LiteLLM Performance Comparison: Baseline vs Optimized"
+echo "  Users: $USERS | Spawn Rate: $SPAWN_RATE | Duration: $DURATION"
+echo "================================================================"
+
+wait_for_service() {
+    local url=$1
+    local name=$2
+    local max_attempts=${3:-60}
+    echo "  Waiting for $name at $url..."
+    for i in $(seq 1 $max_attempts); do
+        if curl -s "$url" > /dev/null 2>&1; then
+            echo "  $name is ready!"
+            return 0
+        fi
+        sleep 1
+    done
+    echo "  ERROR: $name failed to start after $max_attempts seconds"
+    return 1
+}
+
+kill_port() {
+    local port=$1
+    local pids=$(lsof -ti:$port 2>/dev/null || true)
+    if [ -n "$pids" ]; then
+        echo "$pids" | xargs kill -9 2>/dev/null || true
+        sleep 1
+    fi
+}
+
+# Start mock server
+if ! curl -s http://127.0.0.1:18888/health > /dev/null 2>&1; then
+    echo ""
+    echo "Starting mock OpenAI server on port 18888..."
+    cd "$WORKSPACE" && poetry run python tests/load_tests/mock_openai_server.py &
+    MOCK_PID=$!
+    wait_for_service "http://127.0.0.1:18888/health" "Mock Server" 15
+else
+    echo "Mock server already running on port 18888"
+fi
+
+# ---- PHASE 1: BASELINE (stash current changes) ----
+echo ""
+echo "================================================================"
+echo "  PHASE 1: BASELINE (pre-optimization code)"
+echo "================================================================"
+
+cd "$WORKSPACE"
+CURRENT_COMMIT=$(git rev-parse HEAD)
+PARENT_COMMIT=$(git rev-parse HEAD~1)
+
+echo "  Current commit (optimized): ${CURRENT_COMMIT:0:12}"
+echo "  Parent commit (baseline):   ${PARENT_COMMIT:0:12}"
+echo "  Checking out baseline..."
+git checkout "$PARENT_COMMIT" -- \
+    litellm/integrations/prometheus.py \
+    litellm/litellm_core_utils/litellm_logging.py \
+    litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py \
+    litellm/litellm_core_utils/safe_json_dumps.py \
+    litellm/proxy/litellm_pre_call_utils.py \
+    litellm/proxy/spend_tracking/spend_tracking_utils.py \
+    litellm/router.py \
+    litellm/router_strategy/simple_shuffle.py \
+    litellm/types/integrations/prometheus.py \
+    2>/dev/null
+
+kill_port 4000
+
+echo "  Starting proxy on port 4000 (baseline)..."
+cd "$WORKSPACE" && poetry run litellm --config tests/load_tests/loadtest_config_perf.yaml --port 4000 > "$RESULTS_DIR/baseline_proxy.log" 2>&1 &
+PROXY_PID=$!
+wait_for_service "http://localhost:4000/health/liveliness" "LiteLLM Proxy (baseline)" 60
+
+echo "  Running baseline locust test..."
+cd "$WORKSPACE" && poetry run locust -f tests/load_tests/locustfile.py \
+    --headless -u "$USERS" -r "$SPAWN_RATE" --run-time "$DURATION" \
+    --host http://localhost:4000 \
+    --csv "$RESULTS_DIR/baseline" \
+    --only-summary 2>&1 | tee "$RESULTS_DIR/baseline_output.txt"
+
+kill $PROXY_PID 2>/dev/null || true
+sleep 2
+kill_port 4000
+
+# ---- PHASE 2: OPTIMIZED (restore current changes) ----
+echo ""
+echo "================================================================"
+echo "  PHASE 2: OPTIMIZED (with performance fixes)"
+echo "================================================================"
+
+cd "$WORKSPACE"
+echo "  Restoring optimized code..."
+git checkout "$CURRENT_COMMIT" -- \
+    litellm/integrations/prometheus.py \
+    litellm/litellm_core_utils/litellm_logging.py \
+    litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py \
+    litellm/litellm_core_utils/safe_json_dumps.py \
+    litellm/proxy/litellm_pre_call_utils.py \
+    litellm/proxy/spend_tracking/spend_tracking_utils.py \
+    litellm/router.py \
+    litellm/router_strategy/simple_shuffle.py \
+    litellm/types/integrations/prometheus.py \
+    2>/dev/null
+
+kill_port 4000
+
+echo "  Starting proxy on port 4000 (optimized)..."
+cd "$WORKSPACE" && poetry run litellm --config tests/load_tests/loadtest_config_perf.yaml --port 4000 > "$RESULTS_DIR/optimized_proxy.log" 2>&1 &
+PROXY_PID=$!
+wait_for_service "http://localhost:4000/health/liveliness" "LiteLLM Proxy (optimized)" 60
+
+echo "  Running optimized locust test..."
+cd "$WORKSPACE" && poetry run locust -f tests/load_tests/locustfile.py \
+    --headless -u "$USERS" -r "$SPAWN_RATE" --run-time "$DURATION" \
+    --host http://localhost:4000 \
+    --csv "$RESULTS_DIR/optimized" \
+    --only-summary 2>&1 | tee "$RESULTS_DIR/optimized_output.txt"
+
+kill $PROXY_PID 2>/dev/null || true
+
+# ---- PHASE 3: COMPARE ----
+echo ""
+echo "================================================================"
+echo "  RESULTS COMPARISON"
+echo "================================================================"
+cd "$WORKSPACE" && poetry run python tests/load_tests/compare_perf_results.py "$RESULTS_DIR"