diff --git a/tests/load_tests/compare_perf_results.py b/tests/load_tests/compare_perf_results.py new file mode 100644 index 00000000000..09e12d796e3 --- /dev/null +++ b/tests/load_tests/compare_perf_results.py @@ -0,0 +1,135 @@ +"""Compare baseline vs optimized locust load test results.""" +import csv +import sys +import os + + +def parse_stats_csv(filepath): + """Parse a locust stats CSV file.""" + if not os.path.exists(filepath): + return None + + with open(filepath) as f: + reader = csv.DictReader(f) + for row in reader: + if row.get("Name") == "Aggregated": + return { + "requests": int(row.get("Request Count", 0)), + "failures": int(row.get("Failure Count", 0)), + "median": float(row.get("Median Response Time", 0)), + "avg": float(row.get("Average Response Time", 0)), + "min": float(row.get("Min Response Time", 0)), + "max": float(row.get("Max Response Time", 0)), + "p50": float(row.get("50%", 0)), + "p66": float(row.get("66%", 0)), + "p75": float(row.get("75%", 0)), + "p80": float(row.get("80%", 0)), + "p90": float(row.get("90%", 0)), + "p95": float(row.get("95%", 0)), + "p98": float(row.get("98%", 0)), + "p99": float(row.get("99%", 0)), + "p999": float(row.get("99.9%", 0)), + "p9999": float(row.get("99.99%", 0)), + "rps": float(row.get("Requests/s", 0)), + } + return None + + +def print_comparison(label, baseline, optimized): + print() + print(f" {label}") + print("=" * 74) + print(f"{'Metric':<25} {'Baseline':>12} {'Optimized':>12} {'Change':>12} {'':>2}") + print("=" * 74) + + metrics = [ + ("Total Requests", "requests", "", False), + ("Failures", "failures", "", False), + ("Failure %", None, "%", False), + ("Requests/sec", "rps", " rps", True), + ("Median (ms)", "median", " ms", False), + ("Average (ms)", "avg", " ms", False), + ("P50 (ms)", "p50", " ms", False), + ("P75 (ms)", "p75", " ms", False), + ("P90 (ms)", "p90", " ms", False), + ("P95 (ms)", "p95", " ms", False), + ("P98 (ms)", "p98", " ms", False), + ("P99 (ms)", "p99", " ms", False), + ("P99.9 (ms)", "p999", " ms", False), + ("Max (ms)", "max", " ms", False), + ] + + for label_m, key, unit, higher_is_better in metrics: + if key is None: + b_val = (baseline["failures"] / baseline["requests"] * 100) if baseline["requests"] else 0 + o_val = (optimized["failures"] / optimized["requests"] * 100) if optimized["requests"] else 0 + else: + b_val = baseline[key] + o_val = optimized[key] + + if b_val > 0: + change = ((o_val - b_val) / b_val) * 100 + change_str = f"{change:+.1f}%" + if higher_is_better: + indicator = "+" if change > 2 else ("~" if change > -2 else "-") + else: + if key in ("requests", "failures", None): + indicator = "" + else: + indicator = "+" if change < -2 else ("~" if change < 2 else "-") + elif o_val == 0 and b_val == 0: + change_str = "0.0%" + indicator = "~" + else: + change_str = "N/A" + indicator = "" + print( + f"{label_m:<25} {b_val:>10.1f}{unit:>2} {o_val:>10.1f}{unit:>2} {change_str:>10} {indicator}" + ) + + print("=" * 74) + + +def main(): + results_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/load_tests/results" + + # 5000 user comparison + baseline_5k = parse_stats_csv(os.path.join(results_dir, "baseline_stats.csv")) + optimized_5k = parse_stats_csv(os.path.join(results_dir, "optimized_stats.csv")) + + # 2000 user comparison + baseline_2k = parse_stats_csv(os.path.join(results_dir, "baseline_2k_stats.csv")) + optimized_2k = parse_stats_csv(os.path.join(results_dir, "optimized_2k_stats.csv")) + + print() + print("=" * 74) + print(" LOCUST BENCHMARK: BASELINE vs OPTIMIZED (GIL perf fixes)") + print(" 60s run time per test, warmed up proxy, mock LLM backend") + + if baseline_5k and optimized_5k: + print_comparison("5,000 CONCURRENT USERS (saturation test)", baseline_5k, optimized_5k) + + if baseline_2k and optimized_2k: + print_comparison("2,000 CONCURRENT USERS (sub-saturation test)", baseline_2k, optimized_2k) + + if baseline_5k and optimized_5k: + rps_5k = ((optimized_5k["rps"] - baseline_5k["rps"]) / baseline_5k["rps"]) * 100 + p50_5k = ((optimized_5k["p50"] - baseline_5k["p50"]) / baseline_5k["p50"]) * 100 if baseline_5k["p50"] else 0 + + print() + print("SUMMARY") + print("-" * 74) + print(f" 5k users: RPS {rps_5k:+.1f}% | P50 {p50_5k:+.1f}%") + + if baseline_2k and optimized_2k: + rps_2k = ((optimized_2k["rps"] - baseline_2k["rps"]) / baseline_2k["rps"]) * 100 + p50_2k = ((optimized_2k["p50"] - baseline_2k["p50"]) / baseline_2k["p50"]) * 100 if baseline_2k["p50"] else 0 + p98_2k = ((optimized_2k["p98"] - baseline_2k["p98"]) / baseline_2k["p98"]) * 100 if baseline_2k["p98"] else 0 + print(f" 2k users: RPS {rps_2k:+.1f}% | P50 {p50_2k:+.1f}% | P98 {p98_2k:+.1f}%") + + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/tests/load_tests/loadtest_config_perf.yaml b/tests/load_tests/loadtest_config_perf.yaml new file mode 100644 index 00000000000..bf7878a5f26 --- /dev/null +++ b/tests/load_tests/loadtest_config_perf.yaml @@ -0,0 +1,17 @@ +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: http://127.0.0.1:18888/ + +general_settings: + master_key: sk-1234 + disable_spend_logs: False + +litellm_settings: + drop_params: True + telemetry: False + num_retries: 0 + request_timeout: 30 + callbacks: [] diff --git a/tests/load_tests/locustfile_perf.py b/tests/load_tests/locustfile_perf.py new file mode 100644 index 00000000000..6497bdde590 --- /dev/null +++ b/tests/load_tests/locustfile_perf.py @@ -0,0 +1,23 @@ +""" +Locust load test for performance comparison. +Uses a custom shape: 10s ramp to 5000 users, hold for 60s, then stop. +Results from the steady-state period are what matters. +""" +from locust import HttpUser, task, between + + +class ChatCompletionUser(HttpUser): + wait_time = between(0.01, 0.02) + + @task + def post_chat_completions(self): + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer sk-1234", + } + data = { + "model": "fake-openai-endpoint", + "max_tokens": 10, + "messages": [{"role": "user", "content": "Hello"}], + } + self.client.post("/chat/completions", json=data, headers=headers) diff --git a/tests/load_tests/run_perf_comparison.sh b/tests/load_tests/run_perf_comparison.sh new file mode 100755 index 00000000000..ff193ba505d --- /dev/null +++ b/tests/load_tests/run_perf_comparison.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE="$(cd "$SCRIPT_DIR/../.." && pwd)" +RESULTS_DIR="$SCRIPT_DIR/results" +mkdir -p "$RESULTS_DIR" + +USERS=${1:-5000} +SPAWN_RATE=${2:-500} +DURATION=${3:-60s} + +echo "================================================================" +echo " LiteLLM Performance Comparison: Baseline vs Optimized" +echo " Users: $USERS | Spawn Rate: $SPAWN_RATE | Duration: $DURATION" +echo "================================================================" + +wait_for_service() { + local url=$1 + local name=$2 + local max_attempts=${3:-60} + echo " Waiting for $name at $url..." + for i in $(seq 1 $max_attempts); do + if curl -s "$url" > /dev/null 2>&1; then + echo " $name is ready!" + return 0 + fi + sleep 1 + done + echo " ERROR: $name failed to start after $max_attempts seconds" + return 1 +} + +kill_port() { + local port=$1 + local pids=$(lsof -ti:$port 2>/dev/null || true) + if [ -n "$pids" ]; then + echo "$pids" | xargs kill -9 2>/dev/null || true + sleep 1 + fi +} + +# Start mock server +if ! curl -s http://127.0.0.1:18888/health > /dev/null 2>&1; then + echo "" + echo "Starting mock OpenAI server on port 18888..." + cd "$WORKSPACE" && poetry run python tests/load_tests/mock_openai_server.py & + MOCK_PID=$! + wait_for_service "http://127.0.0.1:18888/health" "Mock Server" 15 +else + echo "Mock server already running on port 18888" +fi + +# ---- PHASE 1: BASELINE (stash current changes) ---- +echo "" +echo "================================================================" +echo " PHASE 1: BASELINE (pre-optimization code)" +echo "================================================================" + +cd "$WORKSPACE" +CURRENT_COMMIT=$(git rev-parse HEAD) +PARENT_COMMIT=$(git rev-parse HEAD~1) + +echo " Current commit (optimized): ${CURRENT_COMMIT:0:12}" +echo " Parent commit (baseline): ${PARENT_COMMIT:0:12}" +echo " Checking out baseline..." +git checkout "$PARENT_COMMIT" -- \ + litellm/integrations/prometheus.py \ + litellm/litellm_core_utils/litellm_logging.py \ + litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py \ + litellm/litellm_core_utils/safe_json_dumps.py \ + litellm/proxy/litellm_pre_call_utils.py \ + litellm/proxy/spend_tracking/spend_tracking_utils.py \ + litellm/router.py \ + litellm/router_strategy/simple_shuffle.py \ + litellm/types/integrations/prometheus.py \ + 2>/dev/null + +kill_port 4000 + +echo " Starting proxy on port 4000 (baseline)..." +cd "$WORKSPACE" && poetry run litellm --config tests/load_tests/loadtest_config_perf.yaml --port 4000 > "$RESULTS_DIR/baseline_proxy.log" 2>&1 & +PROXY_PID=$! +wait_for_service "http://localhost:4000/health/liveliness" "LiteLLM Proxy (baseline)" 60 + +echo " Running baseline locust test..." +cd "$WORKSPACE" && poetry run locust -f tests/load_tests/locustfile.py \ + --headless -u "$USERS" -r "$SPAWN_RATE" --run-time "$DURATION" \ + --host http://localhost:4000 \ + --csv "$RESULTS_DIR/baseline" \ + --only-summary 2>&1 | tee "$RESULTS_DIR/baseline_output.txt" + +kill $PROXY_PID 2>/dev/null || true +sleep 2 +kill_port 4000 + +# ---- PHASE 2: OPTIMIZED (restore current changes) ---- +echo "" +echo "================================================================" +echo " PHASE 2: OPTIMIZED (with performance fixes)" +echo "================================================================" + +cd "$WORKSPACE" +echo " Restoring optimized code..." +git checkout "$CURRENT_COMMIT" -- \ + litellm/integrations/prometheus.py \ + litellm/litellm_core_utils/litellm_logging.py \ + litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py \ + litellm/litellm_core_utils/safe_json_dumps.py \ + litellm/proxy/litellm_pre_call_utils.py \ + litellm/proxy/spend_tracking/spend_tracking_utils.py \ + litellm/router.py \ + litellm/router_strategy/simple_shuffle.py \ + litellm/types/integrations/prometheus.py \ + 2>/dev/null + +kill_port 4000 + +echo " Starting proxy on port 4000 (optimized)..." +cd "$WORKSPACE" && poetry run litellm --config tests/load_tests/loadtest_config_perf.yaml --port 4000 > "$RESULTS_DIR/optimized_proxy.log" 2>&1 & +PROXY_PID=$! +wait_for_service "http://localhost:4000/health/liveliness" "LiteLLM Proxy (optimized)" 60 + +echo " Running optimized locust test..." +cd "$WORKSPACE" && poetry run locust -f tests/load_tests/locustfile.py \ + --headless -u "$USERS" -r "$SPAWN_RATE" --run-time "$DURATION" \ + --host http://localhost:4000 \ + --csv "$RESULTS_DIR/optimized" \ + --only-summary 2>&1 | tee "$RESULTS_DIR/optimized_output.txt" + +kill $PROXY_PID 2>/dev/null || true + +# ---- PHASE 3: COMPARE ---- +echo "" +echo "================================================================" +echo " RESULTS COMPARISON" +echo "================================================================" +cd "$WORKSPACE" && poetry run python tests/load_tests/compare_perf_results.py "$RESULTS_DIR"