From b635f92d90448aa72712c35132e647500452701a Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 12 Dec 2025 11:26:34 -0800 Subject: [PATCH] Add benchmark_proxy_vs_provider.py script to scripts directory with usage examples (#17889) --- scripts/benchmark_proxy_vs_provider.py | 774 +++++++++++++++++++++++++ 1 file changed, 774 insertions(+) create mode 100755 scripts/benchmark_proxy_vs_provider.py diff --git a/scripts/benchmark_proxy_vs_provider.py b/scripts/benchmark_proxy_vs_provider.py new file mode 100755 index 00000000000..94fd0ed00c7 --- /dev/null +++ b/scripts/benchmark_proxy_vs_provider.py @@ -0,0 +1,774 @@ +#!/usr/bin/env python3 +""" +Benchmark script comparing LiteLLM proxy vs direct provider endpoint. +Makes parallel calls to each endpoint and compares statistics including latency, throughput, and success rates. + +USAGE EXAMPLES: + +1. Basic Usage (Sequential, Recommended): + # Set required environment variables + export LITELLM_PROXY_URL='http://localhost:4000/chat/completions' + export PROVIDER_URL='https://api.openai.com/v1/chat/completions' + export LITELLM_PROXY_API_KEY='sk-1234' + export PROVIDER_API_KEY='sk-openai-key' + + # Run from scripts directory + cd scripts + python benchmark_proxy_vs_provider.py + +2. Multiple Runs for Statistical Accuracy: + python benchmark_proxy_vs_provider.py --runs 5 + # Averages results across 5 runs for more reliable metrics + +3. Realistic Load Testing with Concurrency Limit: + python benchmark_proxy_vs_provider.py --max-concurrent 100 --requests 2000 + # Limits to 100 concurrent requests (prevents overwhelming the server) + +4. Quick Test with Fewer Requests: + python benchmark_proxy_vs_provider.py --requests 100 + # Faster test with 100 requests instead of default 1000 + +5. Parallel Execution (Not Recommended): + python benchmark_proxy_vs_provider.py --parallel + # Runs both benchmarks simultaneously (may affect accuracy) + +6. Custom Timeout: + python benchmark_proxy_vs_provider.py --timeout 120 + # Sets request timeout to 120 seconds + +7. Combined Options: + python benchmark_proxy_vs_provider.py --runs 3 --requests 500 --max-concurrent 50 + # 3 runs, 500 requests each, max 50 concurrent + +REQUIRED ENVIRONMENT VARIABLES: + - LITELLM_PROXY_URL: Full URL to LiteLLM proxy chat completions endpoint + - PROVIDER_URL: Full URL to direct provider chat completions endpoint + +OPTIONAL ENVIRONMENT VARIABLES: + - LITELLM_PROXY_API_KEY: API key for LiteLLM proxy (if auth required) + - PROVIDER_API_KEY: API key for direct provider (if auth required) + +OUTPUT: + The script provides detailed statistics including: + - Success/error rates + - Latency metrics (mean, median, p95, p99) + - Throughput (requests per second) + - Comparison between proxy and provider performance + - Run-to-run variance (when using --runs > 1) +""" + +import asyncio +import aiohttp +import time +import json +import argparse +import os +from typing import List, Dict, Any, Optional +from dataclasses import dataclass, field +from statistics import mean, median, stdev +import sys +from aiohttp import TCPConnector + + +@dataclass +class RequestStats: + """Statistics for a single request""" + success: bool + latency: float + error: str = "" + status_code: int = 0 + + +@dataclass +class BenchmarkResults: + """Aggregated benchmark results""" + total_requests: int = 0 + successful_requests: int = 0 + failed_requests: int = 0 + latencies: List[float] = field(default_factory=list) + errors: List[str] = field(default_factory=list) + status_codes: Dict[int, int] = field(default_factory=dict) + total_time: float = 0.0 + + def calculate_stats(self) -> Dict[str, Any]: + """Calculate statistics from the results""" + if not self.latencies: + return { + "total_requests": self.total_requests, + "successful_requests": self.successful_requests, + "failed_requests": self.failed_requests, + "success_rate": 0.0, + "error_rate": 1.0, + "total_time": self.total_time, + "requests_per_second": 0.0, + "status_codes": self.status_codes, + "unique_errors": len(set(self.errors)) if self.errors else 0, + } + + return { + "total_requests": self.total_requests, + "successful_requests": self.successful_requests, + "failed_requests": self.failed_requests, + "success_rate": (self.successful_requests / self.total_requests) * 100, + "error_rate": (self.failed_requests / self.total_requests) * 100, + "total_time": self.total_time, + "requests_per_second": self.total_requests / self.total_time if self.total_time > 0 else 0, + "latency_stats": { + "mean": mean(self.latencies), + "median": median(self.latencies), + "min": min(self.latencies), + "max": max(self.latencies), + "std_dev": stdev(self.latencies) if len(self.latencies) > 1 else 0.0, + "p50": median(self.latencies), + "p95": self._percentile(self.latencies, 95), + "p99": self._percentile(self.latencies, 99), + }, + "status_codes": self.status_codes, + "unique_errors": len(set(self.errors)) if self.errors else 0, + } + + @staticmethod + def _percentile(data: List[float], percentile: int) -> float: + """Calculate percentile""" + sorted_data = sorted(data) + index = int(len(sorted_data) * (percentile / 100)) + if index >= len(sorted_data): + index = len(sorted_data) - 1 + return sorted_data[index] + + +async def make_request( + session: aiohttp.ClientSession, + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + timeout: aiohttp.ClientTimeout, +) -> RequestStats: + """Make a single async request and return stats""" + # Use time.perf_counter() for higher precision timing + start_time = time.perf_counter() + try: + async with session.post(url, json=payload, headers=headers, timeout=timeout) as response: + # Read response body to ensure complete transfer + response_body = await response.read() + latency = time.perf_counter() - start_time + status_code = response.status + + if response.status == 200: + # Validate response is valid JSON + try: + json.loads(response_body) + except json.JSONDecodeError: + return RequestStats( + success=False, + latency=latency, + error="Invalid JSON response", + status_code=status_code, + ) + + return RequestStats( + success=True, + latency=latency, + status_code=status_code, + ) + else: + error_text = response_body.decode('utf-8', errors='ignore')[:100] + return RequestStats( + success=False, + latency=latency, + error=f"HTTP {status_code}: {error_text}", + status_code=status_code, + ) + except asyncio.TimeoutError: + latency = time.perf_counter() - start_time + return RequestStats( + success=False, + latency=latency, + error="Timeout", + status_code=0, + ) + except Exception as e: + latency = time.perf_counter() - start_time + return RequestStats( + success=False, + latency=latency, + error=str(e)[:100], + status_code=0, + ) + + +async def warmup_endpoint( + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + num_warmup: int = 5, + timeout_seconds: int = 60, +) -> None: + """Perform warm-up requests to avoid cold start penalties""" + timeout = aiohttp.ClientTimeout(total=timeout_seconds) + connector = TCPConnector( + limit=100, # Max connections + limit_per_host=50, # Max connections per host + ttl_dns_cache=300, # DNS cache TTL + force_close=False, # Reuse connections + ) + + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [ + make_request(session, url, headers, payload, timeout) + for _ in range(num_warmup) + ] + await asyncio.gather(*tasks, return_exceptions=True) + + # Brief pause after warmup to let connections stabilize + await asyncio.sleep(0.5) + + +async def make_request_with_semaphore( + session: aiohttp.ClientSession, + semaphore: asyncio.Semaphore, + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + timeout: aiohttp.ClientTimeout, +) -> RequestStats: + """Make a request with semaphore-based concurrency control""" + async with semaphore: + return await make_request(session, url, headers, payload, timeout) + + +async def benchmark_endpoint( + url: str, + headers: Dict[str, str], + payload: Dict[str, Any], + num_requests: int = 1000, + timeout_seconds: int = 60, + warmup: bool = True, + max_concurrent: Optional[int] = None, +) -> BenchmarkResults: + """Benchmark an endpoint with parallel requests + + Args: + url: Endpoint URL to benchmark + headers: HTTP headers + payload: Request payload + num_requests: Total number of requests to make + timeout_seconds: Request timeout + warmup: Whether to perform warm-up requests + max_concurrent: Maximum concurrent requests (None = unlimited, all at once) + """ + print(f"\nStarting benchmark for {url}") + + if warmup: + print(f" Warming up with 5 requests...") + await warmup_endpoint(url, headers, payload, num_warmup=5, timeout_seconds=timeout_seconds) + + if max_concurrent: + print(f" Making {num_requests} requests with max {max_concurrent} concurrent...") + else: + print(f" Making {num_requests} requests in parallel (unlimited concurrency)...") + + results = BenchmarkResults(total_requests=num_requests) + timeout = aiohttp.ClientTimeout(total=timeout_seconds) + + # Set connector limits based on concurrency + if max_concurrent: + connector_limit = min(max_concurrent * 2, 200) # Allow some headroom + connector_limit_per_host = max_concurrent + else: + connector_limit = 200 + connector_limit_per_host = 100 + + # Use optimized connector for connection pooling and reuse + connector = TCPConnector( + limit=connector_limit, + limit_per_host=connector_limit_per_host, + ttl_dns_cache=300, # DNS cache TTL (5 minutes) + force_close=False, # Reuse connections for better performance + enable_cleanup_closed=True, # Clean up closed connections + ) + + # Use time.perf_counter() for higher precision + start_time = time.perf_counter() + + async with aiohttp.ClientSession(connector=connector) as session: + if max_concurrent: + # Use semaphore to limit concurrency + semaphore = asyncio.Semaphore(max_concurrent) + tasks = [ + make_request_with_semaphore(session, semaphore, url, headers, payload, timeout) + for _ in range(num_requests) + ] + else: + # Create all tasks at once for maximum parallelism + tasks = [ + make_request(session, url, headers, payload, timeout) + for _ in range(num_requests) + ] + + # Execute all requests (with concurrency limit if specified) + request_stats = await asyncio.gather(*tasks) + + results.total_time = time.perf_counter() - start_time + + # Aggregate results + for stats in request_stats: + if stats.success: + results.successful_requests += 1 + results.latencies.append(stats.latency) + else: + results.failed_requests += 1 + results.errors.append(stats.error) + + if stats.status_code > 0: + results.status_codes[stats.status_code] = results.status_codes.get(stats.status_code, 0) + 1 + + return results + + +def print_results(name: str, results: BenchmarkResults): + """Print formatted benchmark results""" + stats = results.calculate_stats() + + print(f"\n{'='*60}") + print(f"Results for {name}") + print(f"{'='*60}") + print(f"Total Requests: {stats['total_requests']}") + print(f"Successful Requests: {stats['successful_requests']}") + print(f"Failed Requests: {stats['failed_requests']}") + print(f"Success Rate: {stats['success_rate']:.2f}%") + print(f"Error Rate: {stats['error_rate']:.2f}%") + print(f"Total Time: {stats['total_time']:.2f}s") + print(f"Requests/Second: {stats['requests_per_second']:.2f}") + + if 'latency_stats' in stats: + latency = stats['latency_stats'] + print(f"\nLatency Statistics (seconds):") + print(f" Mean: {latency['mean']:.4f}s") + print(f" Median (p50): {latency['median']:.4f}s") + print(f" Min: {latency['min']:.4f}s") + print(f" Max: {latency['max']:.4f}s") + print(f" Std Dev: {latency['std_dev']:.4f}s") + print(f" p95: {latency['p95']:.4f}s") + print(f" p99: {latency['p99']:.4f}s") + + if stats['status_codes']: + print(f"\nStatus Codes:") + for code, count in sorted(stats['status_codes'].items()): + print(f" {code}: {count}") + + if results.errors: + print(f"\nErrors (showing first 5 unique):") + unique_errors = list(set(results.errors))[:5] + for error in unique_errors: + count = results.errors.count(error) + print(f" [{count}x] {error}") + + +def aggregate_results(results_list: List[BenchmarkResults]) -> BenchmarkResults: + """Aggregate results from multiple runs""" + if not results_list: + return BenchmarkResults() + + aggregated = BenchmarkResults() + + # Aggregate all latencies + all_latencies = [] + all_errors = [] + total_requests = 0 + total_successful = 0 + total_failed = 0 + total_time_sum = 0.0 + status_codes_combined = {} + + for result in results_list: + all_latencies.extend(result.latencies) + all_errors.extend(result.errors) + total_requests += result.total_requests + total_successful += result.successful_requests + total_failed += result.failed_requests + total_time_sum += result.total_time + + for code, count in result.status_codes.items(): + status_codes_combined[code] = status_codes_combined.get(code, 0) + count + + aggregated.latencies = all_latencies + aggregated.errors = all_errors + aggregated.total_requests = total_requests + aggregated.successful_requests = total_successful + aggregated.failed_requests = total_failed + aggregated.total_time = total_time_sum / len(results_list) # Average time + aggregated.status_codes = status_codes_combined + + return aggregated + + +def print_run_variance(name: str, results_list: List[BenchmarkResults]): + """Print variance statistics across multiple runs""" + if len(results_list) <= 1: + return + + print(f"\n{'='*60}") + print(f"Run-to-Run Variance: {name}") + print(f"{'='*60}") + + # Collect mean latencies from each run + mean_latencies = [] + throughputs = [] + + for result in results_list: + stats = result.calculate_stats() + if 'latency_stats' in stats: + mean_latencies.append(stats['latency_stats']['mean']) + throughputs.append(stats['requests_per_second']) + + if mean_latencies: + print(f"\nMean Latency Variance:") + print(f" Runs: {len(mean_latencies)}") + print(f" Mean: {mean(mean_latencies):.4f}s") + print(f" Min: {min(mean_latencies):.4f}s") + print(f" Max: {max(mean_latencies):.4f}s") + print(f" Std Dev: {stdev(mean_latencies):.4f}s" if len(mean_latencies) > 1 else " Std Dev: N/A") + print(f" Coefficient of Variation: {(stdev(mean_latencies) / mean(mean_latencies) * 100):.2f}%" if len(mean_latencies) > 1 else " Coefficient of Variation: N/A") + + if throughputs: + print(f"\nThroughput Variance:") + print(f" Mean: {mean(throughputs):.2f} req/s") + print(f" Min: {min(throughputs):.2f} req/s") + print(f" Max: {max(throughputs):.2f} req/s") + print(f" Std Dev: {stdev(throughputs):.2f} req/s" if len(throughputs) > 1 else " Std Dev: N/A") + + +def compare_results(proxy_results: BenchmarkResults, provider_results: BenchmarkResults): + """Compare and print differences between proxy and provider results""" + proxy_stats = proxy_results.calculate_stats() + provider_stats = provider_results.calculate_stats() + + print(f"\n{'='*60}") + print(f"Comparison: LiteLLM Proxy vs Direct Provider") + print(f"{'='*60}") + + # Success Rate Comparison + print(f"\nSuccess Rate:") + print(f" Proxy: {proxy_stats['success_rate']:.2f}%") + print(f" Provider: {provider_stats['success_rate']:.2f}%") + diff = proxy_stats['success_rate'] - provider_stats['success_rate'] + print(f" Difference: {diff:+.2f}%") + + # Throughput Comparison + print(f"\nThroughput (requests/second):") + print(f" Proxy: {proxy_stats['requests_per_second']:.2f}") + print(f" Provider: {provider_stats['requests_per_second']:.2f}") + diff = proxy_stats['requests_per_second'] - provider_stats['requests_per_second'] + print(f" Difference: {diff:+.2f} req/s") + + # Latency Comparison + if 'latency_stats' in proxy_stats and 'latency_stats' in provider_stats: + print(f"\nLatency Comparison (seconds):") + proxy_latency = proxy_stats['latency_stats'] + provider_latency = provider_stats['latency_stats'] + + metrics = ['mean', 'median', 'p95', 'p99'] + for metric in metrics: + proxy_val = proxy_latency[metric] + provider_val = provider_latency[metric] + diff = proxy_val - provider_val + diff_pct = (diff / provider_val * 100) if provider_val > 0 else 0 + print(f" {metric.upper():8s}: Proxy={proxy_val:.4f}s, Provider={provider_val:.4f}s, Diff={diff:+.4f}s ({diff_pct:+.2f}%)") + + # Total Time Comparison + print(f"\nTotal Time:") + print(f" Proxy: {proxy_stats['total_time']:.2f}s") + print(f" Provider: {provider_stats['total_time']:.2f}s") + diff = proxy_stats['total_time'] - provider_stats['total_time'] + diff_pct = (diff / provider_stats['total_time'] * 100) if provider_stats['total_time'] > 0 else 0 + print(f" Difference: {diff:+.2f}s ({diff_pct:+.2f}%)") + + +async def main(): + """Main benchmark function""" + parser = argparse.ArgumentParser( + description="Benchmark LiteLLM proxy vs direct provider endpoint", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Environment Variables (required): + LITELLM_PROXY_URL - URL of the LiteLLM proxy endpoint (e.g., http://localhost:4000/chat/completions) + PROVIDER_URL - URL of the direct provider endpoint (e.g., https://api.openai.com/v1/chat/completions) + LITELLM_PROXY_API_KEY - API key for LiteLLM proxy (optional, but may be required) + PROVIDER_API_KEY - API key for direct provider (optional, but may be required) + +Examples: + # 1. Basic usage (recommended - sequential execution) + export LITELLM_PROXY_URL='http://localhost:4000/chat/completions' + export PROVIDER_URL='https://api.openai.com/v1/chat/completions' + export LITELLM_PROXY_API_KEY='sk-1234' + export PROVIDER_API_KEY='sk-openai-key' + python scripts/benchmark_proxy_vs_provider.py + + # 2. Multiple runs for statistical accuracy (recommended) + python scripts/benchmark_proxy_vs_provider.py --runs 5 + + # 3. Realistic load testing with concurrency limit + python scripts/benchmark_proxy_vs_provider.py --max-concurrent 100 --requests 2000 + + # 4. Quick test with fewer requests + python scripts/benchmark_proxy_vs_provider.py --requests 100 + + # 5. Parallel execution (not recommended - may affect accuracy) + python scripts/benchmark_proxy_vs_provider.py --parallel + + # 6. Custom timeout for slower endpoints + python scripts/benchmark_proxy_vs_provider.py --timeout 120 + + # 7. Combined options for comprehensive testing + python scripts/benchmark_proxy_vs_provider.py --runs 3 --requests 500 --max-concurrent 50 + + # 8. Skip warmup (not recommended - may affect first request accuracy) + python scripts/benchmark_proxy_vs_provider.py --no-warmup + """ + ) + parser.add_argument( + "--parallel", + action="store_true", + help="Run both benchmarks in parallel (default: sequential to avoid interference)", + ) + parser.add_argument( + "--requests", + type=int, + default=1000, + help="Number of requests per endpoint (default: 1000)", + ) + parser.add_argument( + "--timeout", + type=int, + default=60, + help="Request timeout in seconds (default: 60)", + ) + parser.add_argument( + "--runs", + type=int, + default=1, + help="Number of benchmark runs to average (default: 1, recommended: 3-5 for accuracy)", + ) + parser.add_argument( + "--no-warmup", + action="store_true", + help="Skip warm-up requests (not recommended)", + ) + parser.add_argument( + "--max-concurrent", + type=int, + default=None, + help="Maximum concurrent requests (default: unlimited - all at once). " + "Useful for realistic load testing (e.g., --max-concurrent 100)", + ) + + args = parser.parse_args() + + # Configuration from environment variables + LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL") + PROVIDER_URL = os.getenv("PROVIDER_URL") + LITELLM_PROXY_API_KEY = os.getenv("LITELLM_PROXY_API_KEY", "") + PROVIDER_API_KEY = os.getenv("PROVIDER_API_KEY", "") + + # Validate required environment variables + if not LITELLM_PROXY_URL: + print("Error: LITELLM_PROXY_URL environment variable is required") + print(" Example: export LITELLM_PROXY_URL='https://your-proxy.com/chat/completions'") + sys.exit(1) + + if not PROVIDER_URL: + print("Error: PROVIDER_URL environment variable is required") + print(" Example: export PROVIDER_URL='https://your-provider.com/v1/chat/completions'") + sys.exit(1) + + # Headers for LiteLLM proxy + proxy_headers = { + "Content-Type": "application/json", + } + if LITELLM_PROXY_API_KEY: + proxy_headers["Authorization"] = f"Bearer {LITELLM_PROXY_API_KEY}" + else: + print("Warning: LITELLM_PROXY_API_KEY not set, requests may fail if authentication is required") + + # Headers for direct provider + provider_headers = { + "Content-Type": "application/json", + } + if PROVIDER_API_KEY: + provider_headers["Authorization"] = f"Bearer {PROVIDER_API_KEY}" + else: + print("Warning: PROVIDER_API_KEY not set, requests may fail if authentication is required") + + # Payload (same for both) + payload = { + "model": "db-openai-endpoint", # For proxy + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "max_tokens": 100, + "user": "new_user" + } + + # For direct provider, might need different model name + provider_payload = payload.copy() + # provider_payload["model"] = "gpt-3.5-turbo" # Uncomment if needed + + num_requests = args.requests + timeout_seconds = args.timeout + + print("="*60) + print("LiteLLM Proxy vs Provider Benchmark") + print("="*60) + print(f"Configuration (from environment variables):") + print(f" Proxy URL: {LITELLM_PROXY_URL}") + print(f" Provider URL: {PROVIDER_URL}") + print(f" Proxy API Key: {'Set' if LITELLM_PROXY_API_KEY else 'Not set (may cause auth errors)'}") + print(f" Provider API Key: {'Set' if PROVIDER_API_KEY else 'Not set (may cause auth errors)'}") + print(f" Requests: {num_requests}") + print(f" Runs: {args.runs}") + print(f" Max Concurrent: {args.max_concurrent if args.max_concurrent else 'Unlimited (all at once)'}") + print(f" Timeout: {timeout_seconds}s") + print(f" Warmup: {'Enabled' if not args.no_warmup else 'Disabled (not recommended)'}") + print(f" Mode: {'Parallel (may affect results)' if args.parallel else 'Sequential (recommended)'}") + + if not args.max_concurrent: + print(f"\nTip: Use --max-concurrent 100 for more realistic load testing") + print(f" (prevents overwhelming the server with all requests at once)") + + if args.parallel: + print(f"\nWARNING: Running benchmarks in parallel may affect results due to:") + print(f" - Shared network bandwidth") + print(f" - Provider endpoint receiving double load (via proxy + direct)") + print(f" - Potential rate limiting issues") + print(f" - Resource contention") + + # Run benchmarks multiple times if requested + all_proxy_results = [] + all_provider_results = [] + + warmup_enabled = not args.no_warmup + + if args.runs > 1: + print(f"\nRunning {args.runs} benchmark runs for statistical accuracy...") + print(f" Results will be averaged across all runs.\n") + + overall_start_time = time.perf_counter() + + # Initialize to satisfy type checker (will always be set in loop) + proxy_results: Optional[BenchmarkResults] = None + provider_results: Optional[BenchmarkResults] = None + + for run_num in range(1, args.runs + 1): + if args.runs > 1: + print(f"\n{'='*60}") + print(f"Run {run_num}/{args.runs}") + print(f"{'='*60}") + + if args.parallel: + print(f"\nRunning both benchmarks in parallel...") + proxy_results, provider_results = await asyncio.gather( + benchmark_endpoint( + LITELLM_PROXY_URL, + proxy_headers, + payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ), + benchmark_endpoint( + PROVIDER_URL, + provider_headers, + provider_payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ), + ) + else: + print(f"\nRunning benchmarks sequentially (proxy first, then provider)...") + if run_num == 1: + print(f" This ensures accurate results without interference.\n") + + proxy_results = await benchmark_endpoint( + LITELLM_PROXY_URL, + proxy_headers, + payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ) + + if run_num < args.runs or args.runs == 1: + print(f"\nWaiting 3 seconds before starting provider benchmark...") + await asyncio.sleep(3) # Longer pause to ensure clean separation + + provider_results = await benchmark_endpoint( + PROVIDER_URL, + provider_headers, + provider_payload, + num_requests, + timeout_seconds, + warmup=warmup_enabled and run_num == 1, # Only warmup on first run + max_concurrent=args.max_concurrent, + ) + + all_proxy_results.append(proxy_results) + all_provider_results.append(provider_results) + + # Brief pause between runs + if run_num < args.runs: + print(f"\nWaiting 5 seconds before next run...") + await asyncio.sleep(5) + + overall_benchmark_time = time.perf_counter() - overall_start_time + print(f"\nAll benchmark runs completed in {overall_benchmark_time:.2f}s") + + # Aggregate results across multiple runs + if args.runs > 1: + final_proxy_results = aggregate_results(all_proxy_results) + final_provider_results = aggregate_results(all_provider_results) + print(f"\nAggregated results across {args.runs} runs:") + else: + # Use results from single run + if proxy_results is None or provider_results is None: + raise RuntimeError("Benchmark results not initialized") + final_proxy_results = proxy_results + final_provider_results = provider_results + print(f"\nResults:") + + # Print individual results + print_results("LiteLLM Proxy", final_proxy_results) + print_results("Direct Provider", final_provider_results) + + # Print comparison + compare_results(final_proxy_results, final_provider_results) + + # Show run-to-run variance if multiple runs + if args.runs > 1: + print_run_variance("LiteLLM Proxy", all_proxy_results) + print_run_variance("Direct Provider", all_provider_results) + + print(f"\n{'='*60}") + print("Benchmark complete!") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\nBenchmark interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n\nError running benchmark: {e}") + import traceback + traceback.print_exc() + sys.exit(1) +