From e284e4c75f10c036b74d7348b5cc29e587d6bbe9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 16 Mar 2026 16:48:10 +0000 Subject: [PATCH] Add benchmark: Standard LiteLLM Proxy vs fast-litellm accelerated proxy Benchmark comparing standard LiteLLM proxy against fast-litellm v0.1.6 (Rust-accelerated via PyO3) using network_mock mode to measure pure proxy overhead. Test setup: - Local PostgreSQL to eliminate remote DB latency - 4 uvicorn workers, concurrency levels 10/50/100/200 - 2000 requests per level, 5 runs (median) Key finding: fast-litellm v0.1.6 does not provide measurable throughput improvement for the proxy. Several Rust patches fail to apply against LiteLLM v1.82.2 (renamed/restructured targets). The proxy bottleneck is async I/O (auth DB lookups), not the CPU-bound operations that fast-litellm accelerates. Co-authored-by: Ishaan Jaff --- benchmark_config.yaml | 15 + benchmark_config_local.yaml | 16 + benchmark_results/BENCHMARK_REPORT.md | 109 ++++++ benchmark_results/comparison_report.txt | 60 +++ benchmark_results/fast_comprehensive.json | 242 ++++++++++++ benchmark_results/fast_comprehensive.txt | 24 ++ benchmark_results/fast_localdb_results.txt | 52 +++ benchmark_results/fast_results.txt | 52 +++ benchmark_results/standard_comprehensive.json | 242 ++++++++++++ benchmark_results/standard_comprehensive.txt | 24 ++ .../standard_localdb_results.txt | 52 +++ benchmark_results/standard_results.txt | 52 +++ scripts/comprehensive_benchmark.py | 153 ++++++++ scripts/generate_comparison.py | 103 ++++++ scripts/run_proxy_benchmark.py | 344 ++++++++++++++++++ scripts/run_proxy_benchmark.sh | 123 +++++++ scripts/start_fast_proxy.py | 7 + 17 files changed, 1670 insertions(+) create mode 100644 benchmark_config.yaml create mode 100644 benchmark_config_local.yaml create mode 100644 benchmark_results/BENCHMARK_REPORT.md create mode 100644 benchmark_results/comparison_report.txt create mode 100644 benchmark_results/fast_comprehensive.json create mode 100644 benchmark_results/fast_comprehensive.txt create mode 100644 benchmark_results/fast_localdb_results.txt create mode 100644 benchmark_results/fast_results.txt create mode 100644 benchmark_results/standard_comprehensive.json create mode 100644 benchmark_results/standard_comprehensive.txt create mode 100644 benchmark_results/standard_localdb_results.txt create mode 100644 benchmark_results/standard_results.txt create mode 100644 scripts/comprehensive_benchmark.py create mode 100644 scripts/generate_comparison.py create mode 100644 scripts/run_proxy_benchmark.py create mode 100755 scripts/run_proxy_benchmark.sh create mode 100644 scripts/start_fast_proxy.py diff --git a/benchmark_config.yaml b/benchmark_config.yaml new file mode 100644 index 00000000000..902152d0d34 --- /dev/null +++ b/benchmark_config.yaml @@ -0,0 +1,15 @@ +model_list: + - model_name: db-openai-endpoint + litellm_params: + model: openai/gpt-4o + api_key: "sk-fake-key" + api_base: "https://api.openai.com" + +litellm_settings: + network_mock: true + callbacks: [] + num_retries: 0 + request_timeout: 30 + +general_settings: + master_key: "sk-1234" diff --git a/benchmark_config_local.yaml b/benchmark_config_local.yaml new file mode 100644 index 00000000000..66b6badf41a --- /dev/null +++ b/benchmark_config_local.yaml @@ -0,0 +1,16 @@ +model_list: + - model_name: db-openai-endpoint + litellm_params: + model: openai/gpt-4o + api_key: "sk-fake-key" + api_base: "https://api.openai.com" + +litellm_settings: + network_mock: true + callbacks: [] + num_retries: 0 + request_timeout: 30 + +general_settings: + master_key: "sk-1234" + database_url: "postgresql://postgres:postgres@localhost:5432/litellm_benchmark" diff --git a/benchmark_results/BENCHMARK_REPORT.md b/benchmark_results/BENCHMARK_REPORT.md new file mode 100644 index 00000000000..8da98ea4032 --- /dev/null +++ b/benchmark_results/BENCHMARK_REPORT.md @@ -0,0 +1,109 @@ +# LiteLLM Proxy Benchmark: Standard vs fast-litellm + +## Overview + +This benchmark compares the **standard LiteLLM proxy** against the **[fast-litellm](https://github.com/neul-labs/fast-litellm) accelerated proxy** (v0.1.6), which uses Rust via PyO3 to speed up internal proxy operations. + +## Test Configuration + +| Parameter | Value | +|-----------|-------| +| Mode | `network_mock` (pure proxy overhead, no real API calls) | +| Database | Local PostgreSQL (eliminates network DB latency) | +| Workers | 4 uvicorn workers | +| Requests per level | 2,000 | +| Runs per level | 5 (median taken) | +| Concurrency levels | 10, 50, 100, 200 | +| Python | 3.12 | +| fast-litellm | v0.1.6 | +| LiteLLM | v1.82.2 | + +## Results + +### Throughput (requests/second) + +| Concurrency | Standard | fast-litellm | Speedup | +|:-----------:|:--------:|:------------:|:-------:| +| 10 | 822 rps | 510 rps | 0.62x | +| 50 | 493 rps | 478 rps | 0.97x | +| 100 | 809 rps | 822 rps | 1.02x | +| 200 | 530 rps | 585 rps | 1.10x | + +### Mean Latency (ms) + +| Concurrency | Standard | fast-litellm | Change | +|:-----------:|:--------:|:------------:|:------:| +| 10 | 12.0 ms | 19.3 ms | +60.8% | +| 50 | 99.9 ms | 101.8 ms | +1.9% | +| 100 | 117.8 ms | 115.3 ms | -2.2% | +| 200 | 313.9 ms | 323.1 ms | +2.9% | + +### Tail Latency + +| Concurrency | Metric | Standard | fast-litellm | Change | +|:-----------:|:------:|:--------:|:------------:|:------:| +| 100 | P95 | 238.6 ms | 219.1 ms | -8.2% | +| 100 | P99 | 397.7 ms | 248.2 ms | **-37.6%** | +| 200 | P95 | 604.9 ms | 575.3 ms | -4.9% | +| 200 | P99 | 853.0 ms | 874.7 ms | +2.5% | + +## Key Findings + +1. **No significant overall speedup**: fast-litellm v0.1.6 does not provide a measurable throughput improvement for the LiteLLM proxy in this benchmark. The overall throughput ratio is **0.90x** (fast-litellm / standard). + +2. **Low concurrency regression**: At concurrency=10, the standard proxy is **~38% faster** in throughput and has ~60% lower mean latency. This is likely due to PyO3 monkeypatching overhead being proportionally larger when individual request latency is very low. + +3. **High concurrency parity**: At concurrency=100-200, performance is essentially equivalent, with fast-litellm showing slight improvements in P99 tail latency at concurrency=100 (-37.6%). + +4. **Partial patch application**: Several fast-litellm patches failed to apply: + - `SimpleRateLimiter` class not found in litellm + - `SimpleConnectionPool` class not found in litellm + - `count_tokens_batch` function not found in litellm.utils + + These classes/functions may have been renamed or restructured in the current LiteLLM version (v1.82.2), limiting fast-litellm's effectiveness. + +5. **Bottleneck is I/O, not CPU**: The proxy's main bottleneck is async I/O (database auth lookups, even with local PostgreSQL), not CPU-bound Python operations. Rust-accelerating CPU-bound operations doesn't help when they're not on the critical path. + +## Interpretation + +The fast-litellm project targets specific CPU-bound operations (connection pooling, rate limiting, token counting) with Rust replacements. In the context of a full proxy request lifecycle — which includes FastAPI routing, authentication, database lookups, request/response transformation, and async I/O — these CPU-bound operations represent a small fraction of total request time. + +For the Rust acceleration to show meaningful improvements, the benchmark would need to: +- Exercise rate limiting under high cardinality (1000+ unique keys) +- Include large token counting workloads (the `network_mock` mode returns small mock responses) +- Use the specific connection pooling patterns that fast-litellm optimizes + +## How to Reproduce + +```bash +# Install dependencies +poetry install +poetry run pip install fast-litellm aiohttp + +# Install and start local PostgreSQL +sudo apt-get install -y postgresql +sudo pg_ctlcluster 16 main start +sudo -u postgres createdb litellm_benchmark +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + +# Run standard proxy benchmark +DATABASE_URL="postgresql://postgres:postgres@localhost:5432/litellm_benchmark" \ + poetry run litellm --config benchmark_config_local.yaml --port 4000 --num_workers 4 & +# Wait for health check... +poetry run python scripts/comprehensive_benchmark.py \ + --url "http://localhost:4000/chat/completions" \ + --label "Standard LiteLLM Proxy" \ + --output benchmark_results/standard_comprehensive.json + +# Run fast-litellm proxy benchmark +DATABASE_URL="postgresql://postgres:postgres@localhost:5432/litellm_benchmark" \ + poetry run python scripts/start_fast_proxy.py --config benchmark_config_local.yaml --port 4001 --num_workers 4 & +# Wait for health check... +poetry run python scripts/comprehensive_benchmark.py \ + --url "http://localhost:4001/chat/completions" \ + --label "fast-litellm Accelerated Proxy" \ + --output benchmark_results/fast_comprehensive.json + +# Generate comparison +poetry run python scripts/generate_comparison.py +``` diff --git a/benchmark_results/comparison_report.txt b/benchmark_results/comparison_report.txt new file mode 100644 index 00000000000..deb525af026 --- /dev/null +++ b/benchmark_results/comparison_report.txt @@ -0,0 +1,60 @@ +================================================================================ + BENCHMARK REPORT: Standard LiteLLM Proxy vs fast-litellm Accelerated Proxy +================================================================================ + + Test Configuration: + - Requests per concurrency level: 2000 + - Runs per level (median taken): 5 + - Mode: network_mock (pure proxy overhead, no real API calls) + - Database: Local PostgreSQL (eliminates network DB latency) + - Workers: 4 uvicorn workers + + ---------------------------------------------------------------------------- + Metric | Conc | Standard | fast-litellm | Diff + ---------------------------------------------------------------------------- + Throughput | 10 | 822.2 rps| 510.5 rps| -37.9% + Mean latency | 10 | 12.0 ms | 19.3 ms | +60.8% + P50 latency | 10 | 10.5 ms | 13.7 ms | +29.7% + P95 latency | 10 | 28.7 ms | 31.6 ms | +10.0% + P99 latency | 10 | 36.7 ms | 36.0 ms | -2.0% + ---------------------------------------------------------------------------- + Throughput | 50 | 493.1 rps| 478.3 rps| -3.0% + Mean latency | 50 | 99.9 ms | 101.8 ms | +1.9% + P50 latency | 50 | 41.1 ms | 58.6 ms | +42.6% + P95 latency | 50 | 247.8 ms | 146.0 ms | -41.1% + P99 latency | 50 | 315.4 ms | 847.4 ms | +168.7% + ---------------------------------------------------------------------------- + Throughput | 100 | 808.6 rps| 822.5 rps| +1.7% + Mean latency | 100 | 117.8 ms | 115.3 ms | -2.2% + P50 latency | 100 | 109.3 ms | 112.5 ms | +3.0% + P95 latency | 100 | 238.6 ms | 219.1 ms | -8.2% + P99 latency | 100 | 397.7 ms | 248.2 ms | -37.6% + ---------------------------------------------------------------------------- + Throughput | 200 | 529.8 rps| 584.7 rps| +10.4% + Mean latency | 200 | 313.9 ms | 323.1 ms | +2.9% + P50 latency | 200 | 223.5 ms | 237.8 ms | +6.4% + P95 latency | 200 | 604.9 ms | 575.3 ms | -4.9% + P99 latency | 200 | 853.0 ms | 874.7 ms | +2.5% + ---------------------------------------------------------------------------- + + SUMMARY TABLE (Throughput & Mean Latency) + + Concurrency | Std Throughput | Fast Throughput | Std Mean | Fast Mean | Speedup + -------------+-----------------+------------------+------------+-------------+--------- + 10 | 822 rps | 510 rps | 12.0 ms | 19.3 ms | 0.62x + 50 | 493 rps | 478 rps | 99.9 ms | 101.8 ms | 0.97x + 100 | 809 rps | 822 rps | 117.8 ms | 115.3 ms | 1.02x + 200 | 530 rps | 585 rps | 313.9 ms | 323.1 ms | 1.10x + + KEY FINDINGS: + - Overall throughput ratio: 0.90x (fast-litellm / standard) + - Avg mean latency: standard=135.9ms, fast-litellm=139.9ms (+2.9%) + + NOTES: + - network_mock mode eliminates real API calls, measuring pure proxy overhead + - Local PostgreSQL eliminates network DB latency from the measurement + - fast-litellm v0.1.6 with Rust acceleration via PyO3 + - Results may vary depending on hardware, OS, and Python version + - fast-litellm patches: routing, token_counting, rate_limiting, connection_pooling + - Some patches (SimpleRateLimiter, SimpleConnectionPool, count_tokens_batch) + failed to apply since the target classes/functions were not found in this version diff --git a/benchmark_results/fast_comprehensive.json b/benchmark_results/fast_comprehensive.json new file mode 100644 index 00000000000..d6b4610d799 --- /dev/null +++ b/benchmark_results/fast_comprehensive.json @@ -0,0 +1,242 @@ +{ + "label": "fast-litellm Accelerated Proxy", + "url": "http://localhost:4001/chat/completions", + "requests_per_level": 2000, + "runs_per_level": 5, + "results": { + "10": { + "concurrency": 10, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 19.327863879982033, + "p50_ms": 13.653956004418433, + "p95_ms": 31.609852012479678, + "p99_ms": 35.96846302389167, + "throughput_rps": 510.4670586342504, + "total_failures": 0 + }, + "50": { + "concurrency": 50, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 101.84724367558374, + "p50_ms": 58.5627639957238, + "p95_ms": 146.02366599137895, + "p99_ms": 847.3685620119795, + "throughput_rps": 478.3279305570934, + "total_failures": 0 + }, + "100": { + "concurrency": 100, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 115.28849936269398, + "p50_ms": 112.52534601953812, + "p95_ms": 219.08276100293733, + "p99_ms": 248.19156399462372, + "throughput_rps": 822.4805773698818, + "total_failures": 0 + }, + "200": { + "concurrency": 200, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 323.10731103419675, + "p50_ms": 237.76277599972673, + "p95_ms": 575.2935209893622, + "p99_ms": 874.7407619957812, + "throughput_rps": 584.7440635197701, + "total_failures": 0 + } + }, + "per_run_details": { + "10": [ + { + "mean_ms": 23.18634218981606, + "p50_ms": 15.95886200084351, + "p95_ms": 55.68123300326988, + "p99_ms": 74.374734016601, + "throughput_rps": 428.4385588514411, + "failures": 0, + "wall_time_s": 4.668113918974996 + }, + { + "mean_ms": 19.839967437670566, + "p50_ms": 10.402254993095994, + "p95_ms": 55.934295000042766, + "p99_ms": 72.76101299794391, + "throughput_rps": 498.67653305225906, + "failures": 0, + "wall_time_s": 4.010615834995406 + }, + { + "mean_ms": 13.30488502755179, + "p50_ms": 11.192481993930414, + "p95_ms": 22.5485069968272, + "p99_ms": 31.405346002429724, + "throughput_rps": 741.1493649338364, + "failures": 0, + "wall_time_s": 2.6985113859991543 + }, + { + "mean_ms": 13.51975990604842, + "p50_ms": 13.653956004418433, + "p95_ms": 19.449116982286796, + "p99_ms": 27.599092019954696, + "throughput_rps": 730.0748197386172, + "failures": 0, + "wall_time_s": 2.7394452540029306 + }, + { + "mean_ms": 19.327863879982033, + "p50_ms": 28.204379021190107, + "p95_ms": 31.609852012479678, + "p99_ms": 35.96846302389167, + "throughput_rps": 510.4670586342504, + "failures": 0, + "wall_time_s": 3.917980536003597 + } + ], + "50": [ + { + "mean_ms": 74.93787815775431, + "p50_ms": 72.56572600454092, + "p95_ms": 133.94331000745296, + "p99_ms": 190.43723001959734, + "throughput_rps": 629.0957736430437, + "failures": 0, + "wall_time_s": 3.179166168003576 + }, + { + "mean_ms": 101.84724367558374, + "p50_ms": 26.30606698221527, + "p95_ms": 314.8379750200547, + "p99_ms": 847.3685620119795, + "throughput_rps": 478.3279305570934, + "failures": 0, + "wall_time_s": 4.181231896014651 + }, + { + "mean_ms": 111.5506120112841, + "p50_ms": 42.62882602051832, + "p95_ms": 334.96011499664746, + "p99_ms": 1504.1227279871237, + "throughput_rps": 439.82330967625785, + "failures": 0, + "wall_time_s": 4.547280591999879 + }, + { + "mean_ms": 108.23956319814897, + "p50_ms": 96.38116601854563, + "p95_ms": 146.02366599137895, + "p99_ms": 1172.680377989309, + "throughput_rps": 456.68766261300726, + "failures": 0, + "wall_time_s": 4.379360696009826 + }, + { + "mean_ms": 58.56425296376983, + "p50_ms": 58.5627639957238, + "p95_ms": 101.93852701922879, + "p99_ms": 160.97755800001323, + "throughput_rps": 721.4784446832903, + "failures": 0, + "wall_time_s": 2.7720855899970047 + } + ], + "100": [ + { + "mean_ms": 115.28849936269398, + "p50_ms": 107.53192499396391, + "p95_ms": 180.77384799835272, + "p99_ms": 227.18665798311122, + "throughput_rps": 822.4805773698818, + "failures": 0, + "wall_time_s": 2.4316683640063275 + }, + { + "mean_ms": 95.02974098436243, + "p50_ms": 78.62613600445911, + "p95_ms": 164.15538798901252, + "p99_ms": 241.78478700923733, + "throughput_rps": 1009.9281889437465, + "failures": 0, + "wall_time_s": 1.9803388220025226 + }, + { + "mean_ms": 109.37638862330641, + "p50_ms": 112.52534601953812, + "p95_ms": 219.08276100293733, + "p99_ms": 248.19156399462372, + "throughput_rps": 887.0262508489574, + "failures": 0, + "wall_time_s": 2.254724703001557 + }, + { + "mean_ms": 167.24811069393763, + "p50_ms": 147.50073000323027, + "p95_ms": 253.14448101562448, + "p99_ms": 991.7771610198542, + "throughput_rps": 543.6423822215804, + "failures": 0, + "wall_time_s": 3.678889036993496 + }, + { + "mean_ms": 172.56855782800994, + "p50_ms": 145.41222102707252, + "p95_ms": 320.6469359865878, + "p99_ms": 615.7889599853661, + "throughput_rps": 528.3521703882573, + "failures": 0, + "wall_time_s": 3.7853539969946723 + } + ], + "200": [ + { + "mean_ms": 236.1762486763182, + "p50_ms": 237.76277599972673, + "p95_ms": 324.4085389887914, + "p99_ms": 535.5818870011717, + "throughput_rps": 805.4813031656818, + "failures": 0, + "wall_time_s": 2.4829874910064973 + }, + { + "mean_ms": 366.90505366899015, + "p50_ms": 221.1398319923319, + "p95_ms": 1198.3245269802865, + "p99_ms": 1213.15528897685, + "throughput_rps": 526.1687766715256, + "failures": 0, + "wall_time_s": 3.80106172899832 + }, + { + "mean_ms": 327.28831522582914, + "p50_ms": 273.14443202340044, + "p95_ms": 575.2935209893622, + "p99_ms": 2548.432882002089, + "throughput_rps": 584.7440635197701, + "failures": 0, + "wall_time_s": 3.420299794001039 + }, + { + "mean_ms": 323.10731103419675, + "p50_ms": 265.33138399827294, + "p95_ms": 701.2160860176664, + "p99_ms": 874.7407619957812, + "throughput_rps": 562.1863416933975, + "failures": 0, + "wall_time_s": 3.5575392920000013 + }, + { + "mean_ms": 203.6881105282373, + "p50_ms": 156.90035501029342, + "p95_ms": 415.1290970039554, + "p99_ms": 528.8655159820337, + "throughput_rps": 879.3393143461739, + "failures": 0, + "wall_time_s": 2.274434871011181 + } + ] + } +} \ No newline at end of file diff --git a/benchmark_results/fast_comprehensive.txt b/benchmark_results/fast_comprehensive.txt new file mode 100644 index 00000000000..1ca0c2a84b8 --- /dev/null +++ b/benchmark_results/fast_comprehensive.txt @@ -0,0 +1,24 @@ + +====================================================================== + Benchmarking: fast-litellm Accelerated Proxy + URL: http://localhost:4001/chat/completions + Requests per level: 2000, Runs per level: 5 + Concurrency levels: [10, 50, 100, 200] +====================================================================== + + Concurrency=10 ... throughput=510 req/s, mean=19.3ms, p50=13.7ms, p95=31.6ms, p99=36.0ms + + Concurrency=50 ... throughput=478 req/s, mean=101.8ms, p50=58.6ms, p95=146.0ms, p99=847.4ms + + Concurrency=100 ... throughput=822 req/s, mean=115.3ms, p50=112.5ms, p95=219.1ms, p99=248.2ms + + Concurrency=200 ... throughput=585 req/s, mean=323.1ms, p50=237.8ms, p95=575.3ms, p99=874.7ms + + Conc | Throughput | Mean | P50 | P95 | P99 + -------+--------------+------------+------------+------------+----------- + 10 | 510 rps | 19.3 ms | 13.7 ms | 31.6 ms | 36.0 ms + 50 | 478 rps | 101.8 ms | 58.6 ms | 146.0 ms | 847.4 ms + 100 | 822 rps | 115.3 ms | 112.5 ms | 219.1 ms | 248.2 ms + 200 | 585 rps | 323.1 ms | 237.8 ms | 575.3 ms | 874.7 ms + + Results saved to benchmark_results/fast_comprehensive.json diff --git a/benchmark_results/fast_localdb_results.txt b/benchmark_results/fast_localdb_results.txt new file mode 100644 index 00000000000..be6819ed08b --- /dev/null +++ b/benchmark_results/fast_localdb_results.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4001/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 3.71s + Throughput: 540 req/s + Mean: 175.84 ms + P50: 175.87 ms + P95: 257.85 ms + P99: 344.06 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 3.72s + Throughput: 538 req/s + Mean: 181.22 ms + P50: 181.43 ms + P95: 277.29 ms + P99: 335.21 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 4.40s + Throughput: 455 req/s + Mean: 209.70 ms + P50: 159.25 ms + P95: 388.17 ms + P99: 1555.72 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 511 req/s (avg across runs) + Mean: 188.92 ms + P50: 172.78 ms + P95: 301.14 ms + P99: 396.76 ms + + Run-to-run variance: + Latency CoV: 9.6% + Throughput CoV: 9.5% diff --git a/benchmark_results/fast_results.txt b/benchmark_results/fast_results.txt new file mode 100644 index 00000000000..8365b928cb0 --- /dev/null +++ b/benchmark_results/fast_results.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4001/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.33s + Throughput: 79 req/s + Mean: 1246.09 ms + P50: 1202.74 ms + P95: 1807.99 ms + P99: 2234.76 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 26.18s + Throughput: 76 req/s + Mean: 1272.32 ms + P50: 1261.65 ms + P95: 2184.16 ms + P99: 2406.41 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.95s + Throughput: 77 req/s + Mean: 1268.84 ms + P50: 1178.08 ms + P95: 1719.05 ms + P99: 3159.39 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 77 req/s (avg across runs) + Mean: 1262.42 ms + P50: 1209.21 ms + P95: 2094.34 ms + P99: 2406.52 ms + + Run-to-run variance: + Latency CoV: 1.1% + Throughput CoV: 1.7% diff --git a/benchmark_results/standard_comprehensive.json b/benchmark_results/standard_comprehensive.json new file mode 100644 index 00000000000..432a0c839c6 --- /dev/null +++ b/benchmark_results/standard_comprehensive.json @@ -0,0 +1,242 @@ +{ + "label": "Standard LiteLLM Proxy", + "url": "http://localhost:4000/chat/completions", + "requests_per_level": 2000, + "runs_per_level": 5, + "results": { + "10": { + "concurrency": 10, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 12.02115576264623, + "p50_ms": 10.528612008783966, + "p95_ms": 28.73887598980218, + "p99_ms": 36.71442501945421, + "throughput_rps": 822.1665666545626, + "total_failures": 0 + }, + "50": { + "concurrency": 50, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 99.90518795080425, + "p50_ms": 41.076640016399324, + "p95_ms": 247.75799599592574, + "p99_ms": 315.38213198655285, + "throughput_rps": 493.12067782378597, + "total_failures": 0 + }, + "100": { + "concurrency": 100, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 117.83136430148443, + "p50_ms": 109.26515198661946, + "p95_ms": 238.6146899953019, + "p99_ms": 397.6895730011165, + "throughput_rps": 808.5763592090435, + "total_failures": 0 + }, + "200": { + "concurrency": 200, + "requests_per_run": 2000, + "runs": 5, + "mean_ms": 313.9467866400082, + "p50_ms": 223.510087991599, + "p95_ms": 604.9391269916669, + "p99_ms": 852.9933450045064, + "throughput_rps": 529.8286974891981, + "total_failures": 0 + } + }, + "per_run_details": { + "10": [ + { + "mean_ms": 15.081677895708708, + "p50_ms": 14.057211024919525, + "p95_ms": 33.032197010470554, + "p99_ms": 48.97488999995403, + "throughput_rps": 658.4915712949247, + "failures": 0, + "wall_time_s": 3.037244646984618 + }, + { + "mean_ms": 10.988114421183127, + "p50_ms": 10.128869995241985, + "p95_ms": 22.21098798327148, + "p99_ms": 33.59383699717, + "throughput_rps": 896.8390292765534, + "failures": 0, + "wall_time_s": 2.2300545969919767 + }, + { + "mean_ms": 11.131436991330702, + "p50_ms": 10.528612008783966, + "p95_ms": 18.001510994508862, + "p99_ms": 26.13307099090889, + "throughput_rps": 886.175886616014, + "failures": 0, + "wall_time_s": 2.25688831100706 + }, + { + "mean_ms": 12.02115576264623, + "p50_ms": 7.877630996517837, + "p95_ms": 28.73887598980218, + "p99_ms": 36.71442501945421, + "throughput_rps": 822.1665666545626, + "failures": 0, + "wall_time_s": 2.432597092993092 + }, + { + "mean_ms": 22.377430284250295, + "p50_ms": 19.7191089973785, + "p95_ms": 42.6688689913135, + "p99_ms": 80.4927260032855, + "throughput_rps": 443.9983865944926, + "failures": 0, + "wall_time_s": 4.504520873015281 + } + ], + "50": [ + { + "mean_ms": 99.90518795080425, + "p50_ms": 88.35588800138794, + "p95_ms": 247.75799599592574, + "p99_ms": 315.38213198655285, + "throughput_rps": 493.12067782378597, + "failures": 0, + "wall_time_s": 4.055802341987146 + }, + { + "mean_ms": 108.03127349969873, + "p50_ms": 108.3063569967635, + "p95_ms": 251.70899598742835, + "p99_ms": 298.62097700242884, + "throughput_rps": 454.2840462880531, + "failures": 0, + "wall_time_s": 4.402531888015801 + }, + { + "mean_ms": 125.18951635381381, + "p50_ms": 41.076640016399324, + "p95_ms": 363.1205649871845, + "p99_ms": 1130.8167510142084, + "throughput_rps": 395.4009453871131, + "failures": 0, + "wall_time_s": 5.0581568489724305 + }, + { + "mean_ms": 62.59332004499448, + "p50_ms": 32.97114497399889, + "p95_ms": 95.78598602092825, + "p99_ms": 835.1694949960802, + "throughput_rps": 785.8441310480217, + "failures": 0, + "wall_time_s": 2.545033959002467 + }, + { + "mean_ms": 52.54145934250846, + "p50_ms": 25.87911201408133, + "p95_ms": 153.29675501561724, + "p99_ms": 181.2838460027706, + "throughput_rps": 925.4785568806632, + "failures": 0, + "wall_time_s": 2.16104412698769 + } + ], + "100": [ + { + "mean_ms": 96.1130797794176, + "p50_ms": 89.77022499311715, + "p95_ms": 156.3608920259867, + "p99_ms": 179.07995902351104, + "throughput_rps": 1013.9009012410786, + "failures": 0, + "wall_time_s": 1.9725793690013234 + }, + { + "mean_ms": 102.29677444961271, + "p50_ms": 86.51860800455324, + "p95_ms": 183.0326660128776, + "p99_ms": 255.11633299174719, + "throughput_rps": 937.1696601389452, + "failures": 0, + "wall_time_s": 2.1340853050060105 + }, + { + "mean_ms": 117.83136430148443, + "p50_ms": 109.26515198661946, + "p95_ms": 238.6146899953019, + "p99_ms": 446.90601699403487, + "throughput_rps": 808.5763592090435, + "failures": 0, + "wall_time_s": 2.473483150009997 + }, + { + "mean_ms": 184.86301546664617, + "p50_ms": 158.55945998919196, + "p95_ms": 345.65597597975284, + "p99_ms": 397.6895730011165, + "throughput_rps": 531.9364077575092, + "failures": 0, + "wall_time_s": 3.759847926994553 + }, + { + "mean_ms": 137.75746933375194, + "p50_ms": 125.74252701597288, + "p95_ms": 239.3846439954359, + "p99_ms": 852.2111640195362, + "throughput_rps": 702.4768886220655, + "failures": 0, + "wall_time_s": 2.8470687539957 + } + ], + "200": [ + { + "mean_ms": 313.9467866400082, + "p50_ms": 214.22403698670678, + "p95_ms": 730.6632529944181, + "p99_ms": 1412.697333988035, + "throughput_rps": 529.8286974891981, + "failures": 0, + "wall_time_s": 3.774804968998069 + }, + { + "mean_ms": 358.4556060676259, + "p50_ms": 344.2879210051615, + "p95_ms": 535.3092000004835, + "p99_ms": 624.3046080053318, + "throughput_rps": 527.666044392877, + "failures": 0, + "wall_time_s": 3.79027610598132 + }, + { + "mean_ms": 383.8176259584434, + "p50_ms": 325.75000898214057, + "p95_ms": 1221.5972899866756, + "p99_ms": 1529.7934050031472, + "throughput_rps": 500.4626059866745, + "failures": 0, + "wall_time_s": 3.9963025730103254 + }, + { + "mean_ms": 257.0812514499412, + "p50_ms": 223.510087991599, + "p95_ms": 559.0225839987397, + "p99_ms": 670.4310380155221, + "throughput_rps": 719.7306778741312, + "failures": 0, + "wall_time_s": 2.7788172180007678 + }, + { + "mean_ms": 217.86537275278533, + "p50_ms": 162.15116999228485, + "p95_ms": 604.9391269916669, + "p99_ms": 852.9933450045064, + "throughput_rps": 879.5233261897602, + "failures": 0, + "wall_time_s": 2.273959018988535 + } + ] + } +} \ No newline at end of file diff --git a/benchmark_results/standard_comprehensive.txt b/benchmark_results/standard_comprehensive.txt new file mode 100644 index 00000000000..188a0ed38cb --- /dev/null +++ b/benchmark_results/standard_comprehensive.txt @@ -0,0 +1,24 @@ + +====================================================================== + Benchmarking: Standard LiteLLM Proxy + URL: http://localhost:4000/chat/completions + Requests per level: 2000, Runs per level: 5 + Concurrency levels: [10, 50, 100, 200] +====================================================================== + + Concurrency=10 ... throughput=822 req/s, mean=12.0ms, p50=10.5ms, p95=28.7ms, p99=36.7ms + + Concurrency=50 ... throughput=493 req/s, mean=99.9ms, p50=41.1ms, p95=247.8ms, p99=315.4ms + + Concurrency=100 ... throughput=809 req/s, mean=117.8ms, p50=109.3ms, p95=238.6ms, p99=397.7ms + + Concurrency=200 ... throughput=530 req/s, mean=313.9ms, p50=223.5ms, p95=604.9ms, p99=853.0ms + + Conc | Throughput | Mean | P50 | P95 | P99 + -------+--------------+------------+------------+------------+----------- + 10 | 822 rps | 12.0 ms | 10.5 ms | 28.7 ms | 36.7 ms + 50 | 493 rps | 99.9 ms | 41.1 ms | 247.8 ms | 315.4 ms + 100 | 809 rps | 117.8 ms | 109.3 ms | 238.6 ms | 397.7 ms + 200 | 530 rps | 313.9 ms | 223.5 ms | 604.9 ms | 853.0 ms + + Results saved to benchmark_results/standard_comprehensive.json diff --git a/benchmark_results/standard_localdb_results.txt b/benchmark_results/standard_localdb_results.txt new file mode 100644 index 00000000000..3e67a793335 --- /dev/null +++ b/benchmark_results/standard_localdb_results.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 2.77s + Throughput: 721 req/s + Mean: 135.88 ms + P50: 107.83 ms + P95: 319.08 ms + P99: 524.78 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 1.86s + Throughput: 1078 req/s + Mean: 88.94 ms + P50: 87.47 ms + P95: 119.74 ms + P99: 145.89 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 3.82s + Throughput: 524 req/s + Mean: 180.33 ms + P50: 133.64 ms + P95: 417.23 ms + P99: 1316.66 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 774 req/s (avg across runs) + Mean: 135.05 ms + P50: 101.34 ms + P95: 347.16 ms + P99: 668.45 ms + + Run-to-run variance: + Latency CoV: 33.8% + Throughput CoV: 36.3% diff --git a/benchmark_results/standard_results.txt b/benchmark_results/standard_results.txt new file mode 100644 index 00000000000..31b40dbb516 --- /dev/null +++ b/benchmark_results/standard_results.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 26.21s + Throughput: 76 req/s + Mean: 1259.84 ms + P50: 1109.71 ms + P95: 2697.35 ms + P99: 5207.40 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.84s + Throughput: 77 req/s + Mean: 1254.33 ms + P50: 1206.06 ms + P95: 1846.73 ms + P99: 2256.60 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 26.04s + Throughput: 77 req/s + Mean: 1264.60 ms + P50: 995.05 ms + P95: 2233.99 ms + P99: 3761.21 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 77 req/s (avg across runs) + Mean: 1259.59 ms + P50: 1123.06 ms + P95: 2219.42 ms + P99: 3984.95 ms + + Run-to-run variance: + Latency CoV: 0.4% + Throughput CoV: 0.7% diff --git a/scripts/comprehensive_benchmark.py b/scripts/comprehensive_benchmark.py new file mode 100644 index 00000000000..66896619474 --- /dev/null +++ b/scripts/comprehensive_benchmark.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +""" +Comprehensive benchmark comparing Standard LiteLLM vs fast-litellm proxy. + +Tests multiple concurrency levels and produces a detailed comparison report. +""" + +import asyncio +import json +import statistics +import time + +import aiohttp + +REQUEST_BODY = { + "model": "db-openai-endpoint", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 100, + "user": "benchmark_user", +} + +HEADERS = { + "Authorization": "Bearer sk-1234", + "Content-Type": "application/json", +} + + +async def send_request(session, url, semaphore): + async with semaphore: + start = time.perf_counter() + try: + async with session.post(url, json=REQUEST_BODY, headers=HEADERS) as resp: + await resp.read() + elapsed = time.perf_counter() - start + return elapsed if resp.status == 200 else None + except Exception: + return None + + +async def run_benchmark(url: str, n_requests: int, max_concurrent: int, n_runs: int = 3): + all_results = [] + + for run_num in range(n_runs): + semaphore = asyncio.Semaphore(max_concurrent) + connector = aiohttp.TCPConnector( + limit=min(max_concurrent * 2, 500), + limit_per_host=max_concurrent, + force_close=False, + enable_cleanup_closed=True, + ) + async with aiohttp.ClientSession(connector=connector) as session: + warmup = min(50, n_requests // 4) + await asyncio.gather( + *[send_request(session, url, semaphore) for _ in range(warmup)] + ) + + wall_start = time.perf_counter() + results = await asyncio.gather( + *[send_request(session, url, semaphore) for _ in range(n_requests)] + ) + wall_elapsed = time.perf_counter() - wall_start + + latencies = sorted([r for r in results if r is not None]) + failures = sum(1 for r in results if r is None) + n = len(latencies) + + if n > 0: + run_result = { + "mean_ms": statistics.mean(latencies) * 1000, + "p50_ms": latencies[n // 2] * 1000, + "p95_ms": latencies[int(n * 0.95)] * 1000, + "p99_ms": latencies[int(n * 0.99)] * 1000, + "throughput_rps": n_requests / wall_elapsed, + "failures": failures, + "wall_time_s": wall_elapsed, + } + else: + run_result = { + "mean_ms": 0, "p50_ms": 0, "p95_ms": 0, "p99_ms": 0, + "throughput_rps": 0, "failures": n_requests, "wall_time_s": wall_elapsed, + } + all_results.append(run_result) + + means = [r["mean_ms"] for r in all_results if r["mean_ms"] > 0] + p50s = [r["p50_ms"] for r in all_results if r["p50_ms"] > 0] + p95s = [r["p95_ms"] for r in all_results if r["p95_ms"] > 0] + p99s = [r["p99_ms"] for r in all_results if r["p99_ms"] > 0] + thrpts = [r["throughput_rps"] for r in all_results if r["throughput_rps"] > 0] + + return { + "concurrency": max_concurrent, + "requests_per_run": n_requests, + "runs": n_runs, + "mean_ms": statistics.median(means) if means else 0, + "p50_ms": statistics.median(p50s) if p50s else 0, + "p95_ms": statistics.median(p95s) if p95s else 0, + "p99_ms": statistics.median(p99s) if p99s else 0, + "throughput_rps": statistics.median(thrpts) if thrpts else 0, + "total_failures": sum(r["failures"] for r in all_results), + "per_run": all_results, + } + + +async def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--url", required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--requests", type=int, default=2000) + parser.add_argument("--runs", type=int, default=3) + args = parser.parse_args() + + concurrency_levels = [10, 50, 100, 200] + results = {} + + print(f"\n{'='*70}") + print(f" Benchmarking: {args.label}") + print(f" URL: {args.url}") + print(f" Requests per level: {args.requests}, Runs per level: {args.runs}") + print(f" Concurrency levels: {concurrency_levels}") + print(f"{'='*70}") + + for conc in concurrency_levels: + print(f"\n Concurrency={conc} ...", end=" ", flush=True) + result = await run_benchmark(args.url, args.requests, conc, args.runs) + results[conc] = result + print(f"throughput={result['throughput_rps']:.0f} req/s, " + f"mean={result['mean_ms']:.1f}ms, p50={result['p50_ms']:.1f}ms, " + f"p95={result['p95_ms']:.1f}ms, p99={result['p99_ms']:.1f}ms") + + print(f"\n {'Conc':>6} | {'Throughput':>12} | {'Mean':>10} | {'P50':>10} | {'P95':>10} | {'P99':>10}") + print(f" {'-'*6}-+-{'-'*12}-+-{'-'*10}-+-{'-'*10}-+-{'-'*10}-+-{'-'*10}") + for conc in concurrency_levels: + r = results[conc] + print(f" {conc:>6} | {r['throughput_rps']:>9.0f} rps | {r['mean_ms']:>7.1f} ms | " + f"{r['p50_ms']:>7.1f} ms | {r['p95_ms']:>7.1f} ms | {r['p99_ms']:>7.1f} ms") + + output_data = { + "label": args.label, + "url": args.url, + "requests_per_level": args.requests, + "runs_per_level": args.runs, + "results": {str(k): {kk: vv for kk, vv in v.items() if kk != "per_run"} for k, v in results.items()}, + "per_run_details": {str(k): v["per_run"] for k, v in results.items()}, + } + with open(args.output, "w") as f: + json.dump(output_data, f, indent=2) + print(f"\n Results saved to {args.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/generate_comparison.py b/scripts/generate_comparison.py new file mode 100644 index 00000000000..7cbbc36e774 --- /dev/null +++ b/scripts/generate_comparison.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Generate a comparison report from benchmark JSON files.""" + +import json +import sys + + +def load_results(path): + with open(path) as f: + return json.load(f) + + +def main(): + standard = load_results("benchmark_results/standard_comprehensive.json") + fast = load_results("benchmark_results/fast_comprehensive.json") + + print("=" * 80) + print(" BENCHMARK REPORT: Standard LiteLLM Proxy vs fast-litellm Accelerated Proxy") + print("=" * 80) + print() + print(" Test Configuration:") + print(f" - Requests per concurrency level: {standard['requests_per_level']}") + print(f" - Runs per level (median taken): {standard['runs_per_level']}") + print(f" - Mode: network_mock (pure proxy overhead, no real API calls)") + print(f" - Database: Local PostgreSQL (eliminates network DB latency)") + print(f" - Workers: 4 uvicorn workers") + print() + + concurrency_levels = ["10", "50", "100", "200"] + + print(" " + "-" * 76) + print(f" {'Metric':<22} | {'Conc':>5} | {'Standard':>12} | {'fast-litellm':>14} | {'Diff':>10}") + print(" " + "-" * 76) + + for conc in concurrency_levels: + std_r = standard["results"][conc] + fast_r = fast["results"][conc] + + metrics = [ + ("Throughput", "throughput_rps", "rps", False), + ("Mean latency", "mean_ms", "ms", True), + ("P50 latency", "p50_ms", "ms", True), + ("P95 latency", "p95_ms", "ms", True), + ("P99 latency", "p99_ms", "ms", True), + ] + + for label, key, unit, lower_better in metrics: + std_val = std_r[key] + fast_val = fast_r[key] + if std_val > 0: + pct = ((fast_val - std_val) / std_val) * 100 + direction = "faster" if (pct < 0 and lower_better) or (pct > 0 and not lower_better) else "slower" + diff_str = f"{pct:+.1f}%" + else: + diff_str = "N/A" + + print(f" {label:<22} | {conc:>5} | {std_val:>9.1f} {unit:<3}| {fast_val:>11.1f} {unit:<3}| {diff_str:>10}") + + print(" " + "-" * 76) + + print() + print(" SUMMARY TABLE (Throughput & Mean Latency)") + print() + print(f" {'Concurrency':>12} | {'Std Throughput':>15} | {'Fast Throughput':>16} | {'Std Mean':>10} | {'Fast Mean':>11} | {'Speedup':>8}") + print(f" {'-'*12}-+-{'-'*15}-+-{'-'*16}-+-{'-'*10}-+-{'-'*11}-+-{'-'*8}") + + for conc in concurrency_levels: + std_r = standard["results"][conc] + fast_r = fast["results"][conc] + + std_tp = std_r["throughput_rps"] + fast_tp = fast_r["throughput_rps"] + speedup = fast_tp / std_tp if std_tp > 0 else 0 + + print(f" {conc:>12} | {std_tp:>12.0f} rps | {fast_tp:>13.0f} rps | " + f"{std_r['mean_ms']:>7.1f} ms | {fast_r['mean_ms']:>8.1f} ms | {speedup:>6.2f}x") + + print() + print(" KEY FINDINGS:") + + avg_std_tp = sum(standard["results"][c]["throughput_rps"] for c in concurrency_levels) / len(concurrency_levels) + avg_fast_tp = sum(fast["results"][c]["throughput_rps"] for c in concurrency_levels) / len(concurrency_levels) + overall_speedup = avg_fast_tp / avg_std_tp if avg_std_tp > 0 else 0 + + avg_std_mean = sum(standard["results"][c]["mean_ms"] for c in concurrency_levels) / len(concurrency_levels) + avg_fast_mean = sum(fast["results"][c]["mean_ms"] for c in concurrency_levels) / len(concurrency_levels) + latency_diff_pct = ((avg_fast_mean - avg_std_mean) / avg_std_mean) * 100 + + print(f" - Overall throughput ratio: {overall_speedup:.2f}x (fast-litellm / standard)") + print(f" - Avg mean latency: standard={avg_std_mean:.1f}ms, fast-litellm={avg_fast_mean:.1f}ms ({latency_diff_pct:+.1f}%)") + print() + print(" NOTES:") + print(" - network_mock mode eliminates real API calls, measuring pure proxy overhead") + print(" - Local PostgreSQL eliminates network DB latency from the measurement") + print(" - fast-litellm v0.1.6 with Rust acceleration via PyO3") + print(" - Results may vary depending on hardware, OS, and Python version") + print(" - fast-litellm patches: routing, token_counting, rate_limiting, connection_pooling") + print(" - Some patches (SimpleRateLimiter, SimpleConnectionPool, count_tokens_batch)") + print(" failed to apply since the target classes/functions were not found in this version") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_proxy_benchmark.py b/scripts/run_proxy_benchmark.py new file mode 100644 index 00000000000..25c2bd8dd20 --- /dev/null +++ b/scripts/run_proxy_benchmark.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +""" +Benchmark: Standard LiteLLM Proxy vs fast-litellm Accelerated Proxy + +Measures pure proxy overhead using network_mock mode (no real API calls). +Tests both configurations back-to-back and produces a comparison report. +""" + +import argparse +import asyncio +import json +import os +import signal +import subprocess +import sys +import statistics +import time + +import aiohttp + +REQUEST_BODY = { + "model": "db-openai-endpoint", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 100, + "user": "benchmark_user", +} + +HEADERS = { + "Authorization": "Bearer sk-1234", + "Content-Type": "application/json", +} + + +async def wait_for_proxy(url: str, timeout: int = 120) -> bool: + health_url = url.rsplit("/", 1)[0] + "/health" + start = time.time() + while time.time() - start < timeout: + try: + async with aiohttp.ClientSession() as session: + async with session.get(health_url, timeout=aiohttp.ClientTimeout(total=5)) as resp: + if resp.status == 200: + return True + except Exception: + pass + await asyncio.sleep(1) + return False + + +async def send_request(session, url, semaphore): + async with semaphore: + start = time.perf_counter() + try: + async with session.post(url, json=REQUEST_BODY, headers=HEADERS) as resp: + await resp.read() + elapsed = time.perf_counter() - start + return elapsed if resp.status == 200 else None + except Exception: + return None + + +async def run_benchmark(url: str, n_requests: int, max_concurrent: int): + semaphore = asyncio.Semaphore(max_concurrent) + connector = aiohttp.TCPConnector( + limit=min(max_concurrent * 2, 500), + limit_per_host=max_concurrent, + force_close=False, + enable_cleanup_closed=True, + ) + async with aiohttp.ClientSession(connector=connector) as session: + warmup_count = min(100, n_requests // 2) + await asyncio.gather( + *[send_request(session, url, semaphore) for _ in range(warmup_count)] + ) + + wall_start = time.perf_counter() + results = await asyncio.gather( + *[send_request(session, url, semaphore) for _ in range(n_requests)] + ) + wall_elapsed = time.perf_counter() - wall_start + + latencies = sorted([r for r in results if r is not None]) + failures = sum(1 for r in results if r is None) + + if not latencies: + return { + "mean_ms": 0, "p50_ms": 0, "p95_ms": 0, "p99_ms": 0, + "min_ms": 0, "max_ms": 0, + "throughput_rps": 0, "failures": n_requests, + "wall_time_s": wall_elapsed, "n_requests": n_requests, + "max_concurrent": max_concurrent, "latencies": [], + } + + n = len(latencies) + return { + "mean_ms": statistics.mean(latencies) * 1000, + "p50_ms": latencies[n // 2] * 1000, + "p95_ms": latencies[int(n * 0.95)] * 1000, + "p99_ms": latencies[int(n * 0.99)] * 1000, + "min_ms": latencies[0] * 1000, + "max_ms": latencies[-1] * 1000, + "throughput_rps": n_requests / wall_elapsed, + "failures": failures, + "wall_time_s": wall_elapsed, + "n_requests": n_requests, + "max_concurrent": max_concurrent, + "latencies": latencies, + } + + +def aggregate_runs(results: list[dict]) -> dict: + all_latencies = [] + for r in results: + all_latencies.extend(r["latencies"]) + all_latencies.sort() + + total_failures = sum(r["failures"] for r in results) + total_requests = sum(r["n_requests"] for r in results) + n = len(all_latencies) + + if not all_latencies: + return {"error": f"All {total_requests} requests failed"} + + return { + "total_requests": total_requests, + "total_failures": total_failures, + "mean_ms": statistics.mean(all_latencies) * 1000, + "p50_ms": all_latencies[n // 2] * 1000, + "p95_ms": all_latencies[int(n * 0.95)] * 1000, + "p99_ms": all_latencies[int(n * 0.99)] * 1000, + "min_ms": all_latencies[0] * 1000, + "max_ms": all_latencies[-1] * 1000, + "avg_throughput_rps": statistics.mean(r["throughput_rps"] for r in results), + } + + +def print_results(label: str, agg: dict): + print(f"\n{'='*60}") + print(f" {label}") + print(f"{'='*60}") + if "error" in agg: + print(f" ERROR: {agg['error']}") + return + print(f" Total Requests: {agg['total_requests']} (failures: {agg['total_failures']})") + print(f" Throughput: {agg['avg_throughput_rps']:.0f} req/s") + print(f" Mean latency: {agg['mean_ms']:.2f} ms") + print(f" P50 latency: {agg['p50_ms']:.2f} ms") + print(f" P95 latency: {agg['p95_ms']:.2f} ms") + print(f" P99 latency: {agg['p99_ms']:.2f} ms") + print(f" Min latency: {agg['min_ms']:.2f} ms") + print(f" Max latency: {agg['max_ms']:.2f} ms") + + +def print_comparison(standard: dict, fast: dict): + print(f"\n{'='*60}") + print(f" COMPARISON: Standard vs fast-litellm") + print(f"{'='*60}") + + if "error" in standard or "error" in fast: + print(" Cannot compare — one or both benchmarks had all failures.") + return + + metrics = [ + ("Mean latency", "mean_ms", "ms", True), + ("P50 latency", "p50_ms", "ms", True), + ("P95 latency", "p95_ms", "ms", True), + ("P99 latency", "p99_ms", "ms", True), + ("Throughput", "avg_throughput_rps", "req/s", False), + ] + + print(f" {'Metric':<18} {'Standard':>12} {'fast-litellm':>14} {'Change':>12}") + print(f" {'-'*18} {'-'*12} {'-'*14} {'-'*12}") + + for label, key, unit, lower_is_better in metrics: + std_val = standard[key] + fast_val = fast[key] + + if std_val == 0: + pct = "N/A" + else: + change = ((fast_val - std_val) / std_val) * 100 + pct = f"{change:+.1f}%" + + print(f" {label:<18} {std_val:>10.2f} {unit[0]} {fast_val:>12.2f} {unit[0]} {pct:>10}") + + if standard.get("avg_throughput_rps", 0) > 0: + speedup = fast.get("avg_throughput_rps", 0) / standard["avg_throughput_rps"] + print(f"\n Throughput multiplier: {speedup:.2f}x") + + if standard.get("mean_ms", 0) > 0: + latency_reduction = (1 - fast.get("mean_ms", 0) / standard["mean_ms"]) * 100 + print(f" Mean latency reduction: {latency_reduction:.1f}%") + + +def start_proxy(config_path: str, port: int, use_fast_litellm: bool, num_workers: int) -> subprocess.Popen: + env = os.environ.copy() + env["LITELLM_LOG"] = "ERROR" + + if use_fast_litellm: + cmd = [ + sys.executable, "-c", + f"import fast_litellm; from litellm.proxy.proxy_cli import run_server; run_server()", + "--config", config_path, + "--port", str(port), + "--num_workers", str(num_workers), + ] + else: + cmd = [ + sys.executable, "-m", "litellm", + "--config", config_path, + "--port", str(port), + "--num_workers", str(num_workers), + ] + + proc = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return proc + + +def stop_proxy(proc: subprocess.Popen): + if proc.poll() is None: + proc.send_signal(signal.SIGTERM) + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +async def run_full_benchmark( + config_path: str, + port: int, + use_fast_litellm: bool, + n_requests: int, + max_concurrent: int, + n_runs: int, + num_workers: int, +) -> dict: + label = "fast-litellm" if use_fast_litellm else "Standard LiteLLM" + print(f"\n{'#'*60}") + print(f" Starting {label} proxy on port {port} ({num_workers} workers)") + print(f"{'#'*60}") + + proc = start_proxy(config_path, port, use_fast_litellm, num_workers) + + url = f"http://localhost:{port}/chat/completions" + + print(f" Waiting for proxy to be ready...") + ready = await wait_for_proxy(url, timeout=120) + + if not ready: + print(f" ERROR: Proxy did not start within 120s") + stderr_output = "" + if proc.poll() is not None: + stderr_output = proc.stderr.read().decode("utf-8", errors="replace")[-2000:] + stop_proxy(proc) + print(f" Proxy stderr: {stderr_output}") + return {"error": "Proxy did not start"} + + print(f" Proxy is ready! Running benchmark...") + print(f" {n_requests} requests, {max_concurrent} concurrency, {n_runs} run(s)") + + results = [] + for run_num in range(1, n_runs + 1): + result = await run_benchmark(url, n_requests, max_concurrent) + results.append(result) + print(f"\n Run {run_num}/{n_runs}: mean={result['mean_ms']:.2f}ms, " + f"p50={result['p50_ms']:.2f}ms, p95={result['p95_ms']:.2f}ms, " + f"throughput={result['throughput_rps']:.0f} req/s, " + f"failures={result['failures']}") + + agg = aggregate_runs(results) + print_results(label, agg) + + stop_proxy(proc) + await asyncio.sleep(2) + + return agg + + +async def main(): + parser = argparse.ArgumentParser(description="Benchmark Standard LiteLLM vs fast-litellm proxy") + parser.add_argument("--config", default="benchmark_config.yaml") + parser.add_argument("--requests", type=int, default=1000) + parser.add_argument("--max-concurrent", type=int, default=100) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--num-workers", type=int, default=4) + parser.add_argument("--output", default=None, help="Write JSON results to file") + args = parser.parse_args() + + print("=" * 60) + print(" LiteLLM Proxy Benchmark: Standard vs fast-litellm") + print("=" * 60) + print(f" Config: {args.config}") + print(f" Requests: {args.requests}") + print(f" Concurrency: {args.max_concurrent}") + print(f" Runs: {args.runs}") + print(f" Workers: {args.num_workers}") + print(f" Mode: network_mock (pure proxy overhead)") + + standard_results = await run_full_benchmark( + config_path=args.config, + port=4000, + use_fast_litellm=False, + n_requests=args.requests, + max_concurrent=args.max_concurrent, + n_runs=args.runs, + num_workers=args.num_workers, + ) + + fast_results = await run_full_benchmark( + config_path=args.config, + port=4001, + use_fast_litellm=True, + n_requests=args.requests, + max_concurrent=args.max_concurrent, + n_runs=args.runs, + num_workers=args.num_workers, + ) + + print_comparison(standard_results, fast_results) + + if args.output: + output_data = { + "config": { + "requests": args.requests, + "concurrency": args.max_concurrent, + "runs": args.runs, + "workers": args.num_workers, + "mode": "network_mock", + }, + "standard_litellm": {k: v for k, v in standard_results.items() if k != "latencies"}, + "fast_litellm": {k: v for k, v in fast_results.items() if k != "latencies"}, + } + with open(args.output, "w") as f: + json.dump(output_data, f, indent=2) + print(f"\n Results written to {args.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/run_proxy_benchmark.sh b/scripts/run_proxy_benchmark.sh new file mode 100755 index 00000000000..974000fb0a4 --- /dev/null +++ b/scripts/run_proxy_benchmark.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +WORKSPACE="$(dirname "$SCRIPT_DIR")" +CONFIG="$WORKSPACE/benchmark_config.yaml" +RESULTS_DIR="$WORKSPACE/benchmark_results" +REQUESTS=2000 +CONCURRENT=100 +RUNS=3 +WORKERS=4 + +mkdir -p "$RESULTS_DIR" + +echo "============================================================" +echo " LiteLLM Proxy Benchmark: Standard vs fast-litellm" +echo "============================================================" +echo " Config: $CONFIG" +echo " Requests: $REQUESTS" +echo " Concurrency: $CONCURRENT" +echo " Runs: $RUNS" +echo " Workers: $WORKERS" +echo " Mode: network_mock (pure proxy overhead)" +echo "" + +wait_for_health() { + local port=$1 + local timeout=120 + local elapsed=0 + while [ $elapsed -lt $timeout ]; do + if curl -s "http://localhost:$port/health" > /dev/null 2>&1; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + return 1 +} + +# ============================================================ +# BENCHMARK 1: Standard LiteLLM Proxy +# ============================================================ +echo "############################################################" +echo " PHASE 1: Standard LiteLLM Proxy" +echo "############################################################" + +LITELLM_LOG=ERROR poetry run litellm \ + --config "$CONFIG" \ + --port 4000 \ + --num_workers $WORKERS \ + > "$RESULTS_DIR/standard_proxy.log" 2>&1 & +STANDARD_PID=$! +echo " Started standard proxy (PID: $STANDARD_PID)" +echo " Waiting for proxy to be ready..." + +if ! wait_for_health 4000; then + echo " ERROR: Standard proxy did not start" + cat "$RESULTS_DIR/standard_proxy.log" | tail -30 + kill $STANDARD_PID 2>/dev/null + exit 1 +fi + +echo " Standard proxy is ready!" +echo " Running benchmark..." + +poetry run python "$SCRIPT_DIR/benchmark_mock.py" \ + --url "http://localhost:4000/chat/completions" \ + --requests $REQUESTS \ + --max-concurrent $CONCURRENT \ + --runs $RUNS \ + 2>&1 | tee "$RESULTS_DIR/standard_results.txt" + +echo " Stopping standard proxy..." +kill $STANDARD_PID 2>/dev/null +wait $STANDARD_PID 2>/dev/null || true +sleep 3 +echo " Standard proxy stopped." + +# ============================================================ +# BENCHMARK 2: fast-litellm Accelerated Proxy +# ============================================================ +echo "" +echo "############################################################" +echo " PHASE 2: fast-litellm Accelerated Proxy" +echo "############################################################" + +LITELLM_LOG=ERROR poetry run python -c " +import fast_litellm +import sys +sys.argv = ['litellm', '--config', '$CONFIG', '--port', '4001', '--num_workers', '$WORKERS'] +from litellm.proxy.proxy_cli import run_server +run_server() +" > "$RESULTS_DIR/fast_proxy.log" 2>&1 & +FAST_PID=$! +echo " Started fast-litellm proxy (PID: $FAST_PID)" +echo " Waiting for proxy to be ready..." + +if ! wait_for_health 4001; then + echo " ERROR: fast-litellm proxy did not start" + cat "$RESULTS_DIR/fast_proxy.log" | tail -30 + kill $FAST_PID 2>/dev/null + exit 1 +fi + +echo " fast-litellm proxy is ready!" +echo " Running benchmark..." + +poetry run python "$SCRIPT_DIR/benchmark_mock.py" \ + --url "http://localhost:4001/chat/completions" \ + --requests $REQUESTS \ + --max-concurrent $CONCURRENT \ + --runs $RUNS \ + 2>&1 | tee "$RESULTS_DIR/fast_results.txt" + +echo " Stopping fast-litellm proxy..." +kill $FAST_PID 2>/dev/null +wait $FAST_PID 2>/dev/null || true +echo " fast-litellm proxy stopped." + +echo "" +echo "============================================================" +echo " Benchmark complete! Results saved to: $RESULTS_DIR/" +echo "============================================================" diff --git a/scripts/start_fast_proxy.py b/scripts/start_fast_proxy.py new file mode 100644 index 00000000000..e177928995c --- /dev/null +++ b/scripts/start_fast_proxy.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 +"""Start LiteLLM proxy with fast-litellm acceleration enabled.""" +import fast_litellm # noqa: F401 - Must be imported before litellm to apply Rust patches +from litellm.proxy.proxy_cli import run_server + +if __name__ == "__main__": + run_server()