From e2baaaddb08647f484f6868cbd6285892d486f04 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 16 Mar 2026 16:39:57 +0000 Subject: [PATCH] Add benchmark results: fast-litellm vs standard LiteLLM proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Benchmark results comparing 4 scenarios: - Scenario 1: Baseline uvicorn (1 worker) — 19 req/s - Scenario 2: Standard gunicorn (4 workers) — 76-78 req/s - Scenario 3: fast-litellm gunicorn (4 workers) — 79 req/s Key findings: - Multi-worker is the biggest win (4x throughput) - fast-litellm gives ~8% better P50 latency - fast-litellm has much lower run-to-run variance (0.9% vs 4.0% CoV) - Remote DB overhead (~1s/req) dominates, masking proxy-level gains - All scenarios: 0 failures across 6000 requests Co-authored-by: Ishaan Jaff --- benchmark_results/RESULTS.md | 94 +++++++++++++++++++ benchmark_results/scenario1_baseline.txt | 52 ++++++++++ .../scenario2_standard_multiworker.txt | 52 ++++++++++ .../scenario2b_standard_gunicorn_direct.txt | 52 ++++++++++ benchmark_results/scenario3_fast_litellm.txt | 52 ++++++++++ fast_app.py | 8 +- standard_app.py | 14 +++ 7 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 benchmark_results/RESULTS.md create mode 100644 benchmark_results/scenario1_baseline.txt create mode 100644 benchmark_results/scenario2_standard_multiworker.txt create mode 100644 benchmark_results/scenario2b_standard_gunicorn_direct.txt create mode 100644 benchmark_results/scenario3_fast_litellm.txt create mode 100644 standard_app.py diff --git a/benchmark_results/RESULTS.md b/benchmark_results/RESULTS.md new file mode 100644 index 00000000000..cafd35927bd --- /dev/null +++ b/benchmark_results/RESULTS.md @@ -0,0 +1,94 @@ +# Benchmark Results: fast-litellm vs Standard LiteLLM Proxy + +**Date**: 2026-03-16 +**Environment**: 4 CPUs, 15 GB RAM, Linux (cloud VM) +**LiteLLM version**: 1.82.2 (dev) +**fast-litellm version**: 0.1.6 +**Python**: 3.12 + +## Methodology + +All scenarios use `network_mock: true` which intercepts at the httpx transport layer, returning +canned OpenAI ChatCompletion responses. This isolates **pure proxy overhead** — routing, auth, +serialization, token counting, spend logging, and DB writes — from actual LLM API latency. + +**Load parameters**: 2,000 requests per run, 100 concurrent connections, 3 runs per scenario, +50 warmup requests before each timed run. + +> **Note**: The proxy connects to a remote Neon PostgreSQL database for Prisma operations +> (spend tracking, key auth, etc.). This remote DB adds ~1s per-request overhead that dominates +> the results. The relative comparison between scenarios is still valid since all face the same +> DB overhead. + +## Results Summary + +| Metric | Scenario 1: Baseline | Scenario 2: Standard | Scenario 2b: Standard | Scenario 3: fast-litellm | +|--------|---------------------|----------------------|----------------------|--------------------------| +| **Server** | uvicorn | gunicorn (CLI) | gunicorn (direct) | gunicorn + fast-litellm | +| **Workers** | 1 | 4 | 4 | 4 | +| **Throughput** | **19 req/s** | **76 req/s** | **78 req/s** | **79 req/s** | +| **Mean latency** | 5,090 ms | 1,286 ms | 1,245 ms | **1,239 ms** | +| **P50 latency** | 4,893 ms | 1,177 ms | 1,149 ms | **1,060 ms** | +| **P95 latency** | 5,592 ms | 2,182 ms | 1,970 ms | **2,176 ms** | +| **P99 latency** | 23,132 ms | 3,019 ms | 2,296 ms | **2,624 ms** | +| **Failures** | 0 | 0 | 0 | 0 | +| **Latency CoV** | 1.3% | 3.3% | 4.0% | **0.9%** | +| **Throughput CoV** | 1.4% | 3.8% | 4.0% | **0.2%** | + +## Head-to-Head: Standard vs fast-litellm (4 workers, same gunicorn setup) + +Comparing Scenario 2b (standard) vs Scenario 3 (fast-litellm) — identical gunicorn configuration: + +| Metric | Standard (2b) | fast-litellm (3) | Difference | +|--------|---------------|------------------|------------| +| **Throughput** | 78 req/s | 79 req/s | **+1.3% faster** | +| **Mean latency** | 1,245 ms | 1,239 ms | **-0.5% (6 ms faster)** | +| **P50 latency** | 1,149 ms | 1,060 ms | **-7.7% (89 ms faster)** | +| **P95 latency** | 1,970 ms | 2,176 ms | +10.5% (206 ms slower) | +| **P99 latency** | 2,296 ms | 2,624 ms | +14.3% (328 ms slower) | +| **Run-to-run variance** | 4.0% CoV | 0.9% CoV | **Much more consistent** | + +## Analysis + +### Key Findings + +1. **Single → Multi-worker is the big win**: Going from 1 to 4 workers gives a **4x throughput improvement** (19 → 78 req/s). This is the most impactful optimization. + +2. **fast-litellm provides marginal improvement in median latency**: P50 is ~8% better with fast-litellm (1,060 ms vs 1,149 ms), meaning the typical request is measurably faster. + +3. **fast-litellm is significantly more consistent**: The run-to-run variance (CoV) drops from 4.0% to 0.9% for latency and from 4.0% to 0.2% for throughput. This means more predictable performance in production. + +4. **Tail latencies are mixed**: P95 and P99 are slightly higher with fast-litellm. This could be due to the overhead of the Rust↔Python FFI bridge for the patched components, or normal variance in the tail. + +5. **DB overhead dominates**: The ~1,000+ ms per-request latency is primarily from the remote Neon PostgreSQL database (spend tracking, auth lookups). The actual proxy routing/serialization overhead is a small fraction of this. In a production setup with a local PostgreSQL instance, the latency would be much lower, and fast-litellm's improvements would be proportionally more visible. + +### Where fast-litellm Shines + +According to the [fast-litellm benchmarks](https://github.com/neul-labs/fast-litellm), the Rust acceleration provides: +- **3.2x faster** connection pooling +- **1.6x faster** rate limiting +- **1.5-1.7x faster** token counting for large texts +- **42x more memory efficient** for high-cardinality rate limiting + +These benefits are most apparent in: +- High-throughput scenarios (1000+ QPS) +- Large token counting workloads +- High-cardinality rate limiting (many unique API keys) +- Memory-constrained environments + +In our benchmark, the remote DB bottleneck masks most of these gains. With a local DB, the differences would be more pronounced. + +### Limitations of This Benchmark + +- **Remote database**: The Neon PostgreSQL connection adds ~1s overhead per request, overwhelming the proxy's internal processing time. +- **Limited Rust patching**: fast-litellm reports `SimpleRateLimiter` and `SimpleConnectionPool` classes not found in the current LiteLLM version, meaning only token counting and routing acceleration are active. +- **Small text payloads**: The benchmark uses minimal message sizes; fast-litellm's token counting advantage scales with text length. +- **Single machine**: Both proxy and benchmark client run on the same 4-CPU VM, causing resource contention. + +## Raw Results + +See individual scenario output files in this directory: +- `scenario1_baseline.txt` — uvicorn, 1 worker +- `scenario2_standard_multiworker.txt` — gunicorn 4 workers (via CLI) +- `scenario2b_standard_gunicorn_direct.txt` — gunicorn 4 workers (direct, for fair comparison) +- `scenario3_fast_litellm.txt` — gunicorn 4 workers + fast-litellm diff --git a/benchmark_results/scenario1_baseline.txt b/benchmark_results/scenario1_baseline.txt new file mode 100644 index 00000000000..8dd14e666cd --- /dev/null +++ b/benchmark_results/scenario1_baseline.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 103.23s + Throughput: 19 req/s + Mean: 5117.36 ms + P50: 5022.53 ms + P95: 5637.89 ms + P99: 19984.61 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 101.42s + Throughput: 20 req/s + Mean: 5016.14 ms + P50: 4910.59 ms + P95: 5592.26 ms + P99: 21913.53 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 104.36s + Throughput: 19 req/s + Mean: 5136.37 ms + P50: 4530.51 ms + P95: 5376.66 ms + P99: 26824.13 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 19 req/s (avg across runs) + Mean: 5089.95 ms + P50: 4893.29 ms + P95: 5592.00 ms + P99: 23131.56 ms + + Run-to-run variance: + Latency CoV: 1.3% + Throughput CoV: 1.4% diff --git a/benchmark_results/scenario2_standard_multiworker.txt b/benchmark_results/scenario2_standard_multiworker.txt new file mode 100644 index 00000000000..04874c38588 --- /dev/null +++ b/benchmark_results/scenario2_standard_multiworker.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.23s + Throughput: 79 req/s + Mean: 1240.01 ms + P50: 1215.33 ms + P95: 1567.70 ms + P99: 2006.41 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 27.22s + Throughput: 73 req/s + Mean: 1325.29 ms + P50: 1049.31 ms + P95: 2494.43 ms + P99: 3044.45 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 26.30s + Throughput: 76 req/s + Mean: 1291.30 ms + P50: 1240.27 ms + P95: 1674.42 ms + P99: 3414.54 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 76 req/s (avg across runs) + Mean: 1285.53 ms + P50: 1176.84 ms + P95: 2181.96 ms + P99: 3018.57 ms + + Run-to-run variance: + Latency CoV: 3.3% + Throughput CoV: 3.8% diff --git a/benchmark_results/scenario2b_standard_gunicorn_direct.txt b/benchmark_results/scenario2b_standard_gunicorn_direct.txt new file mode 100644 index 00000000000..6f0e0fb1128 --- /dev/null +++ b/benchmark_results/scenario2b_standard_gunicorn_direct.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 24.45s + Throughput: 82 req/s + Mean: 1188.82 ms + P50: 1145.08 ms + P95: 1558.45 ms + P99: 1965.98 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 26.50s + Throughput: 75 req/s + Mean: 1284.92 ms + P50: 1208.04 ms + P95: 1730.94 ms + P99: 2130.37 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.63s + Throughput: 78 req/s + Mean: 1261.34 ms + P50: 1079.01 ms + P95: 2145.19 ms + P99: 2382.36 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 78 req/s (avg across runs) + Mean: 1245.03 ms + P50: 1149.26 ms + P95: 1969.57 ms + P99: 2296.02 ms + + Run-to-run variance: + Latency CoV: 4.0% + Throughput CoV: 4.0% diff --git a/benchmark_results/scenario3_fast_litellm.txt b/benchmark_results/scenario3_fast_litellm.txt new file mode 100644 index 00000000000..da469a8a78e --- /dev/null +++ b/benchmark_results/scenario3_fast_litellm.txt @@ -0,0 +1,52 @@ +Benchmarking http://localhost:4000/chat/completions + 2000 requests, 100 concurrency, 3 run(s) + +============================================================ + Run 1/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.35s + Throughput: 79 req/s + Mean: 1235.96 ms + P50: 1060.98 ms + P95: 2018.34 ms + P99: 2611.21 ms + +============================================================ + Run 2/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.47s + Throughput: 79 req/s + Mean: 1250.70 ms + P50: 1134.55 ms + P95: 1971.17 ms + P99: 2655.68 ms + +============================================================ + Run 3/3 +============================================================ + Requests: 2000 (failures: 0) + Concurrency: 100 + Wall time: 25.38s + Throughput: 79 req/s + Mean: 1229.04 ms + P50: 1002.29 ms + P95: 2248.59 ms + P99: 2623.85 ms + +============================================================ + Aggregate (3 runs, 6000 total requests) +============================================================ + Failures: 0 + Throughput: 79 req/s (avg across runs) + Mean: 1238.56 ms + P50: 1060.48 ms + P95: 2175.84 ms + P99: 2623.80 ms + + Run-to-run variance: + Latency CoV: 0.9% + Throughput CoV: 0.2% diff --git a/fast_app.py b/fast_app.py index 49562c54c77..8db42b5200f 100644 --- a/fast_app.py +++ b/fast_app.py @@ -2,9 +2,15 @@ Gunicorn wrapper for LiteLLM proxy with fast-litellm Rust acceleration. Usage: + CONFIG_FILE_PATH=benchmark_config.yaml \ gunicorn fast_app:app --preload -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4000 """ -import fast_litellm # noqa: F401 — Apply Rust acceleration before litellm loads +import os + +# Set config file path before anything else loads +os.environ.setdefault("CONFIG_FILE_PATH", "benchmark_config.yaml") + +import fast_litellm # noqa: F401, E402 — Apply Rust acceleration before litellm loads from litellm.proxy.proxy_server import app # noqa: F401, E402 diff --git a/standard_app.py b/standard_app.py new file mode 100644 index 00000000000..68fc080e5ff --- /dev/null +++ b/standard_app.py @@ -0,0 +1,14 @@ +""" +Gunicorn wrapper for standard LiteLLM proxy (no fast-litellm). + +Usage: + CONFIG_FILE_PATH=benchmark_config.yaml \ + gunicorn standard_app:app --preload -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:4000 +""" + +import os + +# Set config file path before anything else loads +os.environ.setdefault("CONFIG_FILE_PATH", "benchmark_config.yaml") + +from litellm.proxy.proxy_server import app # noqa: F401, E402