From 66815c9f4f3ce882cb2137590aec21a6ec501284 Mon Sep 17 00:00:00 2001
From: Alexsander Hamir <alexsanderhamirgomesbaptista@gmail.com>
Date: Mon, 9 Feb 2026 17:47:26 -0800
Subject: [PATCH] Add shared-client latency measurement script and minimal test
 config

- measure_latency_shared_client.py: batch /chat/completions with single
  httpx.AsyncClient, report per-request latency and p95/p99
- test_config_minimal.yaml: minimal proxy config for perf testing (master_key, gpt-o1)
---
 measure_latency_shared_client.py | 97 ++++++++++++++++++++++++++++++++
 test_config_minimal.yaml         | 18 ++++++
 2 files changed, 115 insertions(+)
 create mode 100644 measure_latency_shared_client.py
 create mode 100644 test_config_minimal.yaml

diff --git a/measure_latency_shared_client.py b/measure_latency_shared_client.py
new file mode 100644
index 00000000000..679d39a037b
--- /dev/null
+++ b/measure_latency_shared_client.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""
+Measure /chat/completions latency using a single shared HTTP client.
+
+Uses one httpx.AsyncClient and fires all requests in one asyncio.gather() batch.
+Eliminates per-user client creation and connection overhead.
+"""
+
+import asyncio
+import time
+
+import httpx
+
+BASE_URL = "http://localhost:4000"
+API_KEY = "sk-1234"
+MODEL = "gpt-o1"
+NUM_REQUESTS = 200
+TIMEOUT = 30.0
+PAYLOAD = {"model": MODEL, "messages": [{"role": "user", "content": "Say hello in one word."}]}
+HEADERS = {
+    "Authorization": f"Bearer {API_KEY}",
+    "Content-Type": "application/json",
+}
+
+# Module-level client configured for throughput
+client = httpx.AsyncClient(
+    limits=httpx.Limits(
+        max_connections=200,
+        max_keepalive_connections=100,
+        keepalive_expiry=30.0,
+    ),
+    timeout=TIMEOUT,
+    http2=True,
+)
+
+
+async def one_request(client: httpx.AsyncClient, req_id: int) -> tuple[int, float, bool]:
+    """Single request; returns (req_id, latency_sec, success)."""
+    start = time.perf_counter()
+    try:
+        resp = await client.post(
+            f"{BASE_URL}/chat/completions",
+            json=PAYLOAD,
+            headers=HEADERS,
+            timeout=TIMEOUT,
+        )
+        resp.read()
+        ok = resp.is_success
+    except Exception:
+        ok = False
+    lat = time.perf_counter() - start
+    return (req_id, lat, ok)
+
+
+async def main() -> None:
+    print(f"=== {NUM_REQUESTS} requests, single shared client, batch fire ===")
+    print("Latency = time from request start until last byte of response received")
+    print()
+
+    try:
+        tasks = [one_request(client, i) for i in range(1, NUM_REQUESTS + 1)]
+        results = await asyncio.gather(*tasks)
+    finally:
+        await client.aclose()
+
+    # Sort by req_id for stable output, print as we go
+    for req_id, lat, ok in sorted(results, key=lambda r: r[0]):
+        status = "[OK]" if ok else "[FAIL]"
+        print(f"  {status} Request {req_id:3d}: {lat:.3f}s", flush=True)
+
+    # Summary
+    lats = [r[1] for r in results if r[2]]
+    if not lats:
+        print(f"\n[OK] 0 succeeded, [FAIL] {NUM_REQUESTS} failed")
+        return
+
+    n = len(lats)
+    avg = sum(lats) / n
+    sorted_lat = sorted(lats)
+    p95 = sorted_lat[int((n - 1) * 0.95)]
+    p99 = sorted_lat[int((n - 1) * 0.99)]
+
+    print(f"\n[OK] {n} succeeded, [FAIL] {NUM_REQUESTS - n} failed")
+    print("\nLatency (successful):")
+    print(f"  min:  {min(lats):.3f}s")
+    print(f"  avg:  {avg:.3f}s")
+    print(f"  max:  {max(lats):.3f}s")
+    print(f"  p95:  {p95:.3f}s")
+    print(f"  p99:  {p99:.3f}s")
+    print("\nRequests above threshold (successful only):")
+    for t in range(1, 11):
+        above = sum(1 for x in lats if x > t)
+        print(f"  Above {t}s: {above}/{n} ({100.0 * above / n:.1f}%)")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
diff --git a/test_config_minimal.yaml b/test_config_minimal.yaml
new file mode 100644
index 00000000000..d3bd2c27bb3
--- /dev/null
+++ b/test_config_minimal.yaml
@@ -0,0 +1,18 @@
+####################################################################
+### Minimal LiteLLM Proxy Configuration for Performance Testing
+####################################################################
+
+general_settings:
+  master_key: "sk-1234"
+
+
+
+
+model_list:
+  - model_name: "gpt-o1"
+    litellm_params:
+      model: openai/gpt-o1
+      api_base: http://localhost:8090
+      api_key: "sk-1234"
+    model_info:
+      context_length: 200000