mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
1cf0097be6
After testing the AWS ECS deployment, discovered that the CloudFormation template does not configure the fake-openai-endpoint model required for running benchmark tests as documented in https://docs.litellm.ai/docs/benchmarks ## Changes - Added KNOWN_ISSUES.md documenting the limitation in detail - Updated README.md with prominent warning about the gap - Updated 00-START-HERE.md with limitation notice - Added simple-loadtest.py for infrastructure-only testing ## What Was Tested ✅ Successfully deployed: - 4 ECS Fargate tasks (4 vCPU, 8 GB RAM each) - RDS PostgreSQL database - Application Load Balancer - Full VPC with security groups ✅ Verified working: - All tasks running and healthy - Database connections - Health endpoints responding - Load balancer routing ❌ Cannot test (missing model config): - API /v1/chat/completions requests - Locust benchmark with 1000 users - LiteLLM overhead measurement - Performance metrics (latency, RPS) ## Root Cause The CloudFormation template only sets environment variables but does not: - Mount a config.yaml file - Configure model_list with fake-openai-endpoint - Set up the test endpoint needed for benchmarking ## Impact Users can deploy the infrastructure matching benchmark specs, but cannot run the actual benchmark without manually configuring models via API or updating the template to mount a config file. ## Stack Cleanup The test deployment was deleted to avoid ongoing costs (~$440/month). Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
70 lines
1.9 KiB
Python
70 lines
1.9 KiB
Python
"""
|
|
Simple load test for LiteLLM that tests health and readiness endpoints
|
|
"""
|
|
|
|
import os
|
|
import time
|
|
from locust import HttpUser, task, between, events
|
|
|
|
|
|
class HealthCheckUser(HttpUser):
|
|
"""
|
|
Simple user that only tests health endpoints to measure infrastructure performance
|
|
"""
|
|
wait_time = between(0.1, 0.3)
|
|
|
|
@task(10)
|
|
def readiness_check(self):
|
|
"""Test readiness endpoint"""
|
|
with self.client.get(
|
|
"/health/readiness",
|
|
catch_response=True,
|
|
name="Readiness Check"
|
|
) as response:
|
|
if response.status_code == 200:
|
|
response.success()
|
|
else:
|
|
response.failure(f"Status: {response.status_code}")
|
|
|
|
@task(5)
|
|
def liveliness_check(self):
|
|
"""Test liveliness endpoint"""
|
|
with self.client.get(
|
|
"/health/liveliness",
|
|
catch_response=True,
|
|
name="Liveliness Check"
|
|
) as response:
|
|
if response.status_code == 200:
|
|
response.success()
|
|
else:
|
|
response.failure(f"Status: {response.status_code}")
|
|
|
|
@task(1)
|
|
def models_list(self):
|
|
"""Test models endpoint"""
|
|
with self.client.get(
|
|
"/v1/models",
|
|
catch_response=True,
|
|
name="List Models"
|
|
) as response:
|
|
if response.status_code == 200:
|
|
response.success()
|
|
else:
|
|
response.failure(f"Status: {response.status_code}")
|
|
|
|
|
|
@events.test_start.add_listener
|
|
def on_test_start(environment, **kwargs):
|
|
print("\n" + "=" * 60)
|
|
print("LiteLLM Infrastructure Performance Test")
|
|
print("=" * 60)
|
|
print(f"Host: {environment.host}")
|
|
print("Testing: Health endpoints and infrastructure")
|
|
print("=" * 60 + "\n")
|
|
|
|
|
|
@events.test_stop.add_listener
|
|
def on_test_stop(environment, **kwargs):
|
|
print("\n" + "=" * 60)
|
|
print("Test Completed")
|
|
print("=" * 60 + "\n")
|