mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
docs(load_test.md): simplify doc
This commit is contained in:
Krrish Dholakia
parent
fcb83781ec
commit
8f830bd948
1 changed files with 112 additions and 105 deletions
|
|
@ -227,7 +227,118 @@ In our test:
|
|||
|
||||
So we'll send 600 requests per minute, but expect only 200 requests per minute to succeed.
|
||||
|
||||
### 0. Setup Fake OpenAI Server
|
||||
:::info
|
||||
|
||||
If you don't want to call a real LLM API endpoint, you can setup a fake openai server. [See code](#extra---setup-fake-openai-server)
|
||||
|
||||
:::
|
||||
|
||||
### 1. Setup config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- litellm_params:
|
||||
api_base: http://0.0.0.0:8080
|
||||
api_key: my-fake-key
|
||||
model: openai/my-fake-model
|
||||
rpm: 100
|
||||
model_name: fake-openai-endpoint
|
||||
- litellm_params:
|
||||
api_base: http://0.0.0.0:8081
|
||||
api_key: my-fake-key
|
||||
model: openai/my-fake-model-2
|
||||
rpm: 100
|
||||
model_name: fake-openai-endpoint
|
||||
router_settings:
|
||||
num_retries: 0
|
||||
enable_pre_call_checks: true
|
||||
redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
|
||||
redis_password: os.environ/REDIS_PASSWORD
|
||||
redis_port: os.environ/REDIS_PORT
|
||||
```
|
||||
|
||||
### 2. Start proxy 2 instances
|
||||
|
||||
**Instance 1**
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --port 4000
|
||||
|
||||
## RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
**Instance 2**
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --port 4001
|
||||
|
||||
## RUNNING on http://0.0.0.0:4001
|
||||
```
|
||||
|
||||
### 3. Run Test
|
||||
|
||||
Let's hit the proxy with 600 requests per minute.
|
||||
|
||||
```python
|
||||
from openai import AsyncOpenAI, AsyncAzureOpenAI
|
||||
import random, uuid
|
||||
import time, asyncio, litellm
|
||||
# import logging
|
||||
# logging.basicConfig(level=logging.DEBUG)
|
||||
#### LITELLM PROXY ####
|
||||
litellm_client = AsyncOpenAI(
|
||||
api_key="sk-1234", # [CHANGE THIS]
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
litellm_client_2 = AsyncOpenAI(
|
||||
api_key="sk-1234", # [CHANGE THIS]
|
||||
base_url="http://0.0.0.0:4001"
|
||||
)
|
||||
|
||||
async def proxy_completion_non_streaming():
|
||||
try:
|
||||
client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients
|
||||
# print(f"client={client}")
|
||||
response = await client.chat.completions.create(
|
||||
model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy)
|
||||
messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}],
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
# print(e)
|
||||
return None
|
||||
|
||||
async def loadtest_fn():
|
||||
start = time.time()
|
||||
n = 500 # Number of concurrent tasks
|
||||
tasks = [proxy_completion_non_streaming() for _ in range(n)]
|
||||
chat_completions = await asyncio.gather(*tasks)
|
||||
successful_completions = [c for c in chat_completions if c is not None]
|
||||
print(n, time.time() - start, len(successful_completions))
|
||||
|
||||
def get_utc_datetime():
|
||||
import datetime as dt
|
||||
from datetime import datetime
|
||||
|
||||
if hasattr(dt, "UTC"):
|
||||
return datetime.now(dt.UTC) # type: ignore
|
||||
else:
|
||||
return datetime.utcnow() # type: ignore
|
||||
|
||||
|
||||
# Run the event loop to execute the async function
|
||||
async def parent_fn():
|
||||
for _ in range(10):
|
||||
dt = get_utc_datetime()
|
||||
current_minute = dt.strftime("%H-%M")
|
||||
print(f"triggered new batch - {current_minute}")
|
||||
await loadtest_fn()
|
||||
await asyncio.sleep(10)
|
||||
|
||||
asyncio.run(parent_fn())
|
||||
|
||||
```
|
||||
|
||||
|
||||
### Extra - Setup Fake OpenAI Server
|
||||
|
||||
Let's setup a fake openai server with a RPM limit of 100.
|
||||
|
||||
|
|
@ -345,107 +456,3 @@ if __name__ == "__main__":
|
|||
```bash
|
||||
python3 fake_openai_server.py
|
||||
```
|
||||
|
||||
### 1. Setup config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- litellm_params:
|
||||
api_base: http://0.0.0.0:8080
|
||||
api_key: my-fake-key
|
||||
model: openai/my-fake-model
|
||||
rpm: 100
|
||||
model_name: fake-openai-endpoint
|
||||
- litellm_params:
|
||||
api_base: http://0.0.0.0:8081
|
||||
api_key: my-fake-key
|
||||
model: openai/my-fake-model-2
|
||||
rpm: 100
|
||||
model_name: fake-openai-endpoint
|
||||
router_settings:
|
||||
num_retries: 0
|
||||
enable_pre_call_checks: true
|
||||
redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
|
||||
redis_password: os.environ/REDIS_PASSWORD
|
||||
redis_port: os.environ/REDIS_PORT
|
||||
```
|
||||
|
||||
### 2. Start proxy 2 instances
|
||||
|
||||
**Instance 1**
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --port 4000
|
||||
|
||||
## RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
**Instance 2**
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml --port 4001
|
||||
|
||||
## RUNNING on http://0.0.0.0:4001
|
||||
```
|
||||
|
||||
### 3. Run Test
|
||||
|
||||
Let's hit the proxy with 600 requests per minute.
|
||||
|
||||
```python
|
||||
from openai import AsyncOpenAI, AsyncAzureOpenAI
|
||||
import random, uuid
|
||||
import time, asyncio, litellm
|
||||
# import logging
|
||||
# logging.basicConfig(level=logging.DEBUG)
|
||||
#### LITELLM PROXY ####
|
||||
litellm_client = AsyncOpenAI(
|
||||
api_key="sk-1234", # [CHANGE THIS]
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
litellm_client_2 = AsyncOpenAI(
|
||||
api_key="sk-1234", # [CHANGE THIS]
|
||||
base_url="http://0.0.0.0:4001"
|
||||
)
|
||||
|
||||
async def proxy_completion_non_streaming():
|
||||
try:
|
||||
client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients
|
||||
# print(f"client={client}")
|
||||
response = await client.chat.completions.create(
|
||||
model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy)
|
||||
messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}],
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
# print(e)
|
||||
return None
|
||||
|
||||
async def loadtest_fn():
|
||||
start = time.time()
|
||||
n = 500 # Number of concurrent tasks
|
||||
tasks = [proxy_completion_non_streaming() for _ in range(n)]
|
||||
chat_completions = await asyncio.gather(*tasks)
|
||||
successful_completions = [c for c in chat_completions if c is not None]
|
||||
print(n, time.time() - start, len(successful_completions))
|
||||
|
||||
def get_utc_datetime():
|
||||
import datetime as dt
|
||||
from datetime import datetime
|
||||
|
||||
if hasattr(dt, "UTC"):
|
||||
return datetime.now(dt.UTC) # type: ignore
|
||||
else:
|
||||
return datetime.utcnow() # type: ignore
|
||||
|
||||
|
||||
# Run the event loop to execute the async function
|
||||
async def parent_fn():
|
||||
for _ in range(10):
|
||||
dt = get_utc_datetime()
|
||||
current_minute = dt.strftime("%H-%M")
|
||||
print(f"triggered new batch - {current_minute}")
|
||||
await loadtest_fn()
|
||||
await asyncio.sleep(10)
|
||||
|
||||
asyncio.run(parent_fn())
|
||||
|
||||
```
|
||||
Loading…
Add table
Reference in a new issue