From 8f830bd94864eb5c81346d658152671ce5cf67fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 29 Apr 2024 16:00:02 -0700 Subject: [PATCH] docs(load_test.md): simplify doc --- docs/my-website/docs/load_test.md | 217 +++++++++++++++--------------- 1 file changed, 112 insertions(+), 105 deletions(-) diff --git a/docs/my-website/docs/load_test.md b/docs/my-website/docs/load_test.md index fb737c0f2e2..bd97c62290b 100644 --- a/docs/my-website/docs/load_test.md +++ b/docs/my-website/docs/load_test.md @@ -227,7 +227,118 @@ In our test: So we'll send 600 requests per minute, but expect only 200 requests per minute to succeed. -### 0. Setup Fake OpenAI Server +:::info + +If you don't want to call a real LLM API endpoint, you can setup a fake openai server. [See code](#extra---setup-fake-openai-server) + +::: + +### 1. Setup config + +```yaml +model_list: +- litellm_params: + api_base: http://0.0.0.0:8080 + api_key: my-fake-key + model: openai/my-fake-model + rpm: 100 + model_name: fake-openai-endpoint +- litellm_params: + api_base: http://0.0.0.0:8081 + api_key: my-fake-key + model: openai/my-fake-model-2 + rpm: 100 + model_name: fake-openai-endpoint +router_settings: + num_retries: 0 + enable_pre_call_checks: true + redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis + redis_password: os.environ/REDIS_PASSWORD + redis_port: os.environ/REDIS_PORT +``` + +### 2. Start proxy 2 instances + +**Instance 1** +```bash +litellm --config /path/to/config.yaml --port 4000 + +## RUNNING on http://0.0.0.0:4000 +``` + +**Instance 2** +```bash +litellm --config /path/to/config.yaml --port 4001 + +## RUNNING on http://0.0.0.0:4001 +``` + +### 3. Run Test + +Let's hit the proxy with 600 requests per minute. + +```python +from openai import AsyncOpenAI, AsyncAzureOpenAI +import random, uuid +import time, asyncio, litellm +# import logging +# logging.basicConfig(level=logging.DEBUG) +#### LITELLM PROXY #### +litellm_client = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4000" +) +litellm_client_2 = AsyncOpenAI( + api_key="sk-1234", # [CHANGE THIS] + base_url="http://0.0.0.0:4001" +) + +async def proxy_completion_non_streaming(): + try: + client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients + # print(f"client={client}") + response = await client.chat.completions.create( + model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) + messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], + ) + return response + except Exception as e: + # print(e) + return None + +async def loadtest_fn(): + start = time.time() + n = 500 # Number of concurrent tasks + tasks = [proxy_completion_non_streaming() for _ in range(n)] + chat_completions = await asyncio.gather(*tasks) + successful_completions = [c for c in chat_completions if c is not None] + print(n, time.time() - start, len(successful_completions)) + +def get_utc_datetime(): + import datetime as dt + from datetime import datetime + + if hasattr(dt, "UTC"): + return datetime.now(dt.UTC) # type: ignore + else: + return datetime.utcnow() # type: ignore + + +# Run the event loop to execute the async function +async def parent_fn(): + for _ in range(10): + dt = get_utc_datetime() + current_minute = dt.strftime("%H-%M") + print(f"triggered new batch - {current_minute}") + await loadtest_fn() + await asyncio.sleep(10) + +asyncio.run(parent_fn()) + +``` + + +### Extra - Setup Fake OpenAI Server Let's setup a fake openai server with a RPM limit of 100. @@ -345,107 +456,3 @@ if __name__ == "__main__": ```bash python3 fake_openai_server.py ``` - -### 1. Setup config - -```yaml -model_list: -- litellm_params: - api_base: http://0.0.0.0:8080 - api_key: my-fake-key - model: openai/my-fake-model - rpm: 100 - model_name: fake-openai-endpoint -- litellm_params: - api_base: http://0.0.0.0:8081 - api_key: my-fake-key - model: openai/my-fake-model-2 - rpm: 100 - model_name: fake-openai-endpoint -router_settings: - num_retries: 0 - enable_pre_call_checks: true - redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT -``` - -### 2. Start proxy 2 instances - -**Instance 1** -```bash -litellm --config /path/to/config.yaml --port 4000 - -## RUNNING on http://0.0.0.0:4000 -``` - -**Instance 2** -```bash -litellm --config /path/to/config.yaml --port 4001 - -## RUNNING on http://0.0.0.0:4001 -``` - -### 3. Run Test - -Let's hit the proxy with 600 requests per minute. - -```python -from openai import AsyncOpenAI, AsyncAzureOpenAI -import random, uuid -import time, asyncio, litellm -# import logging -# logging.basicConfig(level=logging.DEBUG) -#### LITELLM PROXY #### -litellm_client = AsyncOpenAI( - api_key="sk-1234", # [CHANGE THIS] - base_url="http://0.0.0.0:4000" -) -litellm_client_2 = AsyncOpenAI( - api_key="sk-1234", # [CHANGE THIS] - base_url="http://0.0.0.0:4001" -) - -async def proxy_completion_non_streaming(): - try: - client = random.sample([litellm_client, litellm_client_2], 1)[0] # randomly pick b/w clients - # print(f"client={client}") - response = await client.chat.completions.create( - model="fake-openai-endpoint", # [CHANGE THIS] (if you call it something else on your proxy) - messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], - ) - return response - except Exception as e: - # print(e) - return None - -async def loadtest_fn(): - start = time.time() - n = 500 # Number of concurrent tasks - tasks = [proxy_completion_non_streaming() for _ in range(n)] - chat_completions = await asyncio.gather(*tasks) - successful_completions = [c for c in chat_completions if c is not None] - print(n, time.time() - start, len(successful_completions)) - -def get_utc_datetime(): - import datetime as dt - from datetime import datetime - - if hasattr(dt, "UTC"): - return datetime.now(dt.UTC) # type: ignore - else: - return datetime.utcnow() # type: ignore - - -# Run the event loop to execute the async function -async def parent_fn(): - for _ in range(10): - dt = get_utc_datetime() - current_minute = dt.strftime("%H-%M") - print(f"triggered new batch - {current_minute}") - await loadtest_fn() - await asyncio.sleep(10) - -asyncio.run(parent_fn()) - -``` \ No newline at end of file