From a9006b965f6290065a3210d8a16e5f0269adc4a5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:32:28 -0700 Subject: [PATCH 1/7] fix - support fallbacks as list --- litellm/router.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index bfd1dafe98f..c4ea521c5c0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2056,12 +2056,15 @@ class Router: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") generic_fallback_idx: Optional[int] = None ## check for specific model group-specific fallbacks - for idx, item in enumerate(fallbacks): - if list(item.keys())[0] == model_group: - fallback_model_group = item[model_group] - break - elif list(item.keys())[0] == "*": - generic_fallback_idx = idx + if isinstance(fallbacks, list): + fallback_model_group = fallbacks + else: + for idx, item in enumerate(fallbacks): + if list(item.keys())[0] == model_group: + fallback_model_group = item[model_group] + break + elif list(item.keys())[0] == "*": + generic_fallback_idx = idx ## if none, check for generic fallback if ( fallback_model_group is None From 1656bbf03ee20b5a8b39e2f2337fd3d3eb310011 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:33:32 -0700 Subject: [PATCH 2/7] doc - client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 72 +++++++++-------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index e39a6765fca..6d02346ace7 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔥 Fallbacks, Retries, Timeouts, Load Balancing +# 🔥 Load Balancing, Fallbacks, Retries, Timeouts Retry call with multiple instances of the same model. @@ -13,7 +13,7 @@ If the error is a context window exceeded error, fall back to a larger model gro [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/router.py) ## Quick Start - Load Balancing -### Step 1 - Set deployments on config +#### Step 1 - Set deployments on config **Example config below**. Here requests with `model=gpt-3.5-turbo` will be routed across multiple instances of `azure/gpt-3.5-turbo` ```yaml @@ -38,13 +38,13 @@ model_list: rpm: 1440 ``` -### Step 2: Start Proxy with config +#### Step 2: Start Proxy with config ```shell $ litellm --config /path/to/config.yaml ``` -### Step 3: Use proxy - Call a model group [Load Balancing] +### Test - Load Balancing Curl Command ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -61,25 +61,44 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` -### Usage - Call a specific model deployment -If you want to call a specific model defined in the `config.yaml`, you can call the `litellm_params: model` -In this example it will call `azure/gpt-turbo-small-ca`. Defined in the config on Step 1 + +### Test - Client Side Fallbacks ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --data ' { - "model": "azure/gpt-turbo-small-ca", + "model": "zephyr-beta", "messages": [ { "role": "user", "content": "what llm are you" } ], + "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "num_retries": 2, + "timeout": 10 } ' ``` + + ## Fallbacks + Retries + Timeouts + Cooldowns @@ -114,43 +133,6 @@ litellm_settings: context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. ``` - -**Set dynamically** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "zephyr-beta", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "num_retries": 2, - "timeout": 10 - } -' -``` - -### Test it! - - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data-raw '{ - "model": "zephyr-beta", # 👈 MODEL NAME to fallback from - "messages": [ - {"role": "user", "content": "what color is red"} - ], - "mock_testing_fallbacks": true - }' -``` - ## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. From b52802bbfab00343c79c78de05cd13bfee3dcec1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:33:42 -0700 Subject: [PATCH 3/7] doc - client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index 6d02346ace7..dd383bf0e62 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -76,7 +76,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ "content": "what llm are you" } ], - "fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], + "fallbacks": ["gpt-3.5-turbo"]}], "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], "num_retries": 2, "timeout": 10 From 66fd69b7049c0f878b461f2265051b9d63c791a1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Jun 2024 14:49:50 -0700 Subject: [PATCH 4/7] docs - using client side fallbacks --- docs/my-website/docs/proxy/reliability.md | 202 +++++++++++++++++++--- 1 file changed, 178 insertions(+), 24 deletions(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index dd383bf0e62..d53e4bb2927 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -45,44 +45,198 @@ $ litellm --config /path/to/config.yaml ``` ### Test - Load Balancing -Curl Command + +Here requests with model=gpt-3.5-turbo will be routed across multiple instances of azure/gpt-3.5-turbo + +👉 Key Change: `model="gpt-3.5-turbo"` + +**Check the `model_id` in Response Headers to make sure the requests are being load balanced** + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ] +) + +print(response) +``` + + + + +Pass `metadata` as part of the request body + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "gpt-3.5-turbo", - "messages": [ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ { - "role": "user", - "content": "what llm are you" + "role": "user", + "content": "what llm are you" } - ], - } -' + ] +}' ``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="gpt-3.5-turbo", +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + ### Test - Client Side Fallbacks +In this request the following will occur: +1. The request to `model="zephyr-beta"` will fail +2. litellm proxy will loop through all the model_groups specified in `fallbacks=["gpt-3.5-turbo"]` +3. The request to `model="gpt-3.5-turbo"` will succeed and the client making the request will get a response from gpt-3.5-turbo -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "zephyr-beta", - "messages": [ +👉 Key Change: `"fallbacks": ["gpt-3.5-turbo"]` + + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="zephyr-beta", + messages = [ { - "role": "user", - "content": "what llm are you" + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] } - ], - "fallbacks": ["gpt-3.5-turbo"]}], - "context_window_fallbacks": [{"zephyr-beta": ["gpt-3.5-turbo"]}], - "num_retries": 2, - "timeout": 10 } -' +) + +print(response) ``` + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "zephyr-beta"", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] + } +}' +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model="zephyr-beta", + extra_body={ + "metadata": { + "fallbacks": ["gpt-3.5-turbo"] + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + + - -## Fallbacks + Retries + Timeouts + Cooldowns +## Advanced +### Fallbacks + Retries + Timeouts + Cooldowns **Set via config** ```yaml @@ -282,7 +279,7 @@ litellm_settings: context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. ``` -## Advanced - Context Window Fallbacks (Pre-Call Checks + Fallbacks) +### Context Window Fallbacks (Pre-Call Checks + Fallbacks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -418,7 +415,7 @@ print(response) -## Advanced - EU-Region Filtering (Pre-Call Checks) +### EU-Region Filtering (Pre-Call Checks) **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. @@ -481,7 +478,7 @@ print(response) print(f"response.headers.get('x-litellm-model-api-base')") ``` -## Advanced - Custom Timeouts, Stream Timeouts - Per Model +### Custom Timeouts, Stream Timeouts - Per Model For each model you can set `timeout` & `stream_timeout` under `litellm_params` ```yaml model_list: @@ -510,7 +507,7 @@ $ litellm --config /path/to/config.yaml ``` -## Advanced - Setting Dynamic Timeouts - Per Request +### Setting Dynamic Timeouts - Per Request LiteLLM Proxy supports setting a `timeout` per request