From edab94c0a6ed63e91ab7d6b7721c8f374fcaad10 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 1 Mar 2024 12:28:49 -0800 Subject: [PATCH] (docs) load balancing on proxy --- docs/my-website/docs/proxy/load_balancing.md | 28 +++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 7fd4db83b9e..ad5e91203df 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -1,4 +1,4 @@ -# Multiple Instances of 1 model +# Load Balancing - Config Setup Load balance multiple instances of the same model The proxy will handle routing requests (using LiteLLM's Router). **Set `rpm` in the config if you want maximize throughput** @@ -79,6 +79,32 @@ curl --location 'http://0.0.0.0:8000/chat/completions' \ ' ``` +## Load Balancing using multiple litellm instances (Kubernetes, Auto Scaling) + +LiteLLM Proxy supports sharing rpm/tpm shared across multiple litellm instances, pass `redis_host`, `redis_password` and `redis_port` to enable this. (LiteLLM will use Redis to track rpm/tpm usage ) + +Example config + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/ + api_base: + api_key: + rpm: 6 # Rate limit for this deployment: in requests per minute (rpm) + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/gpt-turbo-small-ca + api_base: https://my-endpoint-canada-berri992.openai.azure.com/ + api_key: + rpm: 6 +router_settings: + redis_host: + redis_password: + redis_port: 1992 +``` + ## Router settings on config - routing_strategy, model_group_alias litellm.Router() settings can be set under `router_settings`. You can set `model_group_alias`, `routing_strategy`, `num_retries`,`timeout` . See all Router supported params [here](https://github.com/BerriAI/litellm/blob/1b942568897a48f014fa44618ec3ce54d7570a46/litellm/router.py#L64)