From 3f541fe99991610df5e9ff8c8b88f98df8d17fab Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 4 Dec 2023 17:37:09 -0800 Subject: [PATCH] (docs) custom timeouts proxy --- docs/my-website/docs/proxy/load_balancing.md | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index d9215244edb..0e1f58662cb 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -111,3 +111,31 @@ curl --location 'http://0.0.0.0:8000/chat/completions' \ } ' ``` + +## Custom Timeouts, Stream Timeouts - Per Model +For each model you can set `timeout` & `stream_timeout` under `litellm_params` +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/gpt-turbo-small-eu + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: + timeout: 0.1 # timeout in (seconds) + stream_timeout: 0.01 # timeout stream requests (seconds) + max_retries: 5 + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/gpt-turbo-small-ca + api_base: https://my-endpoint-canada-berri992.openai.azure.com/ + api_key: + timeout: 0.1 # timeout in (seconds) + stream_timeout: 0.01 # timeout stream requests (seconds) + max_retries: 5 + +``` + +#### Start Proxy +```shell +$ litellm --config /path/to/config.yaml +```