diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index dffc9501a8e..2417b0cff43 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -10,11 +10,11 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## How to use LiteLLM You can use litellm through either: -1. [OpenAI proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ### When to use LiteLLM Proxy Server diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index a3f03b3d76f..cb6550a4785 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -50,7 +50,7 @@ Detailed information about [routing strategies can be found here](../routing) $ litellm --config /path/to/config.yaml ``` -### Test - Load Balancing +### Test - Simple Call Here requests with model=gpt-3.5-turbo will be routed across multiple instances of azure/gpt-3.5-turbo @@ -138,6 +138,27 @@ print(response) +### Test - Loadbalancing + +In this request, the following will occur: +1. A rate limit exception will be raised +2. LiteLLM proxy will retry the request on the model group (default is 3). + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hi there!"} + ], + "mock_testing_rate_limit_error": true +}' +``` + +[**See Code**](https://github.com/BerriAI/litellm/blob/6b8806b45f970cb2446654d2c379f8dcaa93ce3c/litellm/router.py#L2535) + ### Test - Client Side Fallbacks In this request the following will occur: 1. The request to `model="zephyr-beta"` will fail diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 238fe7136a6..47b93ccd2fa 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,7 +1,10 @@ model_list: - - model_name: "*" + - model_name: "gpt-4" litellm_params: - model: "*" - -# litellm_settings: -# failure_callback: ["langfuse"] + model: "gpt-4" + - model_name: "gpt-4" + litellm_params: + model: "gpt-4o" + - model_name: "gpt-4o-mini" + litellm_params: + model: "gpt-4o-mini" \ No newline at end of file diff --git a/litellm/router.py b/litellm/router.py index 108ca706c5f..e31de5332eb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2468,6 +2468,8 @@ class Router: verbose_router_logger.info( f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" ) + if hasattr(original_exception, "message"): + original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" raise original_exception for mg in fallback_model_group: """ @@ -2492,14 +2494,20 @@ class Router: return response except Exception as e: raise e - except Exception as e: - verbose_router_logger.error(f"An exception occurred - {str(e)}") - verbose_router_logger.debug(traceback.format_exc()) + except Exception as new_exception: + verbose_router_logger.error( + "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format( + str(new_exception), + traceback.format_exc(), + await self._async_get_cooldown_deployments_with_debug_info(), + ) + ) if hasattr(original_exception, "message"): # add the available fallbacks to the exception original_exception.message += "\nReceived Model Group={}\nAvailable Model Group Fallbacks={}".format( - model_group, fallback_model_group + model_group, + fallback_model_group, ) raise original_exception @@ -2508,6 +2516,9 @@ class Router: f"Inside async function with retries: args - {args}; kwargs - {kwargs}" ) original_function = kwargs.pop("original_function") + mock_testing_rate_limit_error = kwargs.pop( + "mock_testing_rate_limit_error", None + ) fallbacks = kwargs.pop("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.pop( "context_window_fallbacks", self.context_window_fallbacks @@ -2515,13 +2526,25 @@ class Router: content_policy_fallbacks = kwargs.pop( "content_policy_fallbacks", self.content_policy_fallbacks ) - + model_group = kwargs.get("model") num_retries = kwargs.pop("num_retries") verbose_router_logger.debug( f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}" ) try: + if ( + mock_testing_rate_limit_error is not None + and mock_testing_rate_limit_error is True + ): + verbose_router_logger.info( + "litellm.router.py::async_function_with_retries() - mock_testing_rate_limit_error=True. Raising litellm.RateLimitError." + ) + raise litellm.RateLimitError( + model=model_group, + llm_provider="", + message=f"This is a mock exception for model={model_group}, to trigger a rate limit error.", + ) # if the function call is successful, no exception will be raised and we'll break out of the loop response = await original_function(*args, **kwargs) return response