From dc3cdf3ed88fbcdae8620458d82c65441eef30fd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 08:15:05 -0700 Subject: [PATCH 01/17] fix use extra headers for open router --- litellm/main.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 1209306c8b0..789863ecb0e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1856,17 +1856,18 @@ def completion( ) openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" - headers = ( - headers - or litellm.headers - or { - "HTTP-Referer": openrouter_site_url, - "X-Title": openrouter_app_name, - } - ) + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + + _headers = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) + + headers = openrouter_headers ## Load Config config = openrouter.OpenrouterConfig.get_config() From 315bba34e613d526f01baa7b007b16c8e8dbf2ea Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 09:02:03 -0700 Subject: [PATCH 02/17] prom svc logger init if it's None --- litellm/_service_logger.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index da0c99aac37..5e9ab03cf42 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -73,6 +73,7 @@ class ServiceLogging(CustomLogger): ) for callback in litellm.service_callback: if callback == "prometheus_system": + await self.init_prometheus_services_logger_if_none() await self.prometheusServicesLogger.async_service_success_hook( payload=payload ) @@ -88,6 +89,11 @@ class ServiceLogging(CustomLogger): event_metadata=event_metadata, ) + async def init_prometheus_services_logger_if_none(self): + if self.prometheusServicesLogger is None: + self.prometheusServicesLogger = self.prometheusServicesLogger() + return + async def async_service_failure_hook( self, service: ServiceTypes, @@ -120,8 +126,7 @@ class ServiceLogging(CustomLogger): ) for callback in litellm.service_callback: if callback == "prometheus_system": - if self.prometheusServicesLogger is None: - self.prometheusServicesLogger = self.prometheusServicesLogger() + await self.init_prometheus_services_logger_if_none() await self.prometheusServicesLogger.async_service_failure_hook( payload=payload ) From 13d1caf94a4b654679d6dc4f492b7948075bee40 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 09:43:35 -0700 Subject: [PATCH 03/17] add set_remaining_tokens_requests_metric --- litellm/integrations/prometheus.py | 33 +++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4a271d6e004..41679001552 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -8,7 +8,7 @@ import subprocess import sys import traceback import uuid -from typing import Optional, Union +from typing import Optional, TypedDict, Union import dotenv import requests # type: ignore @@ -124,6 +124,29 @@ class PrometheusLogger: "litellm_model_name", ], ) + # Get all keys + _logged_llm_labels = [ + "litellm_model_name", + "model_id", + "api_base", + "api_provider", + ] + + self.deployment_unhealthy = Gauge( + "deployment_unhealthy", + 'Value is "1" when deployment is in an unhealthy state', + labelnames=_logged_llm_labels, + ) + self.deployment_partial_outage = Gauge( + "deployment_partial_outage", + 'Value is "1" when deployment is experiencing a partial outage', + labelnames=_logged_llm_labels, + ) + self.deployment_healthy = Gauge( + "deployment_healthy", + 'Value is "1" when deployment is in an healthy state', + labelnames=_logged_llm_labels, + ) except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") @@ -273,6 +296,7 @@ class PrometheusLogger: model_group = _metadata.get("model_group", None) api_base = _metadata.get("api_base", None) llm_provider = _litellm_params.get("custom_llm_provider", None) + model_id = _metadata.get("model_id") remaining_requests = None remaining_tokens = None @@ -307,6 +331,13 @@ class PrometheusLogger: model_group, llm_provider, api_base, litellm_model_name ).set(remaining_tokens) + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) except Exception as e: verbose_logger.error( "Prometheus Error: set_remaining_tokens_requests_metric. Exception occured - {}".format( From 0304513b67975395a979010fdffce886c7080e3f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 09:46:08 -0700 Subject: [PATCH 04/17] rename to set_llm_deployment_success_metrics --- litellm/integrations/prometheus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 41679001552..66e64aa2f7d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -266,7 +266,7 @@ class PrometheusLogger: # set x-ratelimit headers if premium_user is True: - self.set_remaining_tokens_requests_metric(kwargs) + self.set_llm_deployment_success_metrics(kwargs) ### FAILURE INCREMENT ### if "exception" in kwargs: @@ -286,7 +286,7 @@ class PrometheusLogger: verbose_logger.debug(traceback.format_exc()) pass - def set_remaining_tokens_requests_metric(self, request_kwargs: dict): + def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") _response_headers = request_kwargs.get("response_headers") @@ -340,7 +340,7 @@ class PrometheusLogger: ).set(1) except Exception as e: verbose_logger.error( - "Prometheus Error: set_remaining_tokens_requests_metric. Exception occured - {}".format( + "Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {}".format( str(e) ) ) From 9a8fa9f047b406cb6494ae6bba6e4b41b352e0bb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 09:56:01 -0700 Subject: [PATCH 05/17] emit deployment_partial_outage on prometheus --- litellm/integrations/prometheus.py | 29 +++++++++++++++++++++++++++++ litellm/proxy/proxy_config.yaml | 6 ++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 66e64aa2f7d..0865f64eed5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -279,6 +279,8 @@ class PrometheusLogger: user_api_team_alias, user_id, ).inc() + + self.set_llm_deployment_failure_metrics(kwargs) except Exception as e: verbose_logger.error( "prometheus Layer Error(): Exception occured - {}".format(str(e)) @@ -286,6 +288,33 @@ class PrometheusLogger: verbose_logger.debug(traceback.format_exc()) pass + def set_llm_deployment_failure_metrics(self, request_kwargs: dict): + try: + verbose_logger.debug("setting remaining tokens requests metric") + _response_headers = request_kwargs.get("response_headers") + _litellm_params = request_kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) + litellm_model_name = request_kwargs.get("model", None) + api_base = _metadata.get("api_base", None) + llm_provider = _litellm_params.get("custom_llm_provider", None) + model_id = _metadata.get("model_id") + + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + pass + except: + pass + def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 97cd407d324..36b191c90a5 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,7 +3,7 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: https://exampleopenaiendpoint-production.up.railwaz.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct @@ -50,4 +50,6 @@ general_settings: litellm_settings: - callbacks: ["otel"] # πŸ‘ˆ KEY CHANGE \ No newline at end of file + callbacks: ["otel"] # πŸ‘ˆ KEY CHANGE + success_callback: ["prometheus"] + failure_callback: ["prometheus"] \ No newline at end of file From 99ecde774417cde65cfebc14763468a4eea372e5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 10:36:18 -0700 Subject: [PATCH 06/17] allow setting outage metrics --- litellm/integrations/prometheus.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 0865f64eed5..0c8df96bfb5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -315,6 +315,22 @@ class PrometheusLogger: except: pass + def set_llm_outage_metric( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.deployment_unhealthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + pass + def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") From d1e519afd14f045609cf64db0c28afd6788dc568 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 10:40:55 -0700 Subject: [PATCH 07/17] use router_cooldown_handler --- litellm/router.py | 49 ++++----------------- litellm/router_utils/cooldown_callbacks.py | 51 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 40 deletions(-) create mode 100644 litellm/router_utils/cooldown_callbacks.py diff --git a/litellm/router.py b/litellm/router.py index aa9768ba440..a6ec01b06b8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -57,6 +57,7 @@ from litellm.router_utils.client_initalization_utils import ( set_client, should_initialize_sync_client, ) +from litellm.router_utils.cooldown_callbacks import router_cooldown_handler from litellm.router_utils.handle_error import send_llm_exception_alert from litellm.scheduler import FlowItem, Scheduler from litellm.types.llms.openai import ( @@ -3294,10 +3295,14 @@ class Router: value=cached_value, key=cooldown_key, ttl=cooldown_time ) - self.send_deployment_cooldown_alert( - deployment_id=deployment, - exception_status=exception_status, - cooldown_time=cooldown_time, + # Trigger cooldown handler + asyncio.create_task( + router_cooldown_handler( + litellm_router_instance=self, + deployment_id=deployment, + exception_status=exception_status, + cooldown_time=cooldown_time, + ) ) else: self.failed_calls.set_cache( @@ -4948,42 +4953,6 @@ class Router: ) print("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") # noqa - def send_deployment_cooldown_alert( - self, - deployment_id: str, - exception_status: Union[str, int], - cooldown_time: float, - ): - try: - from litellm.proxy.proxy_server import proxy_logging_obj - - # trigger slack alert saying deployment is in cooldown - if ( - proxy_logging_obj is not None - and proxy_logging_obj.alerting is not None - and "slack" in proxy_logging_obj.alerting - ): - _deployment = self.get_deployment(model_id=deployment_id) - if _deployment is None: - return - - _litellm_params = _deployment["litellm_params"] - temp_litellm_params = copy.deepcopy(_litellm_params) - temp_litellm_params = dict(temp_litellm_params) - _model_name = _deployment.get("model_name", None) - _api_base = litellm.get_api_base( - model=_model_name, optional_params=temp_litellm_params - ) - # asyncio.create_task( - # proxy_logging_obj.slack_alerting_instance.send_alert( - # message=f"Router: Cooling down Deployment:\nModel Name: `{_model_name}`\nAPI Base: `{_api_base}`\nCooldown Time: `{cooldown_time} seconds`\nException Status Code: `{str(exception_status)}`\n\nChange 'cooldown_time' + 'allowed_fails' under 'Router Settings' on proxy UI, or via config - https://docs.litellm.ai/docs/proxy/reliability#fallbacks--retries--timeouts--cooldowns", - # alert_type="cooldown_deployment", - # level="Low", - # ) - # ) - except Exception as e: - pass - def set_custom_routing_strategy( self, CustomRoutingStrategy: CustomRoutingStrategyBase ): diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py new file mode 100644 index 00000000000..00e89274bce --- /dev/null +++ b/litellm/router_utils/cooldown_callbacks.py @@ -0,0 +1,51 @@ +""" +Callbacks triggered on cooling down deployments +""" + +import copy +from typing import TYPE_CHECKING, Any, Union + +import litellm +from litellm._logging import verbose_logger + +if TYPE_CHECKING: + from litellm.router import Router as _Router + + LitellmRouter = _Router +else: + LitellmRouter = Any + + +async def router_cooldown_handler( + litellm_router_instance: LitellmRouter, + deployment_id: str, + exception_status: Union[str, int], + cooldown_time: float, +): + _deployment = litellm_router_instance.get_deployment(model_id=deployment_id) + if _deployment is None: + verbose_logger.warning( + f"in router_cooldown_handler but _deployment is None for deployment_id={deployment_id}. Doing nothing" + ) + return + _litellm_params = _deployment["litellm_params"] + temp_litellm_params = copy.deepcopy(_litellm_params) + temp_litellm_params = dict(temp_litellm_params) + _model_name = _deployment.get("model_name", None) + _api_base = litellm.get_api_base( + model=_model_name, optional_params=temp_litellm_params + ) + model_info = _deployment["model_info"] + model_id = model_info.id + + # Trigger cooldown on Prometheus + from litellm.litellm_core_utils.litellm_logging import prometheusLogger + + if prometheusLogger is not None: + prometheusLogger.set_llm_outage_metric( + litellm_model_name=_model_name, + model_id=model_id, + api_base="", + api_provider="", + ) + pass From 2f402b1cdd67f99eeaa357d52b6f32f12a060fe7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 11:08:06 -0700 Subject: [PATCH 08/17] fix(vertex_ai_partner.py): default vertex ai llama3.1 api to use all openai params Poor vertex docs - not clear what can/can't work Fixes https://github.com/BerriAI/litellm/issues/5090 --- litellm/llms/vertex_ai_partner.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/litellm/llms/vertex_ai_partner.py b/litellm/llms/vertex_ai_partner.py index 08780be7651..378ee7290dd 100644 --- a/litellm/llms/vertex_ai_partner.py +++ b/litellm/llms/vertex_ai_partner.py @@ -94,18 +94,14 @@ class VertexAILlama3Config: } def get_supported_openai_params(self): - return [ - "max_tokens", - "stream", - ] + return litellm.OpenAIConfig().get_supported_openai_params(model="gpt-3.5-turbo") def map_openai_params(self, non_default_params: dict, optional_params: dict): - for param, value in non_default_params.items(): - if param == "max_tokens": - optional_params["max_tokens"] = value - if param == "stream": - optional_params["stream"] = value - return optional_params + return litellm.OpenAIConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-3.5-turbo", + ) class VertexAIPartnerModels(BaseLLM): From ff386f6b6009b5f705aed5b2f5220627bd7bae22 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 11:14:05 -0700 Subject: [PATCH 09/17] fix(utils.py): support deepseek tool calling Fixes https://github.com/BerriAI/litellm/issues/5081 --- litellm/tests/test_completion.py | 23 +++++++++++++++++++++-- litellm/utils.py | 24 ++++++++---------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index eec163f26a4..aee2068ddfe 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -4085,9 +4085,28 @@ async def test_acompletion_gemini(): def test_completion_deepseek(): litellm.set_verbose = True model_name = "deepseek/deepseek-chat" - messages = [{"role": "user", "content": "Hey, how's it going?"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather of an location, the user shoud supply a location first", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } + }, + "required": ["location"], + }, + }, + }, + ] + messages = [{"role": "user", "content": "How's the weather in Hangzhou?"}] try: - response = completion(model=model_name, messages=messages) + response = completion(model=model_name, messages=messages, tools=tools) # Add any assertions here to check the response print(response) except litellm.APIError as e: diff --git a/litellm/utils.py b/litellm/utils.py index 20beb47dc2f..e1a686eaf76 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3536,22 +3536,11 @@ def get_optional_params( ) _check_valid_arg(supported_params=supported_params) - if frequency_penalty is not None: - optional_params["frequency_penalty"] = frequency_penalty - if max_tokens is not None: - optional_params["max_tokens"] = max_tokens - if presence_penalty is not None: - optional_params["presence_penalty"] = presence_penalty - if stop is not None: - optional_params["stop"] = stop - if stream is not None: - optional_params["stream"] = stream - if temperature is not None: - optional_params["temperature"] = temperature - if logprobs is not None: - optional_params["logprobs"] = logprobs - if top_logprobs is not None: - optional_params["top_logprobs"] = top_logprobs + optional_params = litellm.OpenAIConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + ) elif custom_llm_provider == "openrouter": supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider @@ -4141,12 +4130,15 @@ def get_supported_openai_params( "frequency_penalty", "max_tokens", "presence_penalty", + "response_format", "stop", "stream", "temperature", "top_p", "logprobs", "top_logprobs", + "tools", + "tool_choice", ] elif custom_llm_provider == "cohere": return [ From 579f09b33440f07d5dde389d7a74d193c89c2a03 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 11:27:05 -0700 Subject: [PATCH 10/17] fix logging cool down deployment --- litellm/integrations/prometheus.py | 102 +++++++++++++++------ litellm/router_utils/cooldown_callbacks.py | 4 +- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 0c8df96bfb5..06ec7118629 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -132,9 +132,9 @@ class PrometheusLogger: "api_provider", ] - self.deployment_unhealthy = Gauge( - "deployment_unhealthy", - 'Value is "1" when deployment is in an unhealthy state', + self.deployment_complete_outage = Gauge( + "deployment_complete_outage", + 'Value is "1" when deployment is in cooldown and has had a complete outage', labelnames=_logged_llm_labels, ) self.deployment_partial_outage = Gauge( @@ -303,34 +303,17 @@ class PrometheusLogger: log these labels ["litellm_model_name", "model_id", "api_base", "api_provider"] """ - self.deployment_partial_outage.labels( - litellm_model_name, model_id, api_base, llm_provider - ).set(1) - - self.deployment_healthy.labels( - litellm_model_name, model_id, api_base, llm_provider - ).set(0) + self.set_deployment_partial_outage( + litellm_model_name=litellm_model_name, + model_id=model_id, + api_base=api_base, + llm_provider=llm_provider, + ) pass except: pass - def set_llm_outage_metric( - self, - litellm_model_name: str, - model_id: str, - api_base: str, - llm_provider: str, - ): - """ - log these labels - ["litellm_model_name", "model_id", "api_base", "api_provider"] - """ - self.deployment_unhealthy.labels( - litellm_model_name, model_id, api_base, llm_provider - ).set(1) - pass - def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") @@ -380,9 +363,12 @@ class PrometheusLogger: log these labels ["litellm_model_name", "model_id", "api_base", "api_provider"] """ - self.deployment_healthy.labels( - litellm_model_name, model_id, api_base, llm_provider - ).set(1) + self.set_deployment_healthy( + litellm_model_name=litellm_model_name, + model_id=model_id, + api_base=api_base, + llm_provider=llm_provider, + ) except Exception as e: verbose_logger.error( "Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {}".format( @@ -391,6 +377,64 @@ class PrometheusLogger: ) return + def set_deployment_healthy( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + def set_deployment_complete_outage( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + verbose_logger.debug("setting llm outage metric") + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + def set_deployment_partial_outage( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + def safe_get_remaining_budget( max_budget: Optional[float], spend: Optional[float] diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py index 00e89274bce..3a5213ec03d 100644 --- a/litellm/router_utils/cooldown_callbacks.py +++ b/litellm/router_utils/cooldown_callbacks.py @@ -42,10 +42,10 @@ async def router_cooldown_handler( from litellm.litellm_core_utils.litellm_logging import prometheusLogger if prometheusLogger is not None: - prometheusLogger.set_llm_outage_metric( + prometheusLogger.set_deployment_complete_outage( litellm_model_name=_model_name, model_id=model_id, api_base="", - api_provider="", + llm_provider="", ) pass From 8d1f051d8cfbc65269239f63b4891f2263dac0a5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 11:37:05 -0700 Subject: [PATCH 11/17] docs prometheus --- docs/my-website/docs/proxy/prometheus.md | 73 ++++++++---------------- 1 file changed, 23 insertions(+), 50 deletions(-) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 61d1397ac21..24bf08e4124 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -3,6 +3,13 @@ import TabItem from '@theme/TabItem'; # πŸ“ˆ Prometheus metrics [BETA] +:::info +🚨 Prometheus Metrics will be moving to LiteLLM Enterprise by September 15th, 2024 + +[Contact us here to get a license](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + LiteLLM Exposes a `/metrics` endpoint for Prometheus to Poll ## Quick Start @@ -47,9 +54,11 @@ http://localhost:4000/metrics # /metrics ``` -## Metrics Tracked +## πŸ“ˆ Metrics Tracked +### Proxy Requests / Spend Metrics + | Metric Name | Description | |----------------------|--------------------------------------| | `litellm_requests_metric` | Number of requests made, per `"user", "key", "model", "team", "end-user"` | @@ -57,6 +66,19 @@ http://localhost:4000/metrics | `litellm_total_tokens` | input + output tokens per `"user", "key", "model", "team", "end-user"` | | `litellm_llm_api_failed_requests_metric` | Number of failed LLM API requests per `"user", "key", "model", "team", "end-user"` | +### LLM API / Provider Metrics + +| Metric Name | Description | +|----------------------|--------------------------------------| +| `deployment_complete_outage` | Value is "1" when deployment is in cooldown and has had a complete outage. This metric tracks the state of the LLM API Deployment when it's completely unavailable. | +| `deployment_partial_outage` | Value is "1" when deployment is experiencing a partial outage. This metric indicates when the LLM API Deployment is facing issues but is not completely down. | +| `deployment_healthy` | Value is "1" when deployment is in a healthy state. This metric shows when the LLM API Deployment is functioning normally without any outages. | +| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment | +| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment | + + + + ### Budget Metrics | Metric Name | Description | |----------------------|--------------------------------------| @@ -64,55 +86,6 @@ http://localhost:4000/metrics | `litellm_remaining_api_key_budget_metric` | Remaining Budget for API Key (A key Created on LiteLLM)| -### ✨ (Enterprise) LLM Remaining Requests and Remaining Tokens -Set this on your config.yaml to allow you to track how close you are to hitting your TPM / RPM limits on each model group - -```yaml -litellm_settings: - success_callback: ["prometheus"] - failure_callback: ["prometheus"] - return_response_headers: true # ensures the LLM API calls track the response headers -``` - -| Metric Name | Description | -|----------------------|--------------------------------------| -| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment | -| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment | - -Example Metric - - - - -```shell -litellm_remaining_requests -{ - api_base="https://api.openai.com/v1", - api_provider="openai", - litellm_model_name="gpt-3.5-turbo", - model_group="gpt-3.5-turbo" -} -8998.0 -``` - - - - - -```shell -litellm_remaining_tokens -{ - api_base="https://api.openai.com/v1", - api_provider="openai", - litellm_model_name="gpt-3.5-turbo", - model_group="gpt-3.5-turbo" -} -999981.0 -``` - - - - ## Monitor System Health From 72aebe5e59d42dbc287ee873bf63d49af2592fad Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 12:10:47 -0700 Subject: [PATCH 12/17] docs link to enteprise pricing --- docs/my-website/docs/proxy/prometheus.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 24bf08e4124..b3306b1acad 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -6,6 +6,7 @@ import TabItem from '@theme/TabItem'; :::info 🚨 Prometheus Metrics will be moving to LiteLLM Enterprise by September 15th, 2024 +[Enterprise Pricing](https://www.litellm.ai/#pricing) [Contact us here to get a license](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) ::: From 958e0fdfabaa6210fb80510a1037d04452249f18 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 12:46:26 -0700 Subject: [PATCH 13/17] show warning about prometheus moving to enterprise --- docs/my-website/docs/proxy/prometheus.md | 3 ++- litellm/integrations/prometheus.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index b3306b1acad..609c00f8eb2 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -4,9 +4,10 @@ import TabItem from '@theme/TabItem'; # πŸ“ˆ Prometheus metrics [BETA] :::info -🚨 Prometheus Metrics will be moving to LiteLLM Enterprise by September 15th, 2024 +🚨 Prometheus Metrics will be moving to LiteLLM Enterprise on September 15th, 2024 [Enterprise Pricing](https://www.litellm.ai/#pricing) + [Contact us here to get a license](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) ::: diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 06ec7118629..61f4ff02a6c 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -28,6 +28,10 @@ class PrometheusLogger: from litellm.proxy.proxy_server import premium_user + verbose_logger.warning( + "🚨🚨🚨 Prometheus Metrics will be moving to LiteLLM Enterprise on September 15th, 2024.\n🚨 Contact us here to get a license https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat \n🚨 Enterprise Pricing: https://www.litellm.ai/#pricing" + ) + self.litellm_llm_api_failed_requests_metric = Counter( name="litellm_llm_api_failed_requests_metric", documentation="Total number of failed LLM API calls via litellm", From 61ccd5354bf4d0a7970d6cdcf2ea4d389538fc23 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 12:47:06 -0700 Subject: [PATCH 14/17] docs prometheus --- docs/my-website/docs/proxy/prometheus.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 609c00f8eb2..991b08c7b4d 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem'; [Enterprise Pricing](https://www.litellm.ai/#pricing) -[Contact us here to get a license](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) +[Contact us here to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) ::: From 1bf36cd7a40cd4c60940797f9cc73600851c105c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 12:50:03 -0700 Subject: [PATCH 15/17] docs prom metrics --- docs/my-website/docs/enterprise.md | 3 ++- docs/my-website/docs/proxy/enterprise.md | 3 ++- docs/my-website/docs/proxy/prometheus.md | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index fc85333b587..19e45bebf03 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -36,7 +36,8 @@ This covers: - βœ… [Tracking Spend for Custom Tags](./proxy/enterprise#tracking-spend-for-custom-tags) - βœ… [Exporting LLM Logs to GCS Bucket](./proxy/bucket#πŸͺ£-logging-gcs-s3-buckets) - βœ… [API Endpoints to get Spend Reports per Team, API Key, Customer](./proxy/cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend) - - **Advanced Metrics** + - **Prometheus Metrics** + - βœ… [Prometheus Metrics - Num Requests, failures, LLM Provider Outages](./proxy/prometheus) - βœ… [`x-ratelimit-remaining-requests`, `x-ratelimit-remaining-tokens` for LLM APIs on Prometheus](./proxy/prometheus#✨-enterprise-llm-remaining-requests-and-remaining-tokens) - **Guardrails, PII Masking, Content Moderation** - βœ… [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](./proxy/enterprise#content-moderation) diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index d602756812f..33a899222b1 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -30,7 +30,8 @@ Features: - βœ… [Tracking Spend for Custom Tags](#tracking-spend-for-custom-tags) - βœ… [Exporting LLM Logs to GCS Bucket](./proxy/bucket#πŸͺ£-logging-gcs-s3-buckets) - βœ… [`/spend/report` API endpoint](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend) -- **Advanced Metrics** +- **Prometheus Metrics** + - βœ… [Prometheus Metrics - Num Requests, failures, LLM Provider Outages](prometheus) - βœ… [`x-ratelimit-remaining-requests`, `x-ratelimit-remaining-tokens` for LLM APIs on Prometheus](prometheus#✨-enterprise-llm-remaining-requests-and-remaining-tokens) - **Guardrails, PII Masking, Content Moderation** - βœ… [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](#content-moderation) diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 991b08c7b4d..e61ccb1d656 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# πŸ“ˆ Prometheus metrics [BETA] +# πŸ“ˆ Prometheus metrics :::info 🚨 Prometheus Metrics will be moving to LiteLLM Enterprise on September 15th, 2024 From f0f900d69e669ec6ef3fb74ff567462cd3a2f649 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 7 Aug 2024 13:02:03 -0700 Subject: [PATCH 16/17] fix(router.py): add reason for fallback failure to client-side exception string make it easier to debug why a fallback failed to occur --- litellm/proxy/_new_secret_config.yaml | 13 ++++++++++--- litellm/router.py | 15 +++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 1fdcc5e9378..f00d5ec3e75 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,7 +1,14 @@ model_list: - - model_name: "*" + - model_name: "gpt-3.5-turbo" litellm_params: - model: "*" + model: "gpt-3.5-turbo" + - model_name: "gpt-4" + litellm_params: + model: "gpt-4" + api_key: "bad_key" + - model_name: "gpt-4o" + litellm_params: + model: "gpt-4o" litellm_settings: - callbacks: ["lakera_prompt_injection"] + fallbacks: [{"gpt-3.5-turbo": ["gpt-4", "gpt-4o"]}] diff --git a/litellm/router.py b/litellm/router.py index a6ec01b06b8..5a4d83885f9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2317,8 +2317,10 @@ class Router: ) try: if mock_testing_fallbacks is not None and mock_testing_fallbacks is True: - raise Exception( - f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}" + raise litellm.InternalServerError( + model=model_group, + llm_provider="", + message=f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}", ) elif ( mock_testing_context_fallbacks is not None @@ -2348,6 +2350,7 @@ class Router: verbose_router_logger.debug(f"Traceback{traceback.format_exc()}") original_exception = e fallback_model_group = None + fallback_failure_exception_str = "" try: verbose_router_logger.debug("Trying to fallback b/w models") if ( @@ -2506,6 +2509,7 @@ class Router: await self._async_get_cooldown_deployments_with_debug_info(), ) ) + fallback_failure_exception_str = str(new_exception) if hasattr(original_exception, "message"): # add the available fallbacks to the exception @@ -2513,6 +2517,13 @@ class Router: model_group, fallback_model_group, ) + if len(fallback_failure_exception_str) > 0: + original_exception.message += ( + "\nError doing the fallback: {}".format( + fallback_failure_exception_str + ) + ) + raise original_exception async def async_function_with_retries(self, *args, **kwargs): From 0b98959e6dccb64d99c86f06f464b0fb41e0f7c5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 13:12:19 -0700 Subject: [PATCH 17/17] gemini test skip internal server error --- litellm/tests/test_completion.py | 98 +++++++++++++++++--------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index aee2068ddfe..a07aac5bf10 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -892,57 +892,65 @@ def test_completion_claude_3_base64(): "model", ["gemini/gemini-1.5-flash"] # "claude-3-sonnet-20240229", ) def test_completion_function_plus_image(model): - litellm.set_verbose = True + try: + litellm.set_verbose = True - image_content = [ - {"type": "text", "text": "What’s in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" - }, - }, - ] - image_message = {"role": "user", "content": image_content} - - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], + image_content = [ + {"type": "text", "text": "What’s in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" }, }, - } - ] + ] + image_message = {"role": "user", "content": image_content} - tool_choice = {"type": "function", "function": {"name": "get_current_weather"}} - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ] + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] - response = completion( - model=model, - messages=[image_message], - tool_choice=tool_choice, - tools=tools, - stream=False, - ) + tool_choice = {"type": "function", "function": {"name": "get_current_weather"}} + messages = [ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ] - print(response) + response = completion( + model=model, + messages=[image_message], + tool_choice=tool_choice, + tools=tools, + stream=False, + ) + + print(response) + except litellm.InternalServerError: + pass + except Exception as e: + pytest.fail(f"error occurred: {str(e)}") @pytest.mark.parametrize(