diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index 61d1397ac21..b3306b1acad 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -3,6 +3,14 @@ import TabItem from '@theme/TabItem'; # 📈 Prometheus metrics [BETA] +:::info +🚨 Prometheus Metrics will be moving to LiteLLM Enterprise by September 15th, 2024 + +[Enterprise Pricing](https://www.litellm.ai/#pricing) +[Contact us here to get a license](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + LiteLLM Exposes a `/metrics` endpoint for Prometheus to Poll ## Quick Start @@ -47,9 +55,11 @@ http://localhost:4000/metrics # /metrics ``` -## Metrics Tracked +## 📈 Metrics Tracked +### Proxy Requests / Spend Metrics + | Metric Name | Description | |----------------------|--------------------------------------| | `litellm_requests_metric` | Number of requests made, per `"user", "key", "model", "team", "end-user"` | @@ -57,6 +67,19 @@ http://localhost:4000/metrics | `litellm_total_tokens` | input + output tokens per `"user", "key", "model", "team", "end-user"` | | `litellm_llm_api_failed_requests_metric` | Number of failed LLM API requests per `"user", "key", "model", "team", "end-user"` | +### LLM API / Provider Metrics + +| Metric Name | Description | +|----------------------|--------------------------------------| +| `deployment_complete_outage` | Value is "1" when deployment is in cooldown and has had a complete outage. This metric tracks the state of the LLM API Deployment when it's completely unavailable. | +| `deployment_partial_outage` | Value is "1" when deployment is experiencing a partial outage. This metric indicates when the LLM API Deployment is facing issues but is not completely down. | +| `deployment_healthy` | Value is "1" when deployment is in a healthy state. This metric shows when the LLM API Deployment is functioning normally without any outages. | +| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment | +| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment | + + + + ### Budget Metrics | Metric Name | Description | |----------------------|--------------------------------------| @@ -64,55 +87,6 @@ http://localhost:4000/metrics | `litellm_remaining_api_key_budget_metric` | Remaining Budget for API Key (A key Created on LiteLLM)| -### ✨ (Enterprise) LLM Remaining Requests and Remaining Tokens -Set this on your config.yaml to allow you to track how close you are to hitting your TPM / RPM limits on each model group - -```yaml -litellm_settings: - success_callback: ["prometheus"] - failure_callback: ["prometheus"] - return_response_headers: true # ensures the LLM API calls track the response headers -``` - -| Metric Name | Description | -|----------------------|--------------------------------------| -| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment | -| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment | - -Example Metric - - - - -```shell -litellm_remaining_requests -{ - api_base="https://api.openai.com/v1", - api_provider="openai", - litellm_model_name="gpt-3.5-turbo", - model_group="gpt-3.5-turbo" -} -8998.0 -``` - - - - - -```shell -litellm_remaining_tokens -{ - api_base="https://api.openai.com/v1", - api_provider="openai", - litellm_model_name="gpt-3.5-turbo", - model_group="gpt-3.5-turbo" -} -999981.0 -``` - - - - ## Monitor System Health diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 4a271d6e004..06ec7118629 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -8,7 +8,7 @@ import subprocess import sys import traceback import uuid -from typing import Optional, Union +from typing import Optional, TypedDict, Union import dotenv import requests # type: ignore @@ -124,6 +124,29 @@ class PrometheusLogger: "litellm_model_name", ], ) + # Get all keys + _logged_llm_labels = [ + "litellm_model_name", + "model_id", + "api_base", + "api_provider", + ] + + self.deployment_complete_outage = Gauge( + "deployment_complete_outage", + 'Value is "1" when deployment is in cooldown and has had a complete outage', + labelnames=_logged_llm_labels, + ) + self.deployment_partial_outage = Gauge( + "deployment_partial_outage", + 'Value is "1" when deployment is experiencing a partial outage', + labelnames=_logged_llm_labels, + ) + self.deployment_healthy = Gauge( + "deployment_healthy", + 'Value is "1" when deployment is in an healthy state', + labelnames=_logged_llm_labels, + ) except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") @@ -243,7 +266,7 @@ class PrometheusLogger: # set x-ratelimit headers if premium_user is True: - self.set_remaining_tokens_requests_metric(kwargs) + self.set_llm_deployment_success_metrics(kwargs) ### FAILURE INCREMENT ### if "exception" in kwargs: @@ -256,6 +279,8 @@ class PrometheusLogger: user_api_team_alias, user_id, ).inc() + + self.set_llm_deployment_failure_metrics(kwargs) except Exception as e: verbose_logger.error( "prometheus Layer Error(): Exception occured - {}".format(str(e)) @@ -263,7 +288,33 @@ class PrometheusLogger: verbose_logger.debug(traceback.format_exc()) pass - def set_remaining_tokens_requests_metric(self, request_kwargs: dict): + def set_llm_deployment_failure_metrics(self, request_kwargs: dict): + try: + verbose_logger.debug("setting remaining tokens requests metric") + _response_headers = request_kwargs.get("response_headers") + _litellm_params = request_kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) + litellm_model_name = request_kwargs.get("model", None) + api_base = _metadata.get("api_base", None) + llm_provider = _litellm_params.get("custom_llm_provider", None) + model_id = _metadata.get("model_id") + + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.set_deployment_partial_outage( + litellm_model_name=litellm_model_name, + model_id=model_id, + api_base=api_base, + llm_provider=llm_provider, + ) + + pass + except: + pass + + def set_llm_deployment_success_metrics(self, request_kwargs: dict): try: verbose_logger.debug("setting remaining tokens requests metric") _response_headers = request_kwargs.get("response_headers") @@ -273,6 +324,7 @@ class PrometheusLogger: model_group = _metadata.get("model_group", None) api_base = _metadata.get("api_base", None) llm_provider = _litellm_params.get("custom_llm_provider", None) + model_id = _metadata.get("model_id") remaining_requests = None remaining_tokens = None @@ -307,14 +359,82 @@ class PrometheusLogger: model_group, llm_provider, api_base, litellm_model_name ).set(remaining_tokens) + """ + log these labels + ["litellm_model_name", "model_id", "api_base", "api_provider"] + """ + self.set_deployment_healthy( + litellm_model_name=litellm_model_name, + model_id=model_id, + api_base=api_base, + llm_provider=llm_provider, + ) except Exception as e: verbose_logger.error( - "Prometheus Error: set_remaining_tokens_requests_metric. Exception occured - {}".format( + "Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {}".format( str(e) ) ) return + def set_deployment_healthy( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + def set_deployment_complete_outage( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + verbose_logger.debug("setting llm outage metric") + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + def set_deployment_partial_outage( + self, + litellm_model_name: str, + model_id: str, + api_base: str, + llm_provider: str, + ): + self.deployment_complete_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + + self.deployment_partial_outage.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(1) + + self.deployment_healthy.labels( + litellm_model_name, model_id, api_base, llm_provider + ).set(0) + def safe_get_remaining_budget( max_budget: Optional[float], spend: Optional[float] diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 97cd407d324..36b191c90a5 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,7 +3,7 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: https://exampleopenaiendpoint-production.up.railwaz.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct @@ -50,4 +50,6 @@ general_settings: litellm_settings: - callbacks: ["otel"] # 👈 KEY CHANGE \ No newline at end of file + callbacks: ["otel"] # 👈 KEY CHANGE + success_callback: ["prometheus"] + failure_callback: ["prometheus"] \ No newline at end of file diff --git a/litellm/router.py b/litellm/router.py index aa9768ba440..a6ec01b06b8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -57,6 +57,7 @@ from litellm.router_utils.client_initalization_utils import ( set_client, should_initialize_sync_client, ) +from litellm.router_utils.cooldown_callbacks import router_cooldown_handler from litellm.router_utils.handle_error import send_llm_exception_alert from litellm.scheduler import FlowItem, Scheduler from litellm.types.llms.openai import ( @@ -3294,10 +3295,14 @@ class Router: value=cached_value, key=cooldown_key, ttl=cooldown_time ) - self.send_deployment_cooldown_alert( - deployment_id=deployment, - exception_status=exception_status, - cooldown_time=cooldown_time, + # Trigger cooldown handler + asyncio.create_task( + router_cooldown_handler( + litellm_router_instance=self, + deployment_id=deployment, + exception_status=exception_status, + cooldown_time=cooldown_time, + ) ) else: self.failed_calls.set_cache( @@ -4948,42 +4953,6 @@ class Router: ) print("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") # noqa - def send_deployment_cooldown_alert( - self, - deployment_id: str, - exception_status: Union[str, int], - cooldown_time: float, - ): - try: - from litellm.proxy.proxy_server import proxy_logging_obj - - # trigger slack alert saying deployment is in cooldown - if ( - proxy_logging_obj is not None - and proxy_logging_obj.alerting is not None - and "slack" in proxy_logging_obj.alerting - ): - _deployment = self.get_deployment(model_id=deployment_id) - if _deployment is None: - return - - _litellm_params = _deployment["litellm_params"] - temp_litellm_params = copy.deepcopy(_litellm_params) - temp_litellm_params = dict(temp_litellm_params) - _model_name = _deployment.get("model_name", None) - _api_base = litellm.get_api_base( - model=_model_name, optional_params=temp_litellm_params - ) - # asyncio.create_task( - # proxy_logging_obj.slack_alerting_instance.send_alert( - # message=f"Router: Cooling down Deployment:\nModel Name: `{_model_name}`\nAPI Base: `{_api_base}`\nCooldown Time: `{cooldown_time} seconds`\nException Status Code: `{str(exception_status)}`\n\nChange 'cooldown_time' + 'allowed_fails' under 'Router Settings' on proxy UI, or via config - https://docs.litellm.ai/docs/proxy/reliability#fallbacks--retries--timeouts--cooldowns", - # alert_type="cooldown_deployment", - # level="Low", - # ) - # ) - except Exception as e: - pass - def set_custom_routing_strategy( self, CustomRoutingStrategy: CustomRoutingStrategyBase ): diff --git a/litellm/router_utils/cooldown_callbacks.py b/litellm/router_utils/cooldown_callbacks.py new file mode 100644 index 00000000000..3a5213ec03d --- /dev/null +++ b/litellm/router_utils/cooldown_callbacks.py @@ -0,0 +1,51 @@ +""" +Callbacks triggered on cooling down deployments +""" + +import copy +from typing import TYPE_CHECKING, Any, Union + +import litellm +from litellm._logging import verbose_logger + +if TYPE_CHECKING: + from litellm.router import Router as _Router + + LitellmRouter = _Router +else: + LitellmRouter = Any + + +async def router_cooldown_handler( + litellm_router_instance: LitellmRouter, + deployment_id: str, + exception_status: Union[str, int], + cooldown_time: float, +): + _deployment = litellm_router_instance.get_deployment(model_id=deployment_id) + if _deployment is None: + verbose_logger.warning( + f"in router_cooldown_handler but _deployment is None for deployment_id={deployment_id}. Doing nothing" + ) + return + _litellm_params = _deployment["litellm_params"] + temp_litellm_params = copy.deepcopy(_litellm_params) + temp_litellm_params = dict(temp_litellm_params) + _model_name = _deployment.get("model_name", None) + _api_base = litellm.get_api_base( + model=_model_name, optional_params=temp_litellm_params + ) + model_info = _deployment["model_info"] + model_id = model_info.id + + # Trigger cooldown on Prometheus + from litellm.litellm_core_utils.litellm_logging import prometheusLogger + + if prometheusLogger is not None: + prometheusLogger.set_deployment_complete_outage( + litellm_model_name=_model_name, + model_id=model_id, + api_base="", + llm_provider="", + ) + pass