scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

Merge pull request #5134 from BerriAI/litellm_refactor_promt

Browse source 
[Feat] v2 prometheus deployment outage, healthy, partial outage alerting
This commit is contained in:
Ishaan Jaff 2024-08-09 12:18:40 -07:00 • committed by GitHub
commit f260c1a043
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 54 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
docs/my-website/docs/proxy/prometheus.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# 📈 [BETA] Prometheus metrics
:::info
🚨 Prometheus Metrics will be moving to LiteLLM Enterprise on September 15th, 2024
🚨 Prometheus metrics will be out of Beta on September 15, 2024 - as part of this release it will be on LiteLLM Enterprise starting at $250/mo
[Enterprise Pricing](https://www.litellm.ai/#pricing)
@ -72,9 +72,7 @@ http://localhost:4000/metrics
| Metric Name | Description |
|----------------------|--------------------------------------|
| `deployment_complete_outage` | Value is "1" when deployment is in cooldown and has had a complete outage. This metric tracks the state of the LLM API Deployment when it's completely unavailable. |
| `deployment_partial_outage` | Value is "1" when deployment is experiencing a partial outage. This metric indicates when the LLM API Deployment is facing issues but is not completely down. |
| `deployment_healthy` | Value is "1" when deployment is in a healthy state. This metric shows when the LLM API Deployment is functioning normally without any outages. |
| `deployment_state` | The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage. |
| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment |
| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment |

94
litellm/integrations/prometheus.py
View file

@ -136,19 +136,10 @@ class PrometheusLogger:
"api_provider",
]
self.deployment_complete_outage = Gauge(
"deployment_complete_outage",
'Value is "1" when deployment is in cooldown and has had a complete outage',
labelnames=_logged_llm_labels,
)
self.deployment_partial_outage = Gauge(
"deployment_partial_outage",
'Value is "1" when deployment is experiencing a partial outage',
labelnames=_logged_llm_labels,
)
self.deployment_healthy = Gauge(
"deployment_healthy",
'Value is "1" when deployment is in an healthy state',
# Metric for deployment state
self.deployment_state = Gauge(
"deployment_state",
"The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
labelnames=_logged_llm_labels,
)
@ -311,7 +302,7 @@ class PrometheusLogger:
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
llm_provider=llm_provider,
api_provider=llm_provider,
)
pass
@ -371,7 +362,7 @@ class PrometheusLogger:
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
llm_provider=llm_provider,
api_provider=llm_provider,
)
except Exception as e:
verbose_logger.error(
@ -381,63 +372,50 @@ class PrometheusLogger:
)
return
def set_deployment_state(
self,
state: int,
litellm_model_name: str,
model_id: str,
api_base: str,
api_provider: str,
):
self.deployment_state.labels(
litellm_model_name, model_id, api_base, api_provider
).set(state)
def set_deployment_healthy(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
llm_provider: str,
api_provider: str,
):
self.deployment_complete_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
self.deployment_partial_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
self.deployment_healthy.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(1)
def set_deployment_complete_outage(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
llm_provider: str,
):
verbose_logger.debug("setting llm outage metric")
self.deployment_complete_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(1)
self.deployment_partial_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
self.deployment_healthy.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
self.set_deployment_state(
0, litellm_model_name, model_id, api_base, api_provider
)
def set_deployment_partial_outage(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
llm_provider: str,
api_provider: str,
):
self.deployment_complete_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
self.set_deployment_state(
1, litellm_model_name, model_id, api_base, api_provider
)
self.deployment_partial_outage.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(1)
self.deployment_healthy.labels(
litellm_model_name, model_id, api_base, llm_provider
).set(0)
def set_deployment_complete_outage(
self,
litellm_model_name: str,
model_id: str,
api_base: str,
api_provider: str,
):
self.set_deployment_state(
2, litellm_model_name, model_id, api_base, api_provider
)
def safe_get_remaining_budget(

3
litellm/proxy/proxy_config.yaml
View file

@ -37,4 +37,5 @@ general_settings:
master_key: sk-1234
litellm_settings:
callbacks: ["otel"] # 👈 KEY CHANGE
success_callback: ["prometheus"]
failure_callback: ["prometheus"]

17
litellm/router_utils/cooldown_callbacks.py
View file

@ -38,6 +38,17 @@ async def router_cooldown_handler(
model_info = _deployment["model_info"]
model_id = model_info.id
litellm_model_name = temp_litellm_params.get("model")
llm_provider = ""
try:
_, llm_provider, _, _ = litellm.get_llm_provider(
model=litellm_model_name,
custom_llm_provider=temp_litellm_params.get("custom_llm_provider"),
)
except:
pass
# Trigger cooldown on Prometheus
from litellm.litellm_core_utils.litellm_logging import prometheusLogger
@ -45,7 +56,7 @@ async def router_cooldown_handler(
prometheusLogger.set_deployment_complete_outage(
litellm_model_name=_model_name,
model_id=model_id,
api_base="",
llm_provider="",
api_base=_api_base,
api_provider=llm_provider,
)
pass
return