mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
Merge pull request #41578 from BerriAI/litellm_grafana_all_prometheus_metrics_dashboard
feat(grafana): add all-metrics dashboard and fix stale dashboard_v2 gauges
This commit is contained in:
yucheng-berri
GitHub
commit
4c70cb4815
7 changed files with 6507 additions and 623 deletions
|
|
@ -1,614 +0,0 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": 2039,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
|
||||
"legendFormat": "Time to first token",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Time to first token (latency)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "currencyUSD"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "7e4b0627fd32efdd2313c846325575808aadcf2839f0fde90723aab9ab73c78f"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Translata"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
|
||||
"legendFormat": "{{team}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Spend by team",
|
||||
"transformations": [],
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
|
||||
"legendFormat": "{{model}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Requests by model",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"noValue": "0",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 3,
|
||||
"x": 0,
|
||||
"y": 25
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "9.4.17",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Faild Requests",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "currencyUSD"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 3,
|
||||
"x": 3,
|
||||
"y": 25
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
|
||||
"legendFormat": "{{model}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Spend",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 25
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Tokens",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "1m",
|
||||
"revision": 1,
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "prometheus",
|
||||
"value": "edx8memhpd9tsa"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"queryValue": "",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "LLM Proxy",
|
||||
"uid": "rgRrHxESz",
|
||||
"version": 15,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
## This folder contains the `json` for creating the following Grafana Dashboard
|
||||
|
||||
### Pre-Requisites
|
||||
- Setup LiteLLM Proxy Prometheus Metrics https://docs.litellm.ai/docs/proxy/prometheus
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,11 @@
|
|||
# LiteLLM All Prometheus Metrics dashboard
|
||||
|
||||
Every `litellm_*` metric family the proxy can expose on `/metrics` (134 families across 95 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
|
||||
|
||||
Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
|
||||
|
||||
The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
|
||||
|
||||
## Pre-requisites
|
||||
|
||||
Prometheus metrics on the proxy: https://docs.litellm.ai/docs/proxy/prometheus
|
||||
|
|
@ -476,7 +476,7 @@
|
|||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "topk(5, sort(litellm_remaining_requests))",
|
||||
"expr": "topk(5, sort(litellm_remaining_requests_metric))",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
|
|
@ -573,7 +573,7 @@
|
|||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "topk(5, sort(litellm_remaining_tokens))",
|
||||
"expr": "topk(5, sort(litellm_remaining_tokens_metric))",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,14 @@ This folder contains the `json` for creating Grafana Dashboards
|
|||
|
||||
Charts the `gen_ai.*` metrics from the OpenTelemetry v2 integration: spend, tokens, request rate, and latency percentiles by model. Separate from the dashboards below, which chart the `litellm_*` Prometheus metrics.
|
||||
|
||||
## [LiteLLM All Prometheus Metrics dashboard](./dashboard_all_metrics)
|
||||
|
||||
Every `litellm_*` Prometheus metric family the proxy can emit (134 families, 95 panels) grouped by theme: traffic, latency, spend and tokens, cache, deployments, rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, plus the Redis circuit breaker, spend log cleanup and `prometheus_system` service metrics. Start here if you want everything on one screen; see its [readme](./dashboard_all_metrics/readme.md) for import steps and which panels need a feature enabled before they show data
|
||||
|
||||
## [LiteLLM v2 Dashboard](./dashboard_v2)
|
||||
|
||||
A compact view of proxy request rate, failures, latency and the top remaining-request / remaining-token gauges per model group
|
||||
|
||||
<img width="1316" alt="grafana_1" src="https://github.com/user-attachments/assets/d0df802d-0cb9-4906-a679-941c547789ab">
|
||||
<img width="1289" alt="grafana_2" src="https://github.com/user-attachments/assets/b11f755f-e113-42ab-b21d-83f91f451a28">
|
||||
<img width="1323" alt="grafana_3" src="https://github.com/user-attachments/assets/cb29ffdb-477d-4be1-a5cd-c3f7f2cb21c5">
|
||||
|
|
|
|||
|
|
@ -8,9 +8,184 @@ configuration works correctly.
|
|||
Related issue: https://github.com/BerriAI/litellm/issues/18221
|
||||
"""
|
||||
|
||||
from typing import get_args
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, get_args
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY, Gauge
|
||||
from prometheus_client.registry import Collector
|
||||
|
||||
import litellm
|
||||
from litellm.caching.redis_cache import _breaker_metrics
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.integrations.prometheus_services import PrometheusServicesLogger
|
||||
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup_metrics import SpendLogCleanupMetrics
|
||||
from litellm.proxy.middleware.admission_control_middleware import create_prometheus_admission_metrics
|
||||
from litellm.proxy.middleware.in_flight_requests_middleware import InFlightRequestsMiddleware
|
||||
|
||||
_GRAFANA_DIR: Final = Path(__file__).parents[3] / "cookbook" / "litellm_proxy_server" / "grafana_dashboard"
|
||||
_ALL_METRICS_DASHBOARD: Final = _GRAFANA_DIR / "dashboard_all_metrics" / "grafana_dashboard.json"
|
||||
_LITELLM_DASHBOARDS: Final = (_ALL_METRICS_DASHBOARD, _GRAFANA_DIR / "dashboard_v2" / "grafana_dashboard.json")
|
||||
_METRIC_TOKEN_RE: Final = re.compile(r"\blitellm_[a-z0-9_]+")
|
||||
_BY_CLAUSE_RE: Final = re.compile(r"\bby\s*\([^)]*\)")
|
||||
_EXPOSITION_SUFFIXES: Final = ("", "_total", "_bucket", "_sum", "_count", "_created")
|
||||
|
||||
|
||||
def _registered_collectors() -> MappingProxyType[Collector, tuple[str, ...]]:
|
||||
return MappingProxyType({collector: tuple(names) for collector, names in REGISTRY._collector_to_names.items()})
|
||||
|
||||
|
||||
def _unregister_everything() -> None:
|
||||
for collector in tuple(REGISTRY._collector_to_names):
|
||||
REGISTRY.unregister(collector)
|
||||
|
||||
|
||||
def _register_if_absent(collectors: tuple[Collector, ...]) -> None:
|
||||
for collector in collectors:
|
||||
if collector not in REGISTRY._collector_to_names and not any(
|
||||
name in REGISTRY._names_to_collectors for name in REGISTRY._get_names(collector)
|
||||
):
|
||||
REGISTRY.register(collector)
|
||||
|
||||
|
||||
def _lazy_owner_collectors() -> tuple[Collector, ...]:
|
||||
SpendLogCleanupMetrics._ensure_initialized()
|
||||
assert SpendLogCleanupMetrics.rows_deleted is not None
|
||||
assert SpendLogCleanupMetrics.batch_duration is not None
|
||||
assert SpendLogCleanupMetrics.rows_remaining is not None
|
||||
assert SpendLogCleanupMetrics.batch_failures is not None
|
||||
assert SpendLogCleanupMetrics.runs is not None
|
||||
in_flight: Final = InFlightRequestsMiddleware._get_gauge()
|
||||
assert in_flight is not None
|
||||
breaker: Final = _breaker_metrics()
|
||||
assert breaker._state_gauge is not None
|
||||
assert breaker._transitions is not None
|
||||
assert breaker._failures is not None
|
||||
return (
|
||||
SpendLogCleanupMetrics.rows_deleted,
|
||||
SpendLogCleanupMetrics.batch_duration,
|
||||
SpendLogCleanupMetrics.rows_remaining,
|
||||
SpendLogCleanupMetrics.batch_failures,
|
||||
SpendLogCleanupMetrics.runs,
|
||||
in_flight,
|
||||
breaker._state_gauge,
|
||||
breaker._transitions,
|
||||
breaker._failures,
|
||||
)
|
||||
|
||||
|
||||
def _fresh_admission_collectors() -> tuple[Collector, ...]:
|
||||
admission: Final = create_prometheus_admission_metrics()
|
||||
assert admission is not None
|
||||
return (admission.admitted_gauge, admission.queued_gauge, admission.rejected_counter)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _isolated_litellm_metric_families(monkeypatch: pytest.MonkeyPatch) -> Iterator[frozenset[str]]:
|
||||
previous: Final = _registered_collectors()
|
||||
_unregister_everything()
|
||||
monkeypatch.setattr(litellm, "prometheus_metrics_config", None)
|
||||
PrometheusLogger()
|
||||
PrometheusServicesLogger()
|
||||
lazy_owned: Final = _lazy_owner_collectors()
|
||||
_register_if_absent(lazy_owned)
|
||||
_fresh_admission_collectors()
|
||||
try:
|
||||
yield frozenset(metric.name for metric in REGISTRY.collect())
|
||||
finally:
|
||||
_unregister_everything()
|
||||
for collector in previous:
|
||||
REGISTRY.register(collector)
|
||||
_register_if_absent(lazy_owned)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def emitted_metric_families(monkeypatch: pytest.MonkeyPatch) -> Iterator[frozenset[str]]:
|
||||
with _isolated_litellm_metric_families(monkeypatch) as families:
|
||||
yield families
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gauges_registered_by_an_earlier_test() -> Iterator[tuple[Collector, Collector]]:
|
||||
sentinel: Final = Gauge("litellm_unrelated_sentinel", "registered by a test outside the isolated block")
|
||||
already_registered: Final = REGISTRY._names_to_collectors.get("litellm_admission_admitted_requests")
|
||||
admission: Final = already_registered or Gauge(
|
||||
"litellm_admission_admitted_requests", "registered directly, bypassing admission_control_state"
|
||||
)
|
||||
yield (sentinel, admission)
|
||||
for gauge in (sentinel,) if already_registered is not None else (sentinel, admission):
|
||||
if gauge in REGISTRY._collector_to_names:
|
||||
REGISTRY.unregister(gauge)
|
||||
|
||||
|
||||
def test_isolated_metric_families_restore_the_registry_and_keep_lazy_owners_live(
|
||||
monkeypatch: pytest.MonkeyPatch, gauges_registered_by_an_earlier_test: tuple[Collector, Collector]
|
||||
):
|
||||
before: Final = _registered_collectors()
|
||||
with _isolated_litellm_metric_families(monkeypatch) as families:
|
||||
assert "litellm_unrelated_sentinel" not in families
|
||||
assert "litellm_admission_admitted_requests" in families
|
||||
assert "litellm_in_flight_requests" in families
|
||||
assert not any(gauge in REGISTRY._collector_to_names for gauge in gauges_registered_by_an_earlier_test)
|
||||
after: Final = _registered_collectors()
|
||||
assert all(after[collector] == names for collector, names in before.items())
|
||||
lazy_owned: Final = _lazy_owner_collectors()
|
||||
assert frozenset(after) - frozenset(before) <= frozenset(lazy_owned)
|
||||
assert all(collector in after for collector in lazy_owned)
|
||||
|
||||
|
||||
def _dashboard_expressions(path: Path) -> tuple[str, ...]:
|
||||
dashboard: Final = json.loads(path.read_text())
|
||||
return tuple(target["expr"] for panel in dashboard["panels"] for target in panel.get("targets", ()))
|
||||
|
||||
|
||||
def _referenced_metric_tokens(path: Path) -> frozenset[str]:
|
||||
return frozenset(
|
||||
token
|
||||
for expr in _dashboard_expressions(path)
|
||||
for token in _METRIC_TOKEN_RE.findall(_BY_CLAUSE_RE.sub("", expr))
|
||||
)
|
||||
|
||||
|
||||
def _family_of(token: str, families: frozenset[str]) -> str | None:
|
||||
candidates: Final = (token.removesuffix(suffix) for suffix in _EXPOSITION_SUFFIXES if token.endswith(suffix))
|
||||
return next((candidate for candidate in candidates if candidate in families), None)
|
||||
|
||||
|
||||
def test_all_metrics_dashboard_charts_every_emitted_metric_family(emitted_metric_families: frozenset[str]):
|
||||
referenced: Final = _referenced_metric_tokens(_ALL_METRICS_DASHBOARD)
|
||||
charted: Final = frozenset(
|
||||
family for token in referenced for family in (_family_of(token, emitted_metric_families),) if family
|
||||
)
|
||||
assert emitted_metric_families - charted == frozenset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dashboard_path", _LITELLM_DASHBOARDS, ids=lambda p: p.parent.name)
|
||||
def test_dashboards_only_reference_emitted_metrics(dashboard_path: Path, emitted_metric_families: frozenset[str]):
|
||||
dead: Final = frozenset(
|
||||
token
|
||||
for token in _referenced_metric_tokens(dashboard_path)
|
||||
if _family_of(token, emitted_metric_families) is None
|
||||
)
|
||||
assert dead == frozenset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dashboard_path", _LITELLM_DASHBOARDS, ids=lambda p: p.parent.name)
|
||||
def test_dashboards_use_templated_prometheus_datasource(dashboard_path: Path):
|
||||
dashboard: Final = json.loads(dashboard_path.read_text())
|
||||
datasource_variables: Final = tuple(
|
||||
variable["name"] for variable in dashboard["templating"]["list"] if variable["type"] == "datasource"
|
||||
)
|
||||
assert datasource_variables == ("DS_PROMETHEUS",)
|
||||
panel_datasource_uids: Final = frozenset(
|
||||
panel["datasource"]["uid"] for panel in dashboard["panels"] if panel["type"] != "row"
|
||||
)
|
||||
assert panel_datasource_uids == frozenset({"${DS_PROMETHEUS}"})
|
||||
|
||||
|
||||
def test_remaining_requests_metric_name_in_defined_metrics():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue