diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index cedbbd0f07b..f80df30c82e 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -3972,13 +3972,70 @@ "title": "litellm_provider_remaining_budget_metric", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Maximum budget set for provider - used when you set provider budget limits", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 279 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (api_provider) (litellm_provider_max_budget_metric)", + "legendFormat": "{{api_provider}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_provider_max_budget_metric", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 279 + "y": 287 }, "id": 72, "panels": [], @@ -4011,7 +4068,7 @@ "h": 8, "w": 12, "x": 0, - "y": 280 + "y": 288 }, "id": 73, "options": { @@ -4068,7 +4125,7 @@ "h": 8, "w": 12, "x": 12, - "y": 280 + "y": 288 }, "id": 74, "options": { @@ -4125,7 +4182,7 @@ "h": 8, "w": 12, "x": 0, - "y": 288 + "y": 296 }, "id": 75, "options": { @@ -4184,7 +4241,7 @@ "h": 1, "w": 24, "x": 0, - "y": 296 + "y": 304 }, "id": 76, "panels": [], @@ -4217,7 +4274,7 @@ "h": 8, "w": 12, "x": 0, - "y": 297 + "y": 305 }, "id": 77, "options": { @@ -4274,7 +4331,7 @@ "h": 8, "w": 12, "x": 12, - "y": 297 + "y": 305 }, "id": 78, "options": { @@ -4311,7 +4368,7 @@ "h": 1, "w": 24, "x": 0, - "y": 305 + "y": 313 }, "id": 79, "panels": [], @@ -4344,7 +4401,7 @@ "h": 8, "w": 12, "x": 0, - "y": 306 + "y": 314 }, "id": 80, "options": { @@ -4401,7 +4458,7 @@ "h": 8, "w": 12, "x": 12, - "y": 306 + "y": 314 }, "id": 81, "options": { @@ -4458,7 +4515,7 @@ "h": 8, "w": 12, "x": 0, - "y": 314 + "y": 322 }, "id": 82, "options": { @@ -4515,7 +4572,7 @@ "h": 8, "w": 12, "x": 12, - "y": 314 + "y": 322 }, "id": 83, "options": { @@ -4572,7 +4629,7 @@ "h": 8, "w": 12, "x": 0, - "y": 322 + "y": 330 }, "id": 84, "options": { @@ -4651,7 +4708,7 @@ "h": 8, "w": 12, "x": 12, - "y": 322 + "y": 330 }, "id": 85, "options": { @@ -4708,7 +4765,7 @@ "h": 8, "w": 12, "x": 0, - "y": 330 + "y": 338 }, "id": 86, "options": { @@ -4765,7 +4822,7 @@ "h": 8, "w": 12, "x": 12, - "y": 330 + "y": 338 }, "id": 87, "options": { @@ -4822,7 +4879,7 @@ "h": 8, "w": 12, "x": 0, - "y": 338 + "y": 346 }, "id": 88, "options": { @@ -4859,7 +4916,7 @@ "h": 1, "w": 24, "x": 0, - "y": 346 + "y": 354 }, "id": 89, "panels": [], @@ -4892,7 +4949,7 @@ "h": 8, "w": 12, "x": 0, - "y": 347 + "y": 355 }, "id": 90, "options": { @@ -4949,7 +5006,7 @@ "h": 8, "w": 12, "x": 12, - "y": 347 + "y": 355 }, "id": 91, "options": { @@ -5006,7 +5063,7 @@ "h": 8, "w": 12, "x": 0, - "y": 355 + "y": 363 }, "id": 92, "options": { @@ -5063,7 +5120,7 @@ "h": 8, "w": 12, "x": 12, - "y": 355 + "y": 363 }, "id": 93, "options": { @@ -5120,7 +5177,7 @@ "h": 8, "w": 12, "x": 0, - "y": 363 + "y": 371 }, "id": 94, "options": { @@ -5157,7 +5214,7 @@ "h": 1, "w": 24, "x": 0, - "y": 371 + "y": 379 }, "id": 95, "panels": [], @@ -5190,7 +5247,7 @@ "h": 8, "w": 12, "x": 0, - "y": 372 + "y": 380 }, "id": 96, "options": { @@ -5247,7 +5304,7 @@ "h": 8, "w": 12, "x": 12, - "y": 372 + "y": 380 }, "id": 97, "options": { @@ -5304,7 +5361,7 @@ "h": 8, "w": 12, "x": 0, - "y": 380 + "y": 388 }, "id": 98, "options": { @@ -5341,7 +5398,7 @@ "h": 1, "w": 24, "x": 0, - "y": 388 + "y": 396 }, "id": 99, "panels": [], @@ -5374,7 +5431,7 @@ "h": 8, "w": 12, "x": 0, - "y": 389 + "y": 397 }, "id": 100, "options": { @@ -5431,7 +5488,7 @@ "h": 8, "w": 12, "x": 12, - "y": 389 + "y": 397 }, "id": 101, "options": { @@ -5488,7 +5545,7 @@ "h": 8, "w": 12, "x": 0, - "y": 397 + "y": 405 }, "id": 102, "options": { @@ -5545,7 +5602,7 @@ "h": 8, "w": 12, "x": 12, - "y": 397 + "y": 405 }, "id": 103, "options": { @@ -5624,7 +5681,7 @@ "h": 8, "w": 12, "x": 0, - "y": 405 + "y": 413 }, "id": 104, "options": { @@ -5661,7 +5718,7 @@ "h": 1, "w": 24, "x": 0, - "y": 413 + "y": 421 }, "id": 105, "panels": [], @@ -5694,7 +5751,7 @@ "h": 8, "w": 12, "x": 0, - "y": 414 + "y": 422 }, "id": 106, "options": { @@ -5883,7 +5940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 414 + "y": 422 }, "id": 107, "options": { @@ -6083,7 +6140,7 @@ "h": 8, "w": 12, "x": 0, - "y": 422 + "y": 430 }, "id": 108, "options": { @@ -6283,7 +6340,7 @@ "h": 8, "w": 12, "x": 12, - "y": 422 + "y": 430 }, "id": 109, "options": { @@ -6406,7 +6463,7 @@ "h": 8, "w": 12, "x": 0, - "y": 430 + "y": 438 }, "id": 110, "options": { @@ -6443,7 +6500,7 @@ "h": 1, "w": 24, "x": 0, - "y": 438 + "y": 446 }, "id": 112, "panels": [], @@ -6476,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 439 + "y": 447 }, "id": 113, "options": { @@ -6533,7 +6590,7 @@ "h": 8, "w": 12, "x": 12, - "y": 439 + "y": 447 }, "id": 114, "options": { @@ -6570,7 +6627,7 @@ "h": 1, "w": 24, "x": 0, - "y": 447 + "y": 455 }, "id": 115, "panels": [], @@ -6603,7 +6660,7 @@ "h": 8, "w": 24, "x": 0, - "y": 448 + "y": 456 }, "id": 116, "options": { diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fc1a601f9da..8fd8d0bb5e1 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -904,6 +904,11 @@ class PrometheusLogger(CustomLogger): "Remaining budget for provider - used when you set provider budget limits", labelnames=["api_provider"], ) + self.litellm_provider_max_budget_metric = self._gauge_factory( # pyright: ignore[reportUnknownMemberType] # the metric factory returns a partly unknown type, as for every gauge here + "litellm_provider_max_budget_metric", + "Maximum budget set for provider - used when you set provider budget limits", + labelnames=["api_provider"], + ) # Metric for deployment state self.litellm_deployment_state = self._gauge_factory( @@ -4035,7 +4040,7 @@ class PrometheusLogger(CustomLogger): def track_provider_remaining_budget(self, provider: str, spend: float, budget_limit: float): """ - Track provider remaining budget in Prometheus + Track provider remaining and max budget in Prometheus """ self.litellm_provider_remaining_budget_metric.labels(provider).set( self._safe_get_remaining_budget( @@ -4043,6 +4048,7 @@ class PrometheusLogger(CustomLogger): spend=spend, ) ) + self.litellm_provider_max_budget_metric.labels(provider).set(budget_limit) # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue] # same factory type as the remaining gauge above def _safe_get_remaining_budget(self, max_budget: float | None, spend: float | None) -> float: if max_budget is None: diff --git a/tests/unit/enterprise/integrations/test_prometheus.py b/tests/unit/enterprise/integrations/test_prometheus.py index fc4a729ec98..d935ecb9d11 100644 --- a/tests/unit/enterprise/integrations/test_prometheus.py +++ b/tests/unit/enterprise/integrations/test_prometheus.py @@ -734,6 +734,20 @@ def test_exclude_labels_does_not_touch_unrelated_metrics(reset_prometheus_exclud # ============================================================================== +def test_provider_budget_gauges_report_the_limit_next_to_what_remains(): + """The max-budget gauge carries the configured provider limit, so limit minus remaining is the spend.""" + from prometheus_client import generate_latest + + clear_prometheus_registry() + logger = PrometheusLogger() + + logger.track_provider_remaining_budget(provider="openai", spend=30.0, budget_limit=100.0) + + scrape = generate_latest(REGISTRY).decode() + assert 'litellm_provider_max_budget_metric{api_provider="openai"} 100.0' in scrape + assert 'litellm_provider_remaining_budget_metric{api_provider="openai"} 70.0' in scrape + + # ============================================================================== # SEMANTIC VALIDATION TESTS - Detect logical errors in metric increments # ==============================================================================