From f2036160f11545455f9bb77255dff64f6d72cad1 Mon Sep 17 00:00:00 2001 From: Aryan Deshmukh Date: Wed, 4 Nov 2026 16:57:27 +0530 Subject: [PATCH 1/2] feat(prometheus): count the tool calls the model makes, by tool name Adds litellm_model_tool_calls_total with the output token labels plus tool_name. Tool names come from response_tool_call_names, so chat completions (every choice), Responses API output and /v1/messages tool_use blocks are counted the same way the tool usage view counts them. The counter is charted on the all-metrics Grafana dashboard. Refs #49036 Signed-off-by: Aryan Deshmukh --- .../grafana_dashboard.json | 229 +++++++++++------- litellm/integrations/prometheus.py | 29 +++ litellm/types/integrations/prometheus.py | 5 + ...test_prometheus_model_tool_call_metrics.py | 188 ++++++++++++++ 4 files changed, 365 insertions(+), 86 deletions(-) create mode 100644 tests/unit/integrations/test_prometheus_model_tool_call_metrics.py diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index cedbbd0f07b..2c4a2fe38d5 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -1754,13 +1754,70 @@ "title": "litellm_spend_capture_rate", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tool calls the model asked for in its responses, by tool name; counted from the response, so a call the client never runs is included", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 115 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_model_tool_calls_total[$__rate_interval])) by (tool_name)", + "legendFormat": "{{tool_name}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_model_tool_calls rate", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 115 + "y": 123 }, "id": 30, "panels": [], @@ -1793,7 +1850,7 @@ "h": 8, "w": 12, "x": 0, - "y": 116 + "y": 124 }, "id": 31, "options": { @@ -1850,7 +1907,7 @@ "h": 8, "w": 12, "x": 12, - "y": 116 + "y": 124 }, "id": 32, "options": { @@ -1907,7 +1964,7 @@ "h": 8, "w": 12, "x": 0, - "y": 124 + "y": 132 }, "id": 33, "options": { @@ -1964,7 +2021,7 @@ "h": 8, "w": 12, "x": 12, - "y": 124 + "y": 132 }, "id": 34, "options": { @@ -2021,7 +2078,7 @@ "h": 8, "w": 12, "x": 0, - "y": 132 + "y": 140 }, "id": 35, "options": { @@ -2058,7 +2115,7 @@ "h": 1, "w": 24, "x": 0, - "y": 140 + "y": 148 }, "id": 36, "panels": [], @@ -2091,7 +2148,7 @@ "h": 8, "w": 12, "x": 0, - "y": 141 + "y": 149 }, "id": 37, "options": { @@ -2148,7 +2205,7 @@ "h": 8, "w": 12, "x": 12, - "y": 141 + "y": 149 }, "id": 38, "options": { @@ -2205,7 +2262,7 @@ "h": 8, "w": 12, "x": 0, - "y": 149 + "y": 157 }, "id": 39, "options": { @@ -2262,7 +2319,7 @@ "h": 8, "w": 12, "x": 12, - "y": 149 + "y": 157 }, "id": 40, "options": { @@ -2319,7 +2376,7 @@ "h": 8, "w": 12, "x": 0, - "y": 157 + "y": 165 }, "id": 41, "options": { @@ -2376,7 +2433,7 @@ "h": 8, "w": 12, "x": 12, - "y": 157 + "y": 165 }, "id": 42, "options": { @@ -2433,7 +2490,7 @@ "h": 8, "w": 12, "x": 0, - "y": 165 + "y": 173 }, "id": 43, "options": { @@ -2490,7 +2547,7 @@ "h": 8, "w": 12, "x": 12, - "y": 165 + "y": 173 }, "id": 44, "options": { @@ -2547,7 +2604,7 @@ "h": 8, "w": 12, "x": 0, - "y": 173 + "y": 181 }, "id": 45, "options": { @@ -2604,7 +2661,7 @@ "h": 8, "w": 12, "x": 12, - "y": 173 + "y": 181 }, "id": 46, "options": { @@ -2661,7 +2718,7 @@ "h": 8, "w": 12, "x": 0, - "y": 181 + "y": 189 }, "id": 47, "options": { @@ -2698,7 +2755,7 @@ "h": 1, "w": 24, "x": 0, - "y": 189 + "y": 197 }, "id": 48, "panels": [], @@ -2731,7 +2788,7 @@ "h": 8, "w": 12, "x": 0, - "y": 190 + "y": 198 }, "id": 49, "options": { @@ -2788,7 +2845,7 @@ "h": 8, "w": 12, "x": 12, - "y": 190 + "y": 198 }, "id": 50, "options": { @@ -2845,7 +2902,7 @@ "h": 8, "w": 12, "x": 0, - "y": 198 + "y": 206 }, "id": 51, "options": { @@ -2902,7 +2959,7 @@ "h": 8, "w": 12, "x": 12, - "y": 198 + "y": 206 }, "id": 52, "options": { @@ -2959,7 +3016,7 @@ "h": 8, "w": 12, "x": 0, - "y": 206 + "y": 214 }, "id": 53, "options": { @@ -3016,7 +3073,7 @@ "h": 8, "w": 12, "x": 12, - "y": 206 + "y": 214 }, "id": 54, "options": { @@ -3053,7 +3110,7 @@ "h": 1, "w": 24, "x": 0, - "y": 214 + "y": 222 }, "id": 55, "panels": [], @@ -3086,7 +3143,7 @@ "h": 8, "w": 12, "x": 0, - "y": 215 + "y": 223 }, "id": 56, "options": { @@ -3143,7 +3200,7 @@ "h": 8, "w": 12, "x": 12, - "y": 215 + "y": 223 }, "id": 57, "options": { @@ -3200,7 +3257,7 @@ "h": 8, "w": 12, "x": 0, - "y": 223 + "y": 231 }, "id": 58, "options": { @@ -3257,7 +3314,7 @@ "h": 8, "w": 12, "x": 12, - "y": 223 + "y": 231 }, "id": 59, "options": { @@ -3314,7 +3371,7 @@ "h": 8, "w": 12, "x": 0, - "y": 231 + "y": 239 }, "id": 60, "options": { @@ -3371,7 +3428,7 @@ "h": 8, "w": 12, "x": 12, - "y": 231 + "y": 239 }, "id": 61, "options": { @@ -3428,7 +3485,7 @@ "h": 8, "w": 12, "x": 0, - "y": 239 + "y": 247 }, "id": 62, "options": { @@ -3485,7 +3542,7 @@ "h": 8, "w": 12, "x": 12, - "y": 239 + "y": 247 }, "id": 63, "options": { @@ -3542,7 +3599,7 @@ "h": 8, "w": 12, "x": 0, - "y": 247 + "y": 255 }, "id": 64, "options": { @@ -3599,7 +3656,7 @@ "h": 8, "w": 12, "x": 12, - "y": 247 + "y": 255 }, "id": 65, "options": { @@ -3656,7 +3713,7 @@ "h": 8, "w": 12, "x": 0, - "y": 255 + "y": 263 }, "id": 66, "options": { @@ -3713,7 +3770,7 @@ "h": 8, "w": 12, "x": 12, - "y": 255 + "y": 263 }, "id": 67, "options": { @@ -3770,7 +3827,7 @@ "h": 8, "w": 12, "x": 0, - "y": 263 + "y": 271 }, "id": 68, "options": { @@ -3827,7 +3884,7 @@ "h": 8, "w": 12, "x": 12, - "y": 263 + "y": 271 }, "id": 69, "options": { @@ -3884,7 +3941,7 @@ "h": 8, "w": 12, "x": 0, - "y": 271 + "y": 279 }, "id": 70, "options": { @@ -3941,7 +3998,7 @@ "h": 8, "w": 12, "x": 12, - "y": 271 + "y": 279 }, "id": 71, "options": { @@ -3978,7 +4035,7 @@ "h": 1, "w": 24, "x": 0, - "y": 279 + "y": 287 }, "id": 72, "panels": [], @@ -4011,7 +4068,7 @@ "h": 8, "w": 12, "x": 0, - "y": 280 + "y": 288 }, "id": 73, "options": { @@ -4068,7 +4125,7 @@ "h": 8, "w": 12, "x": 12, - "y": 280 + "y": 288 }, "id": 74, "options": { @@ -4125,7 +4182,7 @@ "h": 8, "w": 12, "x": 0, - "y": 288 + "y": 296 }, "id": 75, "options": { @@ -4184,7 +4241,7 @@ "h": 1, "w": 24, "x": 0, - "y": 296 + "y": 304 }, "id": 76, "panels": [], @@ -4217,7 +4274,7 @@ "h": 8, "w": 12, "x": 0, - "y": 297 + "y": 305 }, "id": 77, "options": { @@ -4274,7 +4331,7 @@ "h": 8, "w": 12, "x": 12, - "y": 297 + "y": 305 }, "id": 78, "options": { @@ -4311,7 +4368,7 @@ "h": 1, "w": 24, "x": 0, - "y": 305 + "y": 313 }, "id": 79, "panels": [], @@ -4344,7 +4401,7 @@ "h": 8, "w": 12, "x": 0, - "y": 306 + "y": 314 }, "id": 80, "options": { @@ -4401,7 +4458,7 @@ "h": 8, "w": 12, "x": 12, - "y": 306 + "y": 314 }, "id": 81, "options": { @@ -4458,7 +4515,7 @@ "h": 8, "w": 12, "x": 0, - "y": 314 + "y": 322 }, "id": 82, "options": { @@ -4515,7 +4572,7 @@ "h": 8, "w": 12, "x": 12, - "y": 314 + "y": 322 }, "id": 83, "options": { @@ -4572,7 +4629,7 @@ "h": 8, "w": 12, "x": 0, - "y": 322 + "y": 330 }, "id": 84, "options": { @@ -4651,7 +4708,7 @@ "h": 8, "w": 12, "x": 12, - "y": 322 + "y": 330 }, "id": 85, "options": { @@ -4708,7 +4765,7 @@ "h": 8, "w": 12, "x": 0, - "y": 330 + "y": 338 }, "id": 86, "options": { @@ -4765,7 +4822,7 @@ "h": 8, "w": 12, "x": 12, - "y": 330 + "y": 338 }, "id": 87, "options": { @@ -4822,7 +4879,7 @@ "h": 8, "w": 12, "x": 0, - "y": 338 + "y": 346 }, "id": 88, "options": { @@ -4859,7 +4916,7 @@ "h": 1, "w": 24, "x": 0, - "y": 346 + "y": 354 }, "id": 89, "panels": [], @@ -4892,7 +4949,7 @@ "h": 8, "w": 12, "x": 0, - "y": 347 + "y": 355 }, "id": 90, "options": { @@ -4949,7 +5006,7 @@ "h": 8, "w": 12, "x": 12, - "y": 347 + "y": 355 }, "id": 91, "options": { @@ -5006,7 +5063,7 @@ "h": 8, "w": 12, "x": 0, - "y": 355 + "y": 363 }, "id": 92, "options": { @@ -5063,7 +5120,7 @@ "h": 8, "w": 12, "x": 12, - "y": 355 + "y": 363 }, "id": 93, "options": { @@ -5120,7 +5177,7 @@ "h": 8, "w": 12, "x": 0, - "y": 363 + "y": 371 }, "id": 94, "options": { @@ -5157,7 +5214,7 @@ "h": 1, "w": 24, "x": 0, - "y": 371 + "y": 379 }, "id": 95, "panels": [], @@ -5190,7 +5247,7 @@ "h": 8, "w": 12, "x": 0, - "y": 372 + "y": 380 }, "id": 96, "options": { @@ -5247,7 +5304,7 @@ "h": 8, "w": 12, "x": 12, - "y": 372 + "y": 380 }, "id": 97, "options": { @@ -5304,7 +5361,7 @@ "h": 8, "w": 12, "x": 0, - "y": 380 + "y": 388 }, "id": 98, "options": { @@ -5341,7 +5398,7 @@ "h": 1, "w": 24, "x": 0, - "y": 388 + "y": 396 }, "id": 99, "panels": [], @@ -5374,7 +5431,7 @@ "h": 8, "w": 12, "x": 0, - "y": 389 + "y": 397 }, "id": 100, "options": { @@ -5431,7 +5488,7 @@ "h": 8, "w": 12, "x": 12, - "y": 389 + "y": 397 }, "id": 101, "options": { @@ -5488,7 +5545,7 @@ "h": 8, "w": 12, "x": 0, - "y": 397 + "y": 405 }, "id": 102, "options": { @@ -5545,7 +5602,7 @@ "h": 8, "w": 12, "x": 12, - "y": 397 + "y": 405 }, "id": 103, "options": { @@ -5624,7 +5681,7 @@ "h": 8, "w": 12, "x": 0, - "y": 405 + "y": 413 }, "id": 104, "options": { @@ -5661,7 +5718,7 @@ "h": 1, "w": 24, "x": 0, - "y": 413 + "y": 421 }, "id": 105, "panels": [], @@ -5694,7 +5751,7 @@ "h": 8, "w": 12, "x": 0, - "y": 414 + "y": 422 }, "id": 106, "options": { @@ -5883,7 +5940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 414 + "y": 422 }, "id": 107, "options": { @@ -6083,7 +6140,7 @@ "h": 8, "w": 12, "x": 0, - "y": 422 + "y": 430 }, "id": 108, "options": { @@ -6283,7 +6340,7 @@ "h": 8, "w": 12, "x": 12, - "y": 422 + "y": 430 }, "id": 109, "options": { @@ -6406,7 +6463,7 @@ "h": 8, "w": 12, "x": 0, - "y": 430 + "y": 438 }, "id": 110, "options": { @@ -6443,7 +6500,7 @@ "h": 1, "w": 24, "x": 0, - "y": 438 + "y": 446 }, "id": 112, "panels": [], @@ -6476,7 +6533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 439 + "y": 447 }, "id": 113, "options": { @@ -6533,7 +6590,7 @@ "h": 8, "w": 12, "x": 12, - "y": 439 + "y": 447 }, "id": 114, "options": { @@ -6570,7 +6627,7 @@ "h": 1, "w": 24, "x": 0, - "y": 447 + "y": 455 }, "id": 115, "panels": [], @@ -6603,7 +6660,7 @@ "h": 8, "w": 24, "x": 0, - "y": 448 + "y": 456 }, "id": 116, "options": { diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fc1a601f9da..1f6e0a3bd89 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -7,6 +7,7 @@ import asyncio import math import os import sys +from collections import Counter from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import replace from datetime import datetime, timedelta @@ -671,6 +672,12 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_images_generated_metric"), ) + self.litellm_model_tool_calls_total = self._counter_factory( + "litellm_model_tool_calls_total", + "Tool calls the model asked for in its response, by tool name", + labelnames=self.get_labels_for_metric("litellm_model_tool_calls_total"), + ) + # Remaining Budget for Team self.litellm_remaining_team_budget_metric = self._gauge_factory( "litellm_remaining_team_budget_metric", @@ -1835,6 +1842,11 @@ class PrometheusLogger(CustomLogger): label_context=label_context, ) + try: + self._increment_model_tool_call_metrics(response_obj=response_obj, enum_values=enum_values) + except Exception as e: # noqa: BLE001 # counting tool calls must never stop the rest of success logging + verbose_logger.exception("Prometheus: failed to count the model's tool calls: %s", e) + # MCP tool call metrics self._increment_mcp_tool_call_metrics( standard_logging_payload=standard_logging_payload, @@ -2154,6 +2166,23 @@ class PrometheusLogger(CustomLogger): return value return None + def _increment_model_tool_call_metrics(self, response_obj: object, enum_values: UserAPIKeyLabelValues) -> None: + from litellm.proxy.db.spend_log_tool_index import response_tool_call_names + + tool_names: Final = response_tool_call_names(response_obj) + for tool_name, amount in Counter(tool_names).items(): + self._inc_model_tool_call_counter(replace(enum_values, tool_name=tool_name), amount=amount) + + def _inc_model_tool_call_counter(self, tool_enum_values: UserAPIKeyLabelValues, amount: int) -> None: + PrometheusLogger._inc_labeled_counter( + self, + self.litellm_model_tool_calls_total, + "litellm_model_tool_calls_total", + tool_enum_values, + label_context=PrometheusLabelFactoryContext(tool_enum_values), + amount=amount, + ) + def _increment_mcp_tool_call_metrics( self, standard_logging_payload: StandardLoggingPayload, diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index ca084bb358c..91ee155009e 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -229,6 +229,7 @@ class UserAPIKeyLabelNames(Enum): ORG_ALIAS = "org_alias" MCP_TOOL_NAME = "mcp_tool_name" MCP_SERVER_NAME = "mcp_server_name" + MODEL_TOOL_NAME = "tool_name" SERVICE_TIER = "service_tier" INPUT_SEQUENCE_LENGTH = "input_sequence_length" @@ -256,6 +257,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_output_audio_tokens_metric", "litellm_video_duration_seconds_metric", "litellm_images_generated_metric", + "litellm_model_tool_calls_total", "litellm_deployment_successful_fallbacks", "litellm_deployment_failed_fallbacks", "litellm_remaining_team_budget_metric", @@ -676,6 +678,7 @@ class PrometheusMetricLabels: litellm_video_duration_seconds_metric = litellm_output_tokens_metric litellm_images_generated_metric = litellm_output_tokens_metric + litellm_model_tool_calls_total = [*litellm_output_tokens_metric, UserAPIKeyLabelNames.MODEL_TOOL_NAME.value] litellm_deployment_state = [ UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, @@ -924,6 +927,7 @@ class PrometheusMetricLabels: "litellm_output_tokens_metric", "litellm_video_duration_seconds_metric", "litellm_images_generated_metric", + "litellm_model_tool_calls_total", } ) _input_sequence_length_metrics: ClassVar[frozenset[str]] = frozenset( @@ -1105,6 +1109,7 @@ class UserAPIKeyLabelValues: org_alias: str | None = None mcp_tool_name: str | None = None mcp_server_name: str | None = None + tool_name: str | None = None service_tier: str | None = None input_sequence_length: str | None = None diff --git a/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py new file mode 100644 index 00000000000..f015e2c9d49 --- /dev/null +++ b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py @@ -0,0 +1,188 @@ +import datetime +from collections.abc import Mapping +from typing import Final, cast + +import pytest +from prometheus_client import REGISTRY +from prometheus_client.samples import Sample + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.utils import StandardLoggingPayload + +NOW: Final = datetime.datetime(2026, 11, 3, 12, 0, 0) + + +def _clear_prometheus_registry() -> None: + for collector in tuple(REGISTRY._collector_to_names): # pyright: ignore[reportPrivateUsage] # test registry reset + REGISTRY.unregister(collector) + + +@pytest.fixture(autouse=True) +def isolated_registry(): + _clear_prometheus_registry() + yield + _clear_prometheus_registry() + + +def _success_kwargs() -> dict[str, object]: + payload: Final = cast( + StandardLoggingPayload, + { + "id": "t", + "call_type": "acompletion", + "response_cost": 0.001, + "status": "success", + "total_tokens": 30, + "prompt_tokens": 10, + "completion_tokens": 20, + "startTime": NOW - datetime.timedelta(seconds=3), + "endTime": NOW, + "completionStartTime": NOW - datetime.timedelta(seconds=1), + "model": "gpt-4.1", + "model_id": "model-123", + "model_group": "gpt-4.1-group", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "request_tags": [], + "stream": False, + "metadata": { + "user_api_key_hash": "key-hash", + "user_api_key_alias": "key-alias", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "team-alias", + "user_api_key_user_id": "user-1", + "user_api_key_user_email": None, + "user_api_key_org_id": None, + "user_api_key_org_alias": None, + "requester_metadata": None, + "user_api_key_end_user_id": None, + "usage_object": None, + }, + "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None}, + }, + ) + return { + "model": "gpt-4.1", + "litellm_params": {"metadata": {}}, + "standard_logging_object": payload, + "stream": False, + "start_time": NOW - datetime.timedelta(seconds=3), + "api_call_start_time": NOW - datetime.timedelta(seconds=2), + "completion_start_time": NOW - datetime.timedelta(seconds=1), + "end_time": NOW, + } + + +def _samples(name: str) -> tuple[Sample, ...]: + return tuple(sample for metric in REGISTRY.collect() for sample in metric.samples if sample.name == f"{name}_total") + + +def _tool_call_counts() -> Mapping[str, float]: + return {sample.labels["tool_name"]: sample.value for sample in _samples("litellm_model_tool_calls")} + + +def _chat_tool_call(call_id: str, name: str) -> Mapping[str, object]: + return {"id": call_id, "type": "function", "function": {"name": name, "arguments": "{}"}} + + +def _chat_choice(index: int, *tool_calls: Mapping[str, object]) -> Mapping[str, object]: + return { + "index": index, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": list(tool_calls)}, + } + + +CHAT_RESPONSE: Final = litellm.ModelResponse( + choices=[ + _chat_choice(0, _chat_tool_call("a", "Read"), _chat_tool_call("b", "Read"), _chat_tool_call("c", "Bash")), + _chat_choice(1, _chat_tool_call("d", "Bash")), + ] +) +RESPONSES_API_RESPONSE: Final = { + "id": "resp_1", + "output": [ + {"type": "message", "content": [{"type": "output_text", "text": "checking"}]}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": "{}"}, + ], +} +MESSAGES_TOOL_USE_RESPONSE: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me look"}, + {"type": "tool_use", "id": "toolu_1", "name": "Grep", "input": {"pattern": "x"}}, + {"type": "tool_use", "id": "toolu_2", "name": "Grep", "input": {"pattern": "y"}}, + ], +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("response", "expected"), + ( + pytest.param(CHAT_RESPONSE, {"Read": 2, "Bash": 2}, id="chat-completions-every-choice"), + pytest.param(RESPONSES_API_RESPONSE, {"get_weather": 1}, id="responses-api"), + pytest.param(MESSAGES_TOOL_USE_RESPONSE, {"Grep": 2}, id="messages-tool-use"), + ), +) +async def test_counts_each_tool_the_model_called(response: object, expected: Mapping[str, float]): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == expected + + +@pytest.mark.asyncio +async def test_counts_add_up_across_requests(): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + + assert _tool_call_counts() == {"Grep": 4} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + ( + pytest.param(litellm.ModelResponse(), id="chat-without-tool-calls"), + pytest.param({"id": "msg_2", "content": [{"type": "text", "text": "hi"}]}, id="messages-text-only"), + pytest.param(None, id="no-response"), + ), +) +async def test_emits_nothing_when_the_model_called_no_tool(response: object): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == {} + + +@pytest.mark.asyncio +async def test_a_failure_while_counting_tool_calls_does_not_stop_the_other_metrics(monkeypatch: pytest.MonkeyPatch): + def explode(_response: object) -> list[str]: + raise RuntimeError("unreadable response") + + monkeypatch.setattr("litellm.proxy.db.spend_log_tool_index.response_tool_call_names", explode) + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + + assert _tool_call_counts() == {} + assert _samples("litellm_output_tokens_metric") + + +@pytest.mark.asyncio +async def test_series_carry_the_output_token_labels_plus_tool_name(): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + + token_labels: Final = {**_samples("litellm_output_tokens_metric")[0].labels} + (tool_sample,) = _samples("litellm_model_tool_calls") + assert tool_sample.labels == {**token_labels, "tool_name": "Grep"} From 7780aed7f706693179c6d03d370f4446be17180b Mon Sep 17 00:00:00 2001 From: Aryan Deshmukh Date: Thu, 5 Nov 2026 21:01:15 +0530 Subject: [PATCH 2/2] fix(prometheus): count tool calls by name only, cap tool_name values Counting tool calls went through get_tool_calls_from_response, which parses each call's arguments and logs up to the full argument text when they are malformed. get_tool_call_names_from_response reads only the names, with the same count and order (tool_call_argument_count decides concatenated and repaired arguments the same way, without logging), and response_tool_call_names now uses it, so the spend-log tool index stops logging argument text too. The tool_name label takes names the model picks, so prometheus_model_tool_name_max_values (default 1000, None disables) keeps that many distinct names per process; later names count under "other". --- litellm/__init__.py | 4 + litellm/integrations/prometheus.py | 19 +- .../prompt_templates/common_utils.py | 3 + .../prompt_templates/factory.py | 60 ++++++ .../prompt_templates/tool_call_json_repair.py | 18 ++ litellm/proxy/db/spend_log_tool_index.py | 9 +- litellm/types/integrations/prometheus.py | 2 +- ...test_prometheus_model_tool_call_metrics.py | 187 ++++++++++++++++++ 8 files changed, 294 insertions(+), 8 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..7aa41013ebe 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -488,6 +488,10 @@ prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0 prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0 prometheus_metrics_max_series_per_metric: Optional[int] = None +# Distinct ``tool_name`` values ``litellm_model_tool_calls_total`` gives their own series, per worker +# process; later names count under ``other`` until restart (a real tool named ``other`` shares that series). +# It bounds names, not series: each name still splits by the metric's other labels. None disables the cap. +prometheus_model_tool_name_max_values: Optional[int] = 1000 prometheus_metrics_ttl_seconds: Optional[float] = None prometheus_metrics_cleanup_interval_seconds: Optional[float] = 60.0 disable_add_prefix_to_prompt: bool = False # used by anthropic, to disable adding prefix to prompt diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 1f6e0a3bd89..ccb4a93a0bc 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -118,6 +118,7 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel) _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0 UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other" +OVERFLOW_MODEL_TOOL_NAME_LABEL: Final = "other" _NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset( ( @@ -541,6 +542,7 @@ class PrometheusLogger(CustomLogger): _custom_buckets: Final = litellm.prometheus_latency_buckets self.latency_buckets = tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker() + self._admitted_model_tool_names: set[str] = set() # mutable-ok: names admitted under the tool_name cap _multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR") self._series_cap_tracker = ( BoundedPrometheusSeriesTracker() @@ -2169,10 +2171,25 @@ class PrometheusLogger(CustomLogger): def _increment_model_tool_call_metrics(self, response_obj: object, enum_values: UserAPIKeyLabelValues) -> None: from litellm.proxy.db.spend_log_tool_index import response_tool_call_names - tool_names: Final = response_tool_call_names(response_obj) + tool_names: Final = tuple( + self._bounded_model_tool_name(name) for name in response_tool_call_names(response_obj) + ) for tool_name, amount in Counter(tool_names).items(): self._inc_model_tool_call_counter(replace(enum_values, tool_name=tool_name), amount=amount) + def _bounded_model_tool_name(self, tool_name: str) -> str: + """The model picks tool names, from tools the caller declared or not, so + only the first ``prometheus_model_tool_name_max_values`` distinct names + get their own series; later names count under ``other``. Concurrent + callbacks may admit a few names past the cap, which still bounds it.""" + max_values: Final = litellm.prometheus_model_tool_name_max_values + if max_values is None or tool_name in self._admitted_model_tool_names: + return tool_name + if len(self._admitted_model_tool_names) >= max_values: + return OVERFLOW_MODEL_TOOL_NAME_LABEL + self._admitted_model_tool_names.add(tool_name) + return tool_name + def _inc_model_tool_call_counter(self, tool_enum_values: UserAPIKeyLabelValues, amount: int) -> None: PrometheusLogger._inc_labeled_counter( self, diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8f874edfddf..e0342ee81ad 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -179,6 +179,9 @@ from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( split_concatenated_json_objects as split_concatenated_json_objects, # noqa: PLC0414 # public compatibility re-export ) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + tool_call_argument_count as tool_call_argument_count, # noqa: PLC0414 # public compatibility re-export +) from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import ( _image_part_url, # pyright: ignore[reportPrivateUsage] # Shared implementation for as_openai_image_part ) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c6288227546..01090f7f68b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -76,6 +76,7 @@ from .common_utils import ( native_redacted_thinking_data, parse_tool_call_arguments, salvage_concatenated_tool_arguments, + tool_call_argument_count, ) from .image_handling import convert_url_to_base64, is_pdf_media_type @@ -6482,6 +6483,65 @@ def get_tool_calls_from_response(response: object, include_all_choices: bool = F return [] +def _tool_call_count(raw_arguments: object) -> int: + # A missing ``arguments`` field parses as ``{}`` on the parse path: one call. + if not isinstance(raw_arguments, str): + return 1 + return tool_call_argument_count("{}" if raw_arguments == REDACTED_BY_LITELLM else raw_arguments) + + +def _repeated_name(call: object) -> tuple[str | None, ...]: + return (_optional_tool_name(message_field(call, "name")),) * _tool_call_count(message_field(call, "arguments")) + + +def _choice_tool_call_names(choice: object) -> tuple[str | None, ...]: + functions: Final = tuple(message_field(tool_call, "function") for tool_call in _choice_tool_calls(choice)) + return tuple( + itertools.chain.from_iterable(_repeated_name(function) for function in functions if function is not None) + ) + + +def _chat_tool_call_names(response: object, include_all_choices: bool) -> tuple[str | None, ...]: + return tuple( + itertools.chain.from_iterable( + _choice_tool_call_names(choice) for choice in _selected_choices(response, include_all_choices) + ) + ) + + +def _responses_api_tool_call_names(response: object) -> tuple[str | None, ...]: + return tuple(itertools.chain.from_iterable(_repeated_name(item) for item in _response_function_calls(response))) + + +def _messages_api_tool_call_names(response: object) -> tuple[str | None, ...]: + content: Final = message_field(response, "content") + blocks: Final = parts_of(content) + if not isinstance(content, list): + return () + return tuple( + _optional_tool_name(message_field(block, "name")) + for block in blocks + if message_field(block, "type") == "tool_use" + ) + + +def get_tool_call_names_from_response(response: object, include_all_choices: bool = False) -> tuple[str | None, ...]: + """ + The ``name`` of every call ``get_tool_calls_from_response`` returns, in the + same order and with the same count, without parsing argument values into + objects or logging them. Usage counters (Prometheus, the spend-log tool + index) only need the names, and the parse path logs up to the full argument + text of a malformed call, which may hold prompt data. + """ + chat_names: Final = _chat_tool_call_names(response, include_all_choices=include_all_choices) + if chat_names: + return chat_names + responses_names: Final = _responses_api_tool_call_names(response) + if responses_names: + return responses_names + return _messages_api_tool_call_names(response) + + def has_tool_with_name(tools: object, tool_name: str) -> bool: """ Check whether a tools list (as sent to an LLM) includes a tool with the diff --git a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py index 317ffa7b3b1..1d285bdefb5 100644 --- a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py +++ b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py @@ -250,3 +250,21 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], .. if exceeded: return () return kept + + +def tool_call_argument_count(arguments: str) -> int: + """How many tool calls one raw argument string stands for, decided the way + ``get_tool_calls_from_response`` decides it: valid or repairable JSON is one + call, a concatenation of JSON objects is one call per salvaged object, and + anything else is one call with empty arguments. Unlike that path it never + logs any part of the argument text, so usage counters can call it on every + response.""" + if not arguments.strip(): + return 1 + try: + json.loads(arguments) + except json.JSONDecodeError: + if _attempt_json_repair(arguments) is not None: + return 1 + return len(salvage_concatenated_tool_arguments(arguments)) or 1 + return 1 diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..cf9d10b0fef 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -47,20 +47,17 @@ class ToolUsageTransaction: def response_tool_call_names(completion_response: object) -> tuple[str, ...]: """Tool names invoked in a completion response, in call order, for any response - surface get_tool_calls_from_response understands (chat completions, Responses + surface get_tool_call_names_from_response understands (chat completions, Responses API output items, Anthropic Messages tool_use blocks), plus the MCP tools an auto-executed Responses API call ran. Reads every choice of an ``n>1`` chat response: each choice cost money and its tool calls ran.""" if completion_response is None or isinstance(completion_response, Exception): return () from litellm.litellm_core_utils.prompt_templates.factory import ( - get_tool_calls_from_response, + get_tool_call_names_from_response, ) - called_names: Final = ( - tool_call.get("name") - for tool_call in get_tool_calls_from_response(completion_response, include_all_choices=True) - ) + called_names: Final = get_tool_call_names_from_response(completion_response, include_all_choices=True) return tuple( stripped for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 91ee155009e..a156254837d 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -678,7 +678,7 @@ class PrometheusMetricLabels: litellm_video_duration_seconds_metric = litellm_output_tokens_metric litellm_images_generated_metric = litellm_output_tokens_metric - litellm_model_tool_calls_total = [*litellm_output_tokens_metric, UserAPIKeyLabelNames.MODEL_TOOL_NAME.value] + litellm_model_tool_calls_total = litellm_output_tokens_metric + [UserAPIKeyLabelNames.MODEL_TOOL_NAME.value] litellm_deployment_state = [ UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, diff --git a/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py index f015e2c9d49..fae577286a3 100644 --- a/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py +++ b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py @@ -1,4 +1,5 @@ import datetime +import logging from collections.abc import Mapping from typing import Final, cast @@ -7,7 +8,12 @@ from prometheus_client import REGISTRY from prometheus_client.samples import Sample import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.prompt_templates.factory import ( + get_tool_call_names_from_response, + get_tool_calls_from_response, +) from litellm.types.utils import StandardLoggingPayload NOW: Final = datetime.datetime(2026, 11, 3, 12, 0, 0) @@ -186,3 +192,184 @@ async def test_series_carry_the_output_token_labels_plus_tool_name(): token_labels: Final = {**_samples("litellm_output_tokens_metric")[0].labels} (tool_sample,) = _samples("litellm_model_tool_calls") assert tool_sample.labels == {**token_labels, "tool_name": "Grep"} + + +SECRET: Final = "card 4111-1111-1111-1111" + + +def _chat_response_with_arguments(*arguments: str) -> litellm.ModelResponse: + calls: Final = tuple( + {"id": f"call_{n}", "type": "function", "function": {"name": "pay", "arguments": raw}} + for n, raw in enumerate(arguments) + ) + return litellm.ModelResponse(choices=[_chat_choice(0, *calls)]) + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def litellm_log_records(): + handler: Final = _Records() + lg: Final = logging.getLogger("LiteLLM") + previous_level: Final = lg.level + lg.addHandler(handler) + lg.setLevel(logging.DEBUG) + yield handler.messages + lg.removeHandler(handler) + lg.setLevel(previous_level) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "arguments", + ( + pytest.param('{"note": "' + SECRET, id="truncated-repairable"), + pytest.param('{"note": "' + SECRET + '" oops', id="unparseable"), + ), +) +async def test_counting_never_logs_the_tool_arguments(arguments: str, litellm_log_records: list[str]): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), _chat_response_with_arguments(arguments), NOW, NOW) + + assert _tool_call_counts() == {"pay": 1} + assert not [message for message in litellm_log_records if "4111" in message] + + +@pytest.mark.parametrize( + "arguments", + ( + pytest.param(('{"a": 1}',), id="valid"), + pytest.param(('{"a": 1',), id="truncated"), + pytest.param(('{"a": 1}{"a": 2}',), id="concatenated"), + pytest.param(('{"a": 1}{"a": 1}',), id="concatenated-identical"), + pytest.param(("".join(f'{{"a": {n}}}' for n in range(12)),), id="concatenated-over-cap"), + pytest.param(("not json",), id="garbage"), + pytest.param(("",), id="empty"), + pytest.param((REDACTED_BY_LITELLM,), id="redacted"), + pytest.param(('{"a": 1}', '{"b": 2}{"b": 3}'), id="two-calls"), + ), +) +def test_name_only_extraction_counts_like_the_parse_path(arguments: tuple[str, ...]): + response: Final = _chat_response_with_arguments(*arguments) + + parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True)) + + assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed + + +def _dict_chat_response(*choices: Mapping[str, object]) -> Mapping[str, object]: + return {"id": "chat_1", "choices": list(choices)} + + +@pytest.mark.parametrize( + "response", + ( + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": '{"x": 1}{"x": 2}'}}]}} + ), + id="plain-dict-chat", + ), + pytest.param( + _dict_chat_response({"message": {"tool_calls": [{"id": "a", "function": {"name": "f"}}]}}), + id="missing-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": None}}]}} + ), + id="none-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": {"x": 1}}}]}} + ), + id="parsed-dict-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": None}, {"id": "b", "function": {"name": "g"}}]}} + ), + id="function-none", + ), + pytest.param( + _dict_chat_response( + {"message": {"content": "no tools"}}, + {"message": {"tool_calls": [{"id": "b", "function": {"name": "g", "arguments": '{"y": 1'}}]}}, + ), + id="second-choice-only", + ), + pytest.param( + {"output": [{"type": "function_call", "call_id": "c", "name": "h", "arguments": '{"z": 1}{"z": 2}'}]}, + id="responses-api-concatenated", + ), + pytest.param( + {"output": [{"type": "function_call", "call_id": "c", "name": "h"}]}, id="responses-api-no-arguments" + ), + pytest.param(MESSAGES_TOOL_USE_RESPONSE, id="messages-tool-use"), + pytest.param(RESPONSES_API_RESPONSE, id="responses-api"), + ), +) +def test_name_only_extraction_matches_the_parse_path_on_every_surface(response: object): + parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True)) + + assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed + + +@pytest.mark.asyncio +async def test_responses_api_counting_never_logs_the_tool_arguments(litellm_log_records: list[str]): + logger: Final = PrometheusLogger() + response: Final = { + "id": "resp_2", + "output": [ + {"type": "function_call", "call_id": "c", "name": "pay", "arguments": '{"note": "' + SECRET + '" x'} + ], + } + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == {"pay": 1} + assert not [message for message in litellm_log_records if "4111" in message] + + +@pytest.mark.asyncio +async def test_a_cap_of_zero_counts_every_tool_as_other(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 0) + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + + assert _tool_call_counts() == {"other": 2} + + +@pytest.mark.asyncio +async def test_tool_names_past_the_cap_count_as_other(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 2) + logger: Final = PrometheusLogger() + response: Final = litellm.ModelResponse( + choices=[_chat_choice(0, *(_chat_tool_call(name, name) for name in ("Read", "Bash", "Grep", "Glob", "Read")))] + ) + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == {"Read": 2, "Bash": 1, "other": 2} + + +@pytest.mark.asyncio +async def test_no_cap_when_the_setting_is_none(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", None) + logger: Final = PrometheusLogger() + names: Final = tuple(f"tool_{n}" for n in range(1200)) + response: Final = litellm.ModelResponse(choices=[_chat_choice(0, *(_chat_tool_call(n, n) for n in names))]) + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert len(_tool_call_counts()) == len(names)