diff --git a/litellm/__init__.py b/litellm/__init__.py index 102c4d1440f..5f725c1ba21 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -488,6 +488,10 @@ prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000 prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0 prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0 prometheus_metrics_max_series_per_metric: Optional[int] = None +# Distinct ``tool_name`` values ``litellm_model_tool_calls_total`` gives their own series, per worker +# process; later names count under ``other`` until restart (a real tool named ``other`` shares that series). +# It bounds names, not series: each name still splits by the metric's other labels. None disables the cap. +prometheus_model_tool_name_max_values: Optional[int] = 1000 prometheus_metrics_ttl_seconds: Optional[float] = None prometheus_metrics_cleanup_interval_seconds: Optional[float] = 60.0 disable_add_prefix_to_prompt: bool = False # used by anthropic, to disable adding prefix to prompt diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index a8c92ac45eb..f9e18e5dfbf 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -117,6 +117,7 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel) _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0 UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other" +OVERFLOW_MODEL_TOOL_NAME_LABEL: Final = "other" _NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset( ( @@ -540,6 +541,7 @@ class PrometheusLogger(CustomLogger): _custom_buckets: Final = litellm.prometheus_latency_buckets self.latency_buckets = tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker() + self._admitted_model_tool_names: set[str] = set() # mutable-ok: names admitted under the tool_name cap _multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR") self._series_cap_tracker = ( BoundedPrometheusSeriesTracker() @@ -2168,10 +2170,25 @@ class PrometheusLogger(CustomLogger): def _increment_model_tool_call_metrics(self, response_obj: object, enum_values: UserAPIKeyLabelValues) -> None: from litellm.proxy.db.spend_log_tool_index import response_tool_call_names - tool_names: Final = response_tool_call_names(response_obj) + tool_names: Final = tuple( + self._bounded_model_tool_name(name) for name in response_tool_call_names(response_obj) + ) for tool_name, amount in Counter(tool_names).items(): self._inc_model_tool_call_counter(replace(enum_values, tool_name=tool_name), amount=amount) + def _bounded_model_tool_name(self, tool_name: str) -> str: + """The model picks tool names, from tools the caller declared or not, so + only the first ``prometheus_model_tool_name_max_values`` distinct names + get their own series; later names count under ``other``. Concurrent + callbacks may admit a few names past the cap, which still bounds it.""" + max_values: Final = litellm.prometheus_model_tool_name_max_values + if max_values is None or tool_name in self._admitted_model_tool_names: + return tool_name + if len(self._admitted_model_tool_names) >= max_values: + return OVERFLOW_MODEL_TOOL_NAME_LABEL + self._admitted_model_tool_names.add(tool_name) + return tool_name + def _inc_model_tool_call_counter(self, tool_enum_values: UserAPIKeyLabelValues, amount: int) -> None: PrometheusLogger._inc_labeled_counter( self, diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6ecab166dcd..4dac3b82a25 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -3341,6 +3341,24 @@ def salvage_concatenated_tool_arguments(raw: str) -> tuple[dict[str, object], .. return kept +def tool_call_argument_count(arguments: str) -> int: + """How many tool calls one raw argument string stands for, decided the way + ``get_tool_calls_from_response`` decides it: valid or repairable JSON is one + call, a concatenation of JSON objects is one call per salvaged object, and + anything else is one call with empty arguments. Unlike that path it never + logs any part of the argument text, so usage counters can call it on every + response.""" + if not arguments.strip(): + return 1 + try: + json.loads(arguments) + except json.JSONDecodeError: + if _attempt_json_repair(arguments) is not None: + return 1 + return len(salvage_concatenated_tool_arguments(arguments)) or 1 + return 1 + + def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues, ...]: """ Wrap an OpenAI ``/v1/completions`` ``prompt`` into Chat Completion messages. diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 4c3907dc52f..b5c6fdfb727 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -76,6 +76,7 @@ from .common_utils import ( native_redacted_thinking_data, parse_tool_call_arguments, salvage_concatenated_tool_arguments, + tool_call_argument_count, ) from .image_handling import convert_url_to_base64, is_pdf_media_type @@ -6482,6 +6483,65 @@ def get_tool_calls_from_response(response: object, include_all_choices: bool = F return [] +def _tool_call_count(raw_arguments: object) -> int: + # A missing ``arguments`` field parses as ``{}`` on the parse path: one call. + if not isinstance(raw_arguments, str): + return 1 + return tool_call_argument_count("{}" if raw_arguments == REDACTED_BY_LITELLM else raw_arguments) + + +def _repeated_name(call: object) -> tuple[str | None, ...]: + return (_optional_tool_name(message_field(call, "name")),) * _tool_call_count(message_field(call, "arguments")) + + +def _choice_tool_call_names(choice: object) -> tuple[str | None, ...]: + functions: Final = tuple(message_field(tool_call, "function") for tool_call in _choice_tool_calls(choice)) + return tuple( + itertools.chain.from_iterable(_repeated_name(function) for function in functions if function is not None) + ) + + +def _chat_tool_call_names(response: object, include_all_choices: bool) -> tuple[str | None, ...]: + return tuple( + itertools.chain.from_iterable( + _choice_tool_call_names(choice) for choice in _selected_choices(response, include_all_choices) + ) + ) + + +def _responses_api_tool_call_names(response: object) -> tuple[str | None, ...]: + return tuple(itertools.chain.from_iterable(_repeated_name(item) for item in _response_function_calls(response))) + + +def _messages_api_tool_call_names(response: object) -> tuple[str | None, ...]: + content: Final = message_field(response, "content") + blocks: Final = parts_of(content) + if not isinstance(content, list): + return () + return tuple( + _optional_tool_name(message_field(block, "name")) + for block in blocks + if message_field(block, "type") == "tool_use" + ) + + +def get_tool_call_names_from_response(response: object, include_all_choices: bool = False) -> tuple[str | None, ...]: + """ + The ``name`` of every call ``get_tool_calls_from_response`` returns, in the + same order and with the same count, without parsing argument values into + objects or logging them. Usage counters (Prometheus, the spend-log tool + index) only need the names, and the parse path logs up to the full argument + text of a malformed call, which may hold prompt data. + """ + chat_names: Final = _chat_tool_call_names(response, include_all_choices=include_all_choices) + if chat_names: + return chat_names + responses_names: Final = _responses_api_tool_call_names(response) + if responses_names: + return responses_names + return _messages_api_tool_call_names(response) + + def has_tool_with_name(tools: object, tool_name: str) -> bool: """ Check whether a tools list (as sent to an LLM) includes a tool with the diff --git a/litellm/proxy/db/spend_log_tool_index.py b/litellm/proxy/db/spend_log_tool_index.py index 591ad81a4e5..cf9d10b0fef 100644 --- a/litellm/proxy/db/spend_log_tool_index.py +++ b/litellm/proxy/db/spend_log_tool_index.py @@ -47,20 +47,17 @@ class ToolUsageTransaction: def response_tool_call_names(completion_response: object) -> tuple[str, ...]: """Tool names invoked in a completion response, in call order, for any response - surface get_tool_calls_from_response understands (chat completions, Responses + surface get_tool_call_names_from_response understands (chat completions, Responses API output items, Anthropic Messages tool_use blocks), plus the MCP tools an auto-executed Responses API call ran. Reads every choice of an ``n>1`` chat response: each choice cost money and its tool calls ran.""" if completion_response is None or isinstance(completion_response, Exception): return () from litellm.litellm_core_utils.prompt_templates.factory import ( - get_tool_calls_from_response, + get_tool_call_names_from_response, ) - called_names: Final = ( - tool_call.get("name") - for tool_call in get_tool_calls_from_response(completion_response, include_all_choices=True) - ) + called_names: Final = get_tool_call_names_from_response(completion_response, include_all_choices=True) return tuple( stripped for name in chain(called_names, _auto_executed_mcp_tool_names(completion_response)) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 91ee155009e..a156254837d 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -678,7 +678,7 @@ class PrometheusMetricLabels: litellm_video_duration_seconds_metric = litellm_output_tokens_metric litellm_images_generated_metric = litellm_output_tokens_metric - litellm_model_tool_calls_total = [*litellm_output_tokens_metric, UserAPIKeyLabelNames.MODEL_TOOL_NAME.value] + litellm_model_tool_calls_total = litellm_output_tokens_metric + [UserAPIKeyLabelNames.MODEL_TOOL_NAME.value] litellm_deployment_state = [ UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, diff --git a/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py index f015e2c9d49..fae577286a3 100644 --- a/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py +++ b/tests/unit/integrations/test_prometheus_model_tool_call_metrics.py @@ -1,4 +1,5 @@ import datetime +import logging from collections.abc import Mapping from typing import Final, cast @@ -7,7 +8,12 @@ from prometheus_client import REGISTRY from prometheus_client.samples import Sample import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.integrations.prometheus import PrometheusLogger +from litellm.litellm_core_utils.prompt_templates.factory import ( + get_tool_call_names_from_response, + get_tool_calls_from_response, +) from litellm.types.utils import StandardLoggingPayload NOW: Final = datetime.datetime(2026, 11, 3, 12, 0, 0) @@ -186,3 +192,184 @@ async def test_series_carry_the_output_token_labels_plus_tool_name(): token_labels: Final = {**_samples("litellm_output_tokens_metric")[0].labels} (tool_sample,) = _samples("litellm_model_tool_calls") assert tool_sample.labels == {**token_labels, "tool_name": "Grep"} + + +SECRET: Final = "card 4111-1111-1111-1111" + + +def _chat_response_with_arguments(*arguments: str) -> litellm.ModelResponse: + calls: Final = tuple( + {"id": f"call_{n}", "type": "function", "function": {"name": "pay", "arguments": raw}} + for n, raw in enumerate(arguments) + ) + return litellm.ModelResponse(choices=[_chat_choice(0, *calls)]) + + +class _Records(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + +@pytest.fixture +def litellm_log_records(): + handler: Final = _Records() + lg: Final = logging.getLogger("LiteLLM") + previous_level: Final = lg.level + lg.addHandler(handler) + lg.setLevel(logging.DEBUG) + yield handler.messages + lg.removeHandler(handler) + lg.setLevel(previous_level) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "arguments", + ( + pytest.param('{"note": "' + SECRET, id="truncated-repairable"), + pytest.param('{"note": "' + SECRET + '" oops', id="unparseable"), + ), +) +async def test_counting_never_logs_the_tool_arguments(arguments: str, litellm_log_records: list[str]): + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), _chat_response_with_arguments(arguments), NOW, NOW) + + assert _tool_call_counts() == {"pay": 1} + assert not [message for message in litellm_log_records if "4111" in message] + + +@pytest.mark.parametrize( + "arguments", + ( + pytest.param(('{"a": 1}',), id="valid"), + pytest.param(('{"a": 1',), id="truncated"), + pytest.param(('{"a": 1}{"a": 2}',), id="concatenated"), + pytest.param(('{"a": 1}{"a": 1}',), id="concatenated-identical"), + pytest.param(("".join(f'{{"a": {n}}}' for n in range(12)),), id="concatenated-over-cap"), + pytest.param(("not json",), id="garbage"), + pytest.param(("",), id="empty"), + pytest.param((REDACTED_BY_LITELLM,), id="redacted"), + pytest.param(('{"a": 1}', '{"b": 2}{"b": 3}'), id="two-calls"), + ), +) +def test_name_only_extraction_counts_like_the_parse_path(arguments: tuple[str, ...]): + response: Final = _chat_response_with_arguments(*arguments) + + parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True)) + + assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed + + +def _dict_chat_response(*choices: Mapping[str, object]) -> Mapping[str, object]: + return {"id": "chat_1", "choices": list(choices)} + + +@pytest.mark.parametrize( + "response", + ( + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": '{"x": 1}{"x": 2}'}}]}} + ), + id="plain-dict-chat", + ), + pytest.param( + _dict_chat_response({"message": {"tool_calls": [{"id": "a", "function": {"name": "f"}}]}}), + id="missing-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": None}}]}} + ), + id="none-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": {"name": "f", "arguments": {"x": 1}}}]}} + ), + id="parsed-dict-arguments", + ), + pytest.param( + _dict_chat_response( + {"message": {"tool_calls": [{"id": "a", "function": None}, {"id": "b", "function": {"name": "g"}}]}} + ), + id="function-none", + ), + pytest.param( + _dict_chat_response( + {"message": {"content": "no tools"}}, + {"message": {"tool_calls": [{"id": "b", "function": {"name": "g", "arguments": '{"y": 1'}}]}}, + ), + id="second-choice-only", + ), + pytest.param( + {"output": [{"type": "function_call", "call_id": "c", "name": "h", "arguments": '{"z": 1}{"z": 2}'}]}, + id="responses-api-concatenated", + ), + pytest.param( + {"output": [{"type": "function_call", "call_id": "c", "name": "h"}]}, id="responses-api-no-arguments" + ), + pytest.param(MESSAGES_TOOL_USE_RESPONSE, id="messages-tool-use"), + pytest.param(RESPONSES_API_RESPONSE, id="responses-api"), + ), +) +def test_name_only_extraction_matches_the_parse_path_on_every_surface(response: object): + parsed: Final = tuple(call["name"] for call in get_tool_calls_from_response(response, include_all_choices=True)) + + assert get_tool_call_names_from_response(response, include_all_choices=True) == parsed + + +@pytest.mark.asyncio +async def test_responses_api_counting_never_logs_the_tool_arguments(litellm_log_records: list[str]): + logger: Final = PrometheusLogger() + response: Final = { + "id": "resp_2", + "output": [ + {"type": "function_call", "call_id": "c", "name": "pay", "arguments": '{"note": "' + SECRET + '" x'} + ], + } + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == {"pay": 1} + assert not [message for message in litellm_log_records if "4111" in message] + + +@pytest.mark.asyncio +async def test_a_cap_of_zero_counts_every_tool_as_other(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 0) + logger: Final = PrometheusLogger() + + await logger.async_log_success_event(_success_kwargs(), MESSAGES_TOOL_USE_RESPONSE, NOW, NOW) + + assert _tool_call_counts() == {"other": 2} + + +@pytest.mark.asyncio +async def test_tool_names_past_the_cap_count_as_other(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", 2) + logger: Final = PrometheusLogger() + response: Final = litellm.ModelResponse( + choices=[_chat_choice(0, *(_chat_tool_call(name, name) for name in ("Read", "Bash", "Grep", "Glob", "Read")))] + ) + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert _tool_call_counts() == {"Read": 2, "Bash": 1, "other": 2} + + +@pytest.mark.asyncio +async def test_no_cap_when_the_setting_is_none(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "prometheus_model_tool_name_max_values", None) + logger: Final = PrometheusLogger() + names: Final = tuple(f"tool_{n}" for n in range(1200)) + response: Final = litellm.ModelResponse(choices=[_chat_choice(0, *(_chat_tool_call(n, n) for n in names))]) + + await logger.async_log_success_event(_success_kwargs(), response, NOW, NOW) + + assert len(_tool_call_counts()) == len(names)