diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index f318bb42b82..338afe04e5e 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -222,7 +222,29 @@ lives in [`plumbing/`](./plumbing): otherwise the operator's globally configured `MeterProvider` is reused so its readers/exporters receive them alongside the server metrics, and one is built and registered as the global only when none is set (mirroring how V2 owns trace - export). + export). A **failed** call records `gen_ai.client.operation.duration` too, + carrying the semconv `error.type` (the mapped provider exception's class name), + so the histogram covers the whole traffic and failure-rate panels are buildable; + the other five instruments describe a completed generation and are skipped + rather than filled with a fabricated zero. `error.type` is stamped after the + cardinality filter, so an `otel.attributes` list cannot merge the failure series + back into the success series. A proxy-gate rejection (auth / rate limit) records + nothing, for the same reason it gets no span: no upstream call happened. + Both paths cap their attributes at `METRIC_ATTRIBUTE_CEILING` before the + operator's own `otel.attributes` filter runs, so the filter can narrow the set + but never widen it. The ceiling is what keeps series count bounded by the + deployment's own key/team/user/deployment count instead of by its traffic: a + label value that moves per request mints a time series per request, which both + bills per request on a hosted backend and leaves a histogram that cannot be + aggregated. So client-supplied and per-request metadata (`requester_metadata`, + `spend_logs_metadata`, `user_api_key_end_user_id`, `requester_ip_address`) is + metric-ineligible and stays on the span, where cardinality is free, and the + `hidden_params` label carries only `model_id`, the deployment identity a + per-deployment panel joins on. `api_base` is excluded despite naming the same + deployment, because it is a documented per-call parameter and so is caller-chosen + in SDK use. Because the shared validator accepts every span attribute name, a + filter that names a metric-ineligible one logs a warning once when the filter + resolves rather than silently emitting nothing for it. - [`events.py`](./plumbing/events.py) — GenAI client events. Gated on `enable_events` (`LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS`), a failed LLM call records the semconv `gen_ai.client.operation.exception` log event at severity diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index b33973f0676..b8908696547 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -281,13 +281,29 @@ class OpenTelemetryV2(CustomLogger): self._record_metrics(kwargs, response_obj, start_time, end_time) def _record_metrics(self, kwargs, response_obj, start_time, end_time) -> None: - """Record the GenAI metrics for a successful LLM call. Best-effort: a - recording failure (e.g. a malformed payload) must never break the span - close or the request itself.""" + """Record the GenAI metrics for a successful LLM call.""" + self._guarded_record(lambda recorder: recorder.record(kwargs, response_obj, start_time, end_time)) + + def _record_failure_metrics(self, kwargs, start_time, end_time) -> None: + """Record the GenAI metrics for a failed LLM call, so the duration + histogram covers the whole traffic rather than only what survived. + + A synthetic proxy-gate log (auth / rate-limit rejection) is skipped for the + same reason it gets no span: no upstream call happened, so its duration is + not a GenAI operation's duration and would pull the histogram down.""" + if LLMCallEvent.from_dict(kwargs).is_no_upstream_call: + return + self._guarded_record(lambda recorder: recorder.record_failure(kwargs, start_time, end_time)) + + def _guarded_record(self, record: "Callable[[GenAIMetricRecorder], None]") -> None: + """Run one metric recording. Best-effort: a recording failure (e.g. a + malformed payload) must never break the span close or the request itself. A + misconfigured attribute filter is operator-fixable, so it is surfaced once + at ERROR instead of being swallowed.""" if self._metrics_recorder is None: return try: - self._metrics_recorder.record(kwargs, response_obj, start_time, end_time) + record(self._metrics_recorder) except ValueError as exc: if not self._metric_filter_error_logged: verbose_logger.error( @@ -304,6 +320,7 @@ class OpenTelemetryV2(CustomLogger): if self._emit_mcp_list_tools(kwargs, start_time, end_time): return self._close_llm_call(kwargs, start_time, end_time) + self._record_failure_metrics(kwargs, start_time, end_time) def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context: """Seed authenticated request-identity Baggage onto ``context`` so the Baggage diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 50d0fb75962..4d3c39e33d4 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -1,6 +1,6 @@ """GenAI client metrics: the six ``gen_ai.client.*`` histograms plus the recorder that builds attributes, applies the shared cardinality filter, and -records a request's metrics in the success path. +records a request's metrics on both the success and the failure path. The instrument names/units/descriptions and the recording + timing math mirror the v1 :mod:`litellm.integrations.opentelemetry` integration so both engines emit @@ -10,11 +10,12 @@ identical metrics. The attribute cardinality filter is reused from v1 by import from dataclasses import dataclass from datetime import datetime -from typing import Any, FrozenSet, Mapping, Optional +from typing import Any, Final, FrozenSet, Mapping, Optional, TypeAlias from opentelemetry.metrics import Histogram, Meter import litellm +from litellm._logging import verbose_logger from litellm.integrations.opentelemetry import ( METRIC_METADATA_KEYS, TOKEN_TYPE_ATTRIBUTE, @@ -22,7 +23,7 @@ from litellm.integrations.opentelemetry import ( _resolve_metric_attribute_filter, ) from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds -from litellm.integrations.otel.model.semconv import Metric, resolve_operation +from litellm.integrations.otel.model.semconv import Error, Metric, resolve_operation from litellm.integrations.otel.model.utils import to_seconds from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -72,8 +73,82 @@ def create_genai_metrics(meter: Meter) -> GenAIMetrics: ) +# A metric datapoint's attributes. Values are the strings the recorder builds, except +# the request model, which is whatever the caller passed and may be absent. +MetricAttributes: TypeAlias = Mapping[str, "str | None"] + +ERROR_TYPE_FALLBACK: Final = "_OTHER" + +# Every attribute a metric datapoint may carry, on either path. A label value that +# is unique per request is a new time series that will never be written to again, so +# this set is what keeps the series count bounded by the deployment's own +# key/team/user/deployment count rather than by its traffic. Each entry is a fixed +# enum or an operator-provisioned identifier. +# +# Deliberately excluded is everything the *client* supplies or that moves per +# request: ``metadata.requester_metadata`` and ``metadata.spend_logs_metadata`` (both +# free-form from the request body), ``metadata.user_api_key_end_user_id`` (the body's +# ``user`` field), and ``metadata.requester_ip_address``. Those stay on the span, +# where cardinality is free and where they already are. +# ``metadata.user_api_key_user_email`` is left out too: it is bounded, but it is PII +# duplicating the user id already here. +# +# This is a CEILING, applied before the operator's own include/exclude filter, so an +# operator can narrow it but never widen it back to an unbounded attribute. +METRIC_ATTRIBUTE_CEILING: Final[frozenset[str]] = frozenset( + ( + "gen_ai.operation.name", + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.framework", + "metadata.user_api_key_hash", + "metadata.user_api_key_alias", + "metadata.user_api_key_team_id", + "metadata.user_api_key_team_alias", + "metadata.user_api_key_org_id", + "metadata.user_api_key_user_id", + "hidden_params", + ) +) + +# The only ``hidden_params`` field that becomes part of the ``hidden_params`` label. +# The object as a whole is per-request by construction -- ``response_cost``, +# ``litellm_overhead_time_ms``, ``cache_key``, ``usage_object`` and the provider's +# ``additional_headers`` rate-limit counters all move on every call -- so dumping it +# whole made one series per request out of every instrument. +# +# ``model_id`` is the router's own deployment id, so it is bounded by the deployment +# list and is what a per-deployment panel joins on. ``api_base`` is deliberately NOT +# here even though it names the same thing: it is a documented per-call parameter, so +# in SDK use it is chosen by the caller rather than provisioned by the operator, and a +# caller varying it would put the per-request cardinality straight back. +BOUNDED_HIDDEN_PARAM_KEYS: Final[tuple[str, ...]] = ("model_id",) + + +def resolve_error_type(kwargs: Mapping[str, Any]) -> str: + """The ``error.type`` value for a failed request. + + Bounded by construction: the mapped provider exception's class name (the same + ``error_information.error_class`` the failure span stamps), else the provider + status code, else the raw exception's class name, else ``_OTHER`` — the value + the convention reserves for a failure the instrumentation cannot classify. The + exception *message* is unbounded and never becomes a label; it stays on the + span and its exception event, where high cardinality is free. + """ + std_log = kwargs.get("standard_logging_object") + info = getattr(std_log, "error_information", None) or (std_log or {}).get("error_information") or {} + error_class = info.get("error_class") or info.get("error_code") + if error_class: + return str(error_class) + exception = kwargs.get("exception") + if exception is not None: + return type(exception).__name__ + return ERROR_TYPE_FALLBACK + + class GenAIMetricRecorder: - """Records the six GenAI histograms for one successful LLM call. + """Records the six GenAI histograms for one successful LLM call, and the + duration histogram alone for one failed LLM call (see :meth:`record_failure`). The cardinality filter is resolved lazily on the first record: the proxy populates ``callback_settings.otel.attributes`` after the logger is built, so @@ -96,7 +171,7 @@ class GenAIMetricRecorder: start_time: datetime, end_time: datetime, ) -> None: - common_attrs = self._filter_attributes(self._common_attributes(kwargs)) + common_attrs = self._filter_attributes(self._bounded_attributes(kwargs)) duration_s = (end_time - start_time).total_seconds() self._metrics.operation_duration.record(duration_s, attributes=common_attrs) @@ -110,6 +185,38 @@ class GenAIMetricRecorder: self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs) self._record_response_duration(kwargs, end_time, common_attrs) + def record_failure( + self, + kwargs: Mapping[str, Any], + start_time: datetime, + end_time: datetime, + ) -> None: + """Record the one metric a failed request can honestly report: the + operation's duration, tagged with ``error.type``. + + The other five instruments all describe a completed generation and have + nothing to measure here. litellm hands the failure callback no + ``response_obj`` at all, so there is no usage to split into input/output + tokens and no completion-token count to divide generation time by; it also + zeroes ``response_cost`` on failure. Recording them anyway would put a + fabricated zero into series that dashboards average. + + The attribute set is :data:`METRIC_ATTRIBUTE_CEILING`, the same cap the + success path uses. A failure needs no provider spend, so a caller who can put + a unique value into a client-supplied attribute could mint one histogram + series per request for free; the cap is what makes that impossible on either + path. + + ``error.type`` is stamped after both filters, exactly like + ``gen_ai.token.type``, so an operator's include/exclude list cannot strip + the discriminator and silently merge failures back into the success series. + """ + attributes = { + **self._filter_attributes(self._bounded_attributes(kwargs)), + Error.TYPE: resolve_error_type(kwargs), + } + self._metrics.operation_duration.record((end_time - start_time).total_seconds(), attributes=attributes) + # ------------------------------------------------------------------ # # Attribute building + cardinality filter # ------------------------------------------------------------------ # @@ -136,11 +243,25 @@ class GenAIMetricRecorder: common_attrs[f"metadata.{key}"] = str(value) hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get("hidden_params", {}) - if hidden_params: - common_attrs["hidden_params"] = safe_dumps(hidden_params) + bounded_hidden_params = { + key: hidden_params[key] + for key in BOUNDED_HIDDEN_PARAM_KEYS + if isinstance(hidden_params, Mapping) and hidden_params.get(key) is not None + } + if bounded_hidden_params: + common_attrs["hidden_params"] = safe_dumps(bounded_hidden_params) return common_attrs + def _bounded_attributes(self, kwargs: Mapping[str, Any]) -> MetricAttributes: + """The datapoint attributes, capped at :data:`METRIC_ATTRIBUTE_CEILING`. + + The cap runs BEFORE the operator's include/exclude filter so the filter can + only narrow it. An operator who names an excluded attribute in an include + list gets nothing for it rather than reintroducing an unbounded label. + """ + return {k: v for k, v in self._common_attributes(kwargs).items() if k in METRIC_ATTRIBUTE_CEILING} + def _ensure_filter(self) -> None: if self._filter_resolved: return @@ -157,8 +278,29 @@ class GenAIMetricRecorder: # without reconstructing the recorder. self._include, self._exclude = _resolve_metric_attribute_filter(attributes) self._filter_resolved = True + self._warn_about_metric_ineligible_names() - def _filter_attributes(self, attrs: dict) -> dict: + def _warn_about_metric_ineligible_names(self) -> None: + """Say so when the operator's filter names an attribute the ceiling removes. + + The shared validator accepts every span attribute name, so a name that is + legal on a span but metric-ineligible would otherwise be a silent no-op: an + ``include_list`` naming it emits nothing for it and an ``exclude_list`` naming + it looks like it worked. Logged once, when the filter resolves, rather than + per request. + """ + named = (self._include or frozenset()) | (self._exclude or frozenset()) + ineligible = sorted(named - METRIC_ATTRIBUTE_CEILING - {TOKEN_TYPE_ATTRIBUTE}) + if ineligible: + verbose_logger.warning( + "OTel metrics: %s cannot be a metric attribute and is being ignored; it varies " + "per request or is client-supplied, so it would make one time series per request. " + "It is still on the span. Metric attributes are limited to: %s", + ", ".join(ineligible), + ", ".join(sorted(METRIC_ATTRIBUTE_CEILING)), + ) + + def _filter_attributes(self, attrs: MetricAttributes) -> MetricAttributes: self._ensure_filter() if self._include is not None: return {k: v for k, v in attrs.items() if k in self._include} diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py index b2d89053ba3..56607414de4 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_metrics.py @@ -12,9 +12,17 @@ raises out of ``GenAIMetricRecorder.record`` -- asserted directly at the recorde layer -- and the logger turns that raise into a single ERROR ("metrics disabled") plus a quiet no-op for the rest of the process, asserted at the logger layer so the misconfig never breaks a request nor spams a log line per request. + +The failure path is driven the same way, through the real +``OpenTelemetryV2.async_log_failure_event``: a failed call records +``gen_ai.client.operation.duration`` and nothing else, tagged with ``error.type``, +and a success driven through the same reader keeps a datapoint whose attributes are +byte-for-byte what it had before the failure path existed -- the guard for every +dashboard already querying that histogram. """ import asyncio +import json from datetime import datetime, timedelta import pytest @@ -25,6 +33,9 @@ from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 import litellm # noqa: E402 +from litellm.constants import ( # noqa: E402 + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, +) from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402 from litellm.integrations.otel.model.config import ( # noqa: E402 OpenTelemetryV2Config, @@ -58,14 +69,13 @@ ALL_METRICS = frozenset( TOKEN_TYPE = "gen_ai.token.type" MODEL_KEY = "gen_ai.request.model" -# Each is a member of VALID_METRIC_ATTRIBUTE_NAMES and is stamped on the metric -# by default (proven by the no-filter test below). -HIGH_CARDINALITY_KEYS = ( +# Keys inside the ceiling that an operator's filter must still be able to remove. +# Every one is bounded, so it survives the ceiling and only the operator's own +# exclude_list takes it off; that is what makes the filter tests non-vacuous. +FILTERABLE_KEYS = ( "hidden_params", "metadata.user_api_key_hash", - "metadata.requester_ip_address", - "metadata.requester_metadata", - "metadata.applied_guardrails", + "metadata.user_api_key_team_id", ) PROMPT_TOKENS = 137 @@ -93,6 +103,7 @@ def _build_call(stream: bool = True): "standard_logging_object": { "metadata": { "user_api_key_hash": "hash-abc123", + "user_api_key_team_id": "team-1", "requester_ip_address": "10.0.0.7", "requester_metadata": {"team": "alpha", "tier": "gold"}, "applied_guardrails": ["pii", "toxicity"], @@ -205,14 +216,14 @@ def test_metrics_off_by_default_records_nothing(): def test_exclude_list_strips_high_cardinality_across_metrics(): - """exclude_list set AFTER construction (the proxy path) removes every - high-cardinality key from more than one metric while the low-cardinality - model attribute survives.""" + """exclude_list set AFTER construction (the proxy path) removes every listed + key from more than one metric while the low-cardinality model attribute + survives.""" metrics = _drive_success( InMemoryMetricReader(), - callback_settings_attributes={"exclude_list": list(HIGH_CARDINALITY_KEYS)}, + callback_settings_attributes={"exclude_list": list(FILTERABLE_KEYS)}, ) - excluded = set(HIGH_CARDINALITY_KEYS) + excluded = set(FILTERABLE_KEYS) for name in (OPERATION_DURATION, TOKEN_USAGE): points = metrics[name] @@ -241,18 +252,132 @@ def test_include_list_allows_only_listed_attributes(): assert set(dp.attributes.keys()) - {TOKEN_TYPE} == allowed -def test_no_filter_keeps_high_cardinality_keys(): - """Backward compatibility: without an attributes config every high-cardinality - key the call carries is still stamped, so the filter tests above prove a real - removal rather than a key that was never present.""" +def test_no_filter_still_keeps_the_filterable_keys(): + """Without an attributes config every key the filter tests remove is present, + so those tests prove a real removal rather than a key that was never there.""" metrics = _drive_success(InMemoryMetricReader()) - expected = set(HIGH_CARDINALITY_KEYS) + expected = set(FILTERABLE_KEYS) for name in (OPERATION_DURATION, TOKEN_USAGE): for dp in metrics[name]: assert expected.issubset(set(dp.attributes.keys())) +def test_a_metric_ineligible_filter_name_is_reported_not_silently_dropped(caplog): + """Naming a metric-ineligible attribute in a filter has to say so. + + The shared validator accepts every span attribute name, so an operator can put + one in an ``include_list``, get nothing for it, and have no way to tell that from + a value that happened to be absent. The ceiling is deliberate, but silent is what + makes it a support ticket. + """ + with caplog.at_level("WARNING"): + _drive_success( + InMemoryMetricReader(), + callback_settings_attributes={ + "include_list": [MODEL_KEY, "metadata.requester_ip_address"] + }, + ) + + reported = [ + r.getMessage().split(" cannot be a metric attribute")[0].removeprefix("OTel metrics: ") + for r in caplog.records + if r.levelname == "WARNING" and "cannot be a metric attribute" in r.getMessage() + ] + assert reported == ["metadata.requester_ip_address"], reported + + +def test_two_calls_differing_only_per_request_share_one_series(): + """The whole point of the ceiling: metric cardinality must not grow with traffic. + + Every field here moves on every real request -- the response cost, the call id, + the cache key, the provider's remaining-rate-limit headers -- and each one used + to reach the datapoint inside a single ``hidden_params`` label. A unique label + value is a new time series, so each of the six instruments minted one series per + request, which is both a Grafana Cloud bill proportional to traffic and a + histogram that cannot be aggregated. Identical attribute sets is what "one + series" means to the SDK. + """ + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + + for index, cost in enumerate((RESPONSE_COST, RESPONSE_COST * 3)): + kwargs, response_obj, start, end = _build_call() + kwargs["response_cost"] = cost + kwargs["standard_logging_object"]["hidden_params"] = { + "model_id": "m-1", + # A documented per-call parameter, so it varies here on purpose: the same + # deployment reached under a caller-chosen base must not split the series. + "api_base": f"https://proxy-{index}.example.com/v1", + "litellm_call_id": f"call-{index}", + "cache_key": f"cache-{index}", + "response_cost": cost, + "litellm_overhead_time_ms": 1.5 + index, + "usage_object": {"prompt_tokens": index, "completion_tokens": index}, + "additional_headers": {"x_ratelimit_remaining_requests": 100 - index}, + } + asyncio.run(logger.async_log_success_event(kwargs, response_obj, start, end)) + + for name in ALL_METRICS: + attribute_sets = { + tuple(sorted((k, v) for k, v in dp.attributes.items() if k != TOKEN_TYPE)) + for dp in _metrics_by_name(reader)[name] + } + assert len(attribute_sets) == 1, f"{name} split into {len(attribute_sets)} series across 2 requests" + + +def test_hidden_params_label_carries_only_bounded_deployment_fields(): + """``hidden_params`` survives the ceiling, but only as the deployment identity. + + ``model_id`` is the router's deployment id, bounded by the deployment list, and is + what a per-deployment dashboard reads. Everything else in the object is + per-request or caller-chosen and belongs on the span, which already carries it. + ``api_base`` is excluded despite naming the same deployment: it is a documented + per-call parameter, so a caller varying it would restore the per-request + cardinality this cap exists to remove. + """ + kwargs, response_obj, start, end = _build_call() + kwargs["standard_logging_object"]["hidden_params"] = { + "model_id": "m-1", + "api_base": "https://api.openai.com/v1", + "litellm_call_id": "abc", + "cache_key": "ck-1", + "response_cost": RESPONSE_COST, + } + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + asyncio.run(logger.async_log_success_event(kwargs, response_obj, start, end)) + + label = _metrics_by_name(reader)[OPERATION_DURATION][0].attributes["hidden_params"] + assert json.loads(label) == {"model_id": "m-1"} + + +def test_success_attributes_are_capped_at_the_ceiling(): + """The success path carries exactly the ceiling, no client-supplied attributes. + + The fixture deliberately sets every excluded key, so this asserts a real removal + rather than keys that were never present. + """ + kwargs, response_obj, start, end = _build_call() + metadata = kwargs["standard_logging_object"]["metadata"] + metadata.update( + { + "spend_logs_metadata": {"cost_center": "abc"}, + "user_api_key_end_user_id": "end-user-1", + "user_api_key_user_email": "someone@example.com", + } + ) + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + asyncio.run(logger.async_log_success_event(kwargs, response_obj, start, end)) + metrics = _metrics_by_name(reader) + + for name in ALL_METRICS: + for dp in metrics[name]: + leaked = set(dp.attributes) - set(BOUNDED_KEYS) - {TOKEN_TYPE} + assert not leaked, f"{name} leaked {leaked}" + + def test_metrics_reach_operator_configured_global_provider(monkeypatch): """Regression: with no meter provider injected, the six gen_ai.client.* histograms must record through the operator's globally configured @@ -331,3 +456,261 @@ def test_token_type_rejected_from_either_list(attributes, monkeypatch): # the specific reason so dropping that guard (and falling through to "unknown # attribute name") is caught. assert "discriminator" in str(exc_info.value) + + +# --- failure path ------------------------------------------------------------ # + +ERROR_TYPE = "error.type" +ERROR_CLASS = "RateLimitError" +FAILURE_DURATION_S = 1.0 + +# Attributes a failure datapoint must never carry. Each is either supplied by the +# caller (so a caller could mint a fresh series per request, and a failure costs +# them no provider spend) or varies per request, or is PII duplicating an id that +# is already on the series. +UNBOUNDED_KEYS = ( + "metadata.requester_metadata", + "metadata.requester_ip_address", + "metadata.spend_logs_metadata", + "metadata.user_api_key_end_user_id", + "metadata.user_api_key_user_email", +) + +# The exact set a datapoint may carry on either path: the operation, the +# operator-provisioned identity, and the deployment that served it. +BOUNDED_KEYS = ( + "hidden_params", + "gen_ai.operation.name", + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.framework", + "metadata.user_api_key_hash", + "metadata.user_api_key_alias", + "metadata.user_api_key_team_id", + "metadata.user_api_key_team_alias", + "metadata.user_api_key_org_id", + "metadata.user_api_key_user_id", +) + + +def _build_failure( + *, + error_information=None, + exception=None, + no_upstream_call=False, +): + """A captured failure-call ``(kwargs, start, end)``. + + Mirrors what litellm actually hands ``async_log_failure_event``: no + ``response_obj`` at all, but the streaming timings and the recovered + ``response_cost`` a mid-stream failure still carries -- so routing the failure + path through the full success recorder would show up here as extra series + rather than passing unnoticed. The metadata carries both the bounded identity + keys and every caller-supplied / per-request key, so the allowlist test below + proves a real removal rather than a key that was never there. + """ + start = datetime(2026, 6, 12, 12, 0, 0) + api_call_start = start + timedelta(seconds=0.1) + completion_start = start + timedelta(seconds=0.5) + end = start + timedelta(seconds=FAILURE_DURATION_S) + standard_logging_object = { + "status": "failure", + "metadata": { + "user_api_key_hash": "hash-abc123", + "user_api_key_alias": "alias-abc", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "team-alpha", + "user_api_key_org_id": "org-1", + "user_api_key_user_id": "user-1", + "user_api_key_user_email": "user@example.com", + "user_api_key_end_user_id": "end-user-42", + "requester_ip_address": "10.0.0.7", + "requester_metadata": {"trace": "caller-supplied-unique-value"}, + "spend_logs_metadata": {"ticket": "caller-supplied-unique-value"}, + }, + "hidden_params": { + "litellm_call_id": "abc", + "model_id": "m-1", + "api_base": "https://api.openai.com/v1", + }, + } + if error_information is not None: + standard_logging_object["error_information"] = error_information + kwargs = { + "model": "gpt-4o-mini", + "call_type": "completion", + "litellm_params": {"custom_llm_provider": "openai"}, + "optional_params": {"stream": True}, + "response_cost": RESPONSE_COST, + "api_call_start_time": api_call_start, + "completion_start_time": completion_start, + "end_time": end, + "standard_logging_object": standard_logging_object, + } + if exception is not None: + kwargs["exception"] = exception + if no_upstream_call: + kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + return kwargs, start, end + + +def _drive_failure(reader, callback_settings_attributes=None, **failure_kwargs): + logger = _logger(reader, enable_metrics=True) + previous = litellm.callback_settings + if callback_settings_attributes is not None: + litellm.callback_settings = {"otel": {"attributes": callback_settings_attributes}} + try: + kwargs, start, end = _build_failure(**failure_kwargs) + asyncio.run(logger.async_log_failure_event(kwargs, None, start, end)) + finally: + litellm.callback_settings = previous + return _metrics_by_name(reader) + + +def test_failure_records_only_the_duration_histogram(): + """A failed call contributes to gen_ai.client.operation.duration -- before this + existed a failure recorded nothing at all, so the histogram measured only the + traffic that survived. It contributes to nothing else: the other five + instruments describe a completed generation, and the call carries a streaming + timing pair and a recovered response_cost that would light four of them up if + the failure were routed through the success recorder.""" + metrics = _drive_failure( + InMemoryMetricReader(), + error_information={"error_class": ERROR_CLASS, "error_code": "429"}, + ) + + assert set(metrics.keys()) == {OPERATION_DURATION} + points = metrics[OPERATION_DURATION] + assert len(points) == 1 + assert points[0].count == 1 + assert points[0].sum == pytest.approx(FAILURE_DURATION_S) + assert points[0].attributes[ERROR_TYPE] == ERROR_CLASS + + +def test_success_and_failure_are_separable_and_success_attributes_unchanged(): + """The pooled histogram stays queryable per outcome, and the existing + dashboards keep working. + + A success and a failure through one reader must land on two distinct series -- + one with error.type, one without -- so a failure-rate panel is expressible and + an operator can still get success-only latency by filtering error.type="". The + success datapoint's attribute map must be byte-for-byte the map a success-only + run produces, which is what stops the new attribute from leaking onto the + series every current query reads.""" + baseline_reader = InMemoryMetricReader() + baseline = _drive_success(baseline_reader) + baseline_points = baseline[OPERATION_DURATION] + assert len(baseline_points) == 1 + baseline_attributes = dict(baseline_points[0].attributes) + + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + ok_kwargs, response_obj, ok_start, ok_end = _build_call() + asyncio.run(logger.async_log_success_event(ok_kwargs, response_obj, ok_start, ok_end)) + bad_kwargs, bad_start, bad_end = _build_failure(error_information={"error_class": ERROR_CLASS}) + asyncio.run(logger.async_log_failure_event(bad_kwargs, None, bad_start, bad_end)) + + points = metrics = _metrics_by_name(reader)[OPERATION_DURATION] + assert len(points) == 2, f"success and failure collapsed into {len(points)} series: {metrics}" + succeeded = [dp for dp in points if ERROR_TYPE not in dp.attributes] + failed = [dp for dp in points if dp.attributes.get(ERROR_TYPE) == ERROR_CLASS] + assert len(succeeded) == 1 and len(failed) == 1 + assert dict(succeeded[0].attributes) == baseline_attributes + + +def test_failure_attributes_are_a_bounded_allowlist(): + """A failure datapoint carries exactly the bounded allowlist plus error.type. + + A failed request needs no provider spend, so nothing rate-limits a caller who + puts a unique value into an attribute they control and mints one histogram + series per request. The same payload is driven through the success path first, + which does carry those keys, so this asserts a real removal on the failure path + rather than keys that were never present. The exact-set assertion is the guard + against the natural refactor of "just reuse _common_attributes".""" + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + kwargs, start, end = _build_failure(error_information={"error_class": ERROR_CLASS}) + usage = {"usage": {"prompt_tokens": 1, "completion_tokens": 1}} + asyncio.run(logger.async_log_success_event(kwargs, usage, start, end)) + asyncio.run(logger.async_log_failure_event(kwargs, None, start, end)) + + points = _metrics_by_name(reader)[OPERATION_DURATION] + succeeded = next(dp for dp in points if ERROR_TYPE not in dp.attributes) + failed = next(dp for dp in points if ERROR_TYPE in dp.attributes) + + supplied = set(kwargs["standard_logging_object"]["metadata"]) + missing = {key for key in UNBOUNDED_KEYS if key.removeprefix("metadata.") not in supplied} + assert not missing, f"fixture never carried {missing}, so the exclusion below proves nothing" + leaked = set(UNBOUNDED_KEYS) & set(failed.attributes) + assert not leaked, f"failure datapoint leaked unbounded attributes: {leaked}" + assert set(failed.attributes) == set(BOUNDED_KEYS) | {ERROR_TYPE} + assert json.loads(failed.attributes["hidden_params"]) == {"model_id": "m-1"} + + +def test_operator_filter_can_still_narrow_the_failure_allowlist(): + """The allowlist is a ceiling, not a floor: an exclude_list an operator sets + still removes a listed key from the failure series.""" + metrics = _drive_failure( + InMemoryMetricReader(), + callback_settings_attributes={"exclude_list": ["metadata.user_api_key_hash"]}, + error_information={"error_class": ERROR_CLASS}, + ) + attributes = metrics[OPERATION_DURATION][0].attributes + assert "metadata.user_api_key_hash" not in attributes + assert attributes[ERROR_TYPE] == ERROR_CLASS + assert attributes[MODEL_KEY] == "gpt-4o-mini" + + +@pytest.mark.parametrize( + "failure_kwargs, expected", + [ + ({"error_information": {"error_class": ERROR_CLASS, "error_code": "429"}}, ERROR_CLASS), + ({"error_information": {"error_code": "429"}}, "429"), + ({"exception": ValueError("boom")}, "ValueError"), + ({}, "_OTHER"), + ], + ids=["error_class", "error_code_only", "exception_fallback", "unclassifiable"], +) +def test_error_type_is_bounded_and_falls_back(failure_kwargs, expected): + """error.type is always a bounded value: the mapped exception's class name, the + provider status code, the raw exception's class name, or the semconv _OTHER + fallback. Never the exception message, which is unbounded.""" + metrics = _drive_failure(InMemoryMetricReader(), **failure_kwargs) + assert metrics[OPERATION_DURATION][0].attributes[ERROR_TYPE] == expected + + +def test_include_list_cannot_strip_error_type(): + """error.type is a structural discriminator like gen_ai.token.type: an + include_list that does not mention it must not merge the failure series back + into the success series, so it is stamped after the filter runs.""" + metrics = _drive_failure( + InMemoryMetricReader(), + callback_settings_attributes={"include_list": [MODEL_KEY]}, + error_information={"error_class": ERROR_CLASS}, + ) + attributes = metrics[OPERATION_DURATION][0].attributes + assert dict(attributes) == {MODEL_KEY: "gpt-4o-mini", ERROR_TYPE: ERROR_CLASS} + + +def test_proxy_gate_rejection_records_no_duration(): + """A synthetic proxy-gate failure log (auth / rate-limit rejection) never made + an upstream call, so its wall time is not a GenAI operation's duration; it is + skipped for the same reason it gets no span. Recording it would pull the + histogram toward the proxy's own latency. + + Both failures go through one reader so the assertion is that exactly the + upstream one landed, rather than the vacuous "nothing was recorded" a + failure path that records nothing at all would also satisfy.""" + reader = InMemoryMetricReader() + logger = _logger(reader, enable_metrics=True) + gate_kwargs, gate_start, gate_end = _build_failure( + error_information={"error_class": "AuthenticationError"}, + no_upstream_call=True, + ) + asyncio.run(logger.async_log_failure_event(gate_kwargs, None, gate_start, gate_end)) + upstream_kwargs, upstream_start, upstream_end = _build_failure(error_information={"error_class": ERROR_CLASS}) + asyncio.run(logger.async_log_failure_event(upstream_kwargs, None, upstream_start, upstream_end)) + + points = _metrics_by_name(reader)[OPERATION_DURATION] + assert [dp.attributes[ERROR_TYPE] for dp in points] == [ERROR_CLASS] + assert points[0].count == 1