diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 24780eb4bfc..fc37b6a34d8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1,7 +1,18 @@ import os from dataclasses import dataclass, field from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + FrozenSet, + List, + Optional, + Set, + Tuple, + Union, + cast, +) import litellm from litellm._logging import verbose_logger @@ -82,6 +93,88 @@ _VALID_CAPTURE_MODES = { CAPTURE_MODE_SPAN_AND_EVENT, } +METRIC_METADATA_KEYS: Tuple[str, ...] = ( + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_team_id", + "user_api_key_org_id", + "user_api_key_user_id", + "user_api_key_team_alias", + "user_api_key_user_email", + "spend_logs_metadata", + "requester_ip_address", + "requester_metadata", + "user_api_key_end_user_id", + "prompt_management_metadata", + "applied_guardrails", + "mcp_tool_call_metadata", + "vector_store_request_metadata", +) + +TOKEN_TYPE_ATTRIBUTE: str = "gen_ai.token.type" + +VALID_METRIC_ATTRIBUTE_NAMES: FrozenSet[str] = frozenset( + ( + "gen_ai.operation.name", + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.framework", + "hidden_params", + ) + + tuple(f"metadata.{key}" for key in METRIC_METADATA_KEYS) +) + + +@dataclass(frozen=True) +class OTELMetricAttributeFilter: + include_list: Optional[List[str]] = None + exclude_list: Optional[List[str]] = None + + +def _build_metric_attribute_filter(value: Any) -> OTELMetricAttributeFilter: + if isinstance(value, OTELMetricAttributeFilter): + return value + if not isinstance(value, dict): + raise ValueError( + "otel.attributes must be a mapping with optional 'include_list' / " + f"'exclude_list', got {type(value).__name__}" + ) + return OTELMetricAttributeFilter( + include_list=value.get("include_list"), + exclude_list=value.get("exclude_list"), + ) + + +def _resolve_metric_attribute_filter( + attributes: Optional[OTELMetricAttributeFilter], +) -> Tuple[Optional[FrozenSet[str]], Optional[FrozenSet[str]]]: + if attributes is None: + return None, None + include = attributes.include_list or None + exclude = attributes.exclude_list or None + if include and exclude: + raise ValueError( + "otel.attributes: include_list and exclude_list are mutually exclusive" + ) + requested = include or exclude or [] + if TOKEN_TYPE_ATTRIBUTE in requested: + raise ValueError( + f"otel.attributes: {TOKEN_TYPE_ATTRIBUTE} is a structural token-usage " + "discriminator and cannot be filtered" + ) + unknown = sorted( + name for name in requested if name not in VALID_METRIC_ATTRIBUTE_NAMES + ) + if unknown: + raise ValueError( + f"otel.attributes: unknown attribute name(s) {unknown}. " + f"Valid names: {sorted(VALID_METRIC_ATTRIBUTE_NAMES)}" + ) + return ( + frozenset(include) if include else None, + frozenset(exclude) if exclude else None, + ) + def _normalize_team_metadata_keys(value: Any) -> List[str]: """Coerce a team-metadata allowlist from a list or comma-separated string. @@ -117,6 +210,9 @@ class OpenTelemetryConfig: # under ``litellm.team.metadata``. Empty by default so none of a team's # metadata leaves the process until explicitly allowlisted. baggage_team_metadata_keys: List[str] = field(default_factory=list) + # Prometheus-style include/exclude control over which attributes are stamped + # on emitted metrics, to cap metric cardinality. + attributes: Optional[OTELMetricAttributeFilter] = None def __post_init__(self) -> None: # If endpoint is specified but exporter is still the default "console", @@ -211,15 +307,29 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): **kwargs, ): team_metadata_keys_override = kwargs.pop("baggage_team_metadata_keys", None) + metric_attributes_override = kwargs.pop("attributes", None) if config is None: config = OpenTelemetryConfig.from_env() if team_metadata_keys_override is not None: config.baggage_team_metadata_keys = _normalize_team_metadata_keys( team_metadata_keys_override ) + if metric_attributes_override is not None: + config.attributes = _build_metric_attribute_filter( + metric_attributes_override + ) self.config = config self.callback_name = callback_name + # Resolved on first metric record, not here: the proxy populates + # callback_settings.otel.attributes after this logger is constructed, so + # reading it now would miss it. An explicit config is validated eagerly so + # a bad config still fails at startup. + self._metric_attr_include: Optional[FrozenSet[str]] = None + self._metric_attr_exclude: Optional[FrozenSet[str]] = None + self._metric_attr_filter_resolved = False + if config.attributes is not None: + self._ensure_metric_attribute_filter() self.OTEL_EXPORTER = self.config.exporter self.OTEL_ENDPOINT = self.config.endpoint self.OTEL_HEADERS = self.config.headers @@ -1318,6 +1428,38 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): return None return safe_dumps(filtered) + def _ensure_metric_attribute_filter(self) -> None: + """Resolve the include/exclude filter once, falling back to the proxy's + callback_settings.otel.attributes when no explicit config was passed.""" + if self._metric_attr_filter_resolved: + return + attributes = self.config.attributes + if attributes is None and self.callback_name in (None, "otel"): + otel_settings = (litellm.callback_settings or {}).get("otel") or {} + raw = ( + otel_settings.get("attributes") + if isinstance(otel_settings, dict) + else None + ) + if raw is not None: + attributes = _build_metric_attribute_filter(raw) + ( + self._metric_attr_include, + self._metric_attr_exclude, + ) = _resolve_metric_attribute_filter(attributes) + self._metric_attr_filter_resolved = True + + def _filter_metric_attributes(self, attrs: Dict[str, Any]) -> Dict[str, Any]: + if not self._metric_attr_filter_resolved: + self._ensure_metric_attribute_filter() + if self._metric_attr_include is not None: + return {k: v for k, v in attrs.items() if k in self._metric_attr_include} + if self._metric_attr_exclude is not None: + return { + k: v for k, v in attrs.items() if k not in self._metric_attr_exclude + } + return attrs + def _record_metrics(self, kwargs, response_obj, start_time, end_time): duration_s = (end_time - start_time).total_seconds() params = kwargs.get("litellm_params") or {} @@ -1336,23 +1478,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): std_log = kwargs.get("standard_logging_object") md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {}) - for key in [ - "user_api_key_hash", - "user_api_key_alias", - "user_api_key_team_id", - "user_api_key_org_id", - "user_api_key_user_id", - "user_api_key_team_alias", - "user_api_key_user_email", - "spend_logs_metadata", - "requester_ip_address", - "requester_metadata", - "user_api_key_end_user_id", - "prompt_management_metadata", - "applied_guardrails", - "mcp_tool_call_metadata", - "vector_store_request_metadata", - ]: + for key in METRIC_METADATA_KEYS: value = md.get(key) if value is None: continue @@ -1368,6 +1494,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if hidden_params: common_attrs["hidden_params"] = safe_dumps(hidden_params) + common_attrs = self._filter_metric_attributes(common_attrs) + if self._operation_duration_histogram: self._operation_duration_histogram.record( duration_s, attributes=common_attrs @@ -1377,8 +1505,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): and (usage := response_obj.get("usage")) and self._token_usage_histogram ): - in_attrs = {**common_attrs, "gen_ai.token.type": "input"} - out_attrs = {**common_attrs, "gen_ai.token.type": "output"} + in_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"} + out_attrs = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"} self._token_usage_histogram.record( usage.get("prompt_tokens", 0), attributes=in_attrs ) diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index 0601f9c0eef..e47e437a131 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -19,9 +19,11 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import litellm from litellm.integrations.opentelemetry import ( OpenTelemetry, OpenTelemetryConfig, + OTELMetricAttributeFilter, OTELSemconvCategory, _normalize_team_metadata_keys, ) @@ -5301,6 +5303,8 @@ class TestEndProxySpanLitellmMetadataFallback(unittest.TestCase): otel._end_proxy_span_from_kwargs(kwargs, end_time=datetime.now()) mock_span.end.assert_called_once() + + class TestOpenTelemetryInferenceIdentityAttributes(unittest.TestCase): """team_metadata, http.route, and both model names (the user-facing model_group alias and the dispatched provider model) must land on the @@ -5467,3 +5471,281 @@ class TestOpenTelemetryTeamMetadataKeysConfig(unittest.TestCase): ): cfg = OpenTelemetryConfig(baggage_team_metadata_keys=["from_arg"]) assert cfg.baggage_team_metadata_keys == ["from_arg"] + + +class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): + """LIT-3600: include/exclude control over which attributes are stamped on + emitted metrics, to cap metric cardinality. These drive the real + _handle_success -> _record_metrics path through an in-memory reader and + read attributes straight off the recorded data points, so they fail if the + filtering feature is reverted and pass only when it works end to end.""" + + HERE = os.path.dirname(__file__) + POLL_INTERVAL = 0.05 + POLL_TIMEOUT = 2.0 + DURATION_METRIC = "gen_ai.client.operation.duration" + TOKEN_METRIC = "gen_ai.client.token.usage" + + # High-cardinality attributes the captured fixture emits by default. Each is + # a member of VALID_METRIC_ATTRIBUTE_NAMES and is present on the recorded + # metric when no filter is configured (verified by the backward-compat test). + HIGH_CARDINALITY_KEYS = ( + "hidden_params", + "metadata.user_api_key_hash", + "metadata.requester_ip_address", + "metadata.requester_metadata", + "metadata.applied_guardrails", + ) + RETAINED_LOW_CARDINALITY_KEY = "gen_ai.request.model" + + def _load_fixtures(self): + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json") + ) as f: + kwargs = json.load(f) + with open( + os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json") + ) as f: + response_obj = json.load(f) + return kwargs, response_obj + + def _record(self, attributes): + """Run a real success hook with metrics enabled and return the reader.""" + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(InMemorySpanExporter())) + otel = OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", enable_metrics=True, attributes=attributes + ), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + otel.tracer = tracer_provider.get_tracer(__name__) + + kwargs, response_obj = self._load_fixtures() + start = datetime.utcnow() + end = start + timedelta(seconds=1) + otel._handle_success(kwargs, response_obj, start, end) + return metric_reader + + def _keysets(self, reader, metric_name): + """Attribute-key sets, one per recorded data point of `metric_name`.""" + deadline = time.time() + self.POLL_TIMEOUT + while time.time() < deadline: + data = reader.get_metrics_data() + if data and hasattr(data, "resource_metrics"): + for rm in data.resource_metrics: + for sm in rm.scope_metrics: + for m in sm.metrics: + if m.name == metric_name: + return [ + set(dp.attributes.keys()) + for dp in m.data.data_points + ] + time.sleep(self.POLL_INTERVAL) + return None + + def test_exclude_list_strips_high_cardinality_keys_across_metrics(self): + """The bug: high-cardinality metadata/hidden_params explode metric + cardinality. With exclude_list set, none of them reach any data point, + while the retained low-cardinality model attribute survives. Asserted + on both the duration and token-usage histograms.""" + reader = self._record( + OTELMetricAttributeFilter(exclude_list=list(self.HIGH_CARDINALITY_KEYS)) + ) + excluded = set(self.HIGH_CARDINALITY_KEYS) + + for metric_name in (self.DURATION_METRIC, self.TOKEN_METRIC): + keysets = self._keysets(reader, metric_name) + self.assertTrue(keysets, f"{metric_name} was not recorded") + for keys in keysets: + self.assertTrue( + excluded.isdisjoint(keys), + f"{metric_name} leaked excluded keys: {excluded & keys}", + ) + self.assertIn(self.RETAINED_LOW_CARDINALITY_KEY, keys) + + def test_include_list_allows_only_listed_attributes(self): + """An allowlist caps emitted attributes to exactly the listed set. + gen_ai.token.type is a structural discriminator added to the token + histogram after filtering, so it is the only key permitted beyond the + allowlist, and only on that metric.""" + include = ["gen_ai.request.model", "gen_ai.system"] + reader = self._record(OTELMetricAttributeFilter(include_list=include)) + allowed = set(include) + + duration_keysets = self._keysets(reader, self.DURATION_METRIC) + self.assertTrue(duration_keysets, "duration metric was not recorded") + for keys in duration_keysets: + self.assertEqual(keys, allowed) + + token_keysets = self._keysets(reader, self.TOKEN_METRIC) + self.assertTrue(token_keysets, "token-usage metric was not recorded") + for keys in token_keysets: + self.assertEqual(keys - {"gen_ai.token.type"}, allowed) + + def test_no_filter_preserves_high_cardinality_keys(self): + """Backward compatibility: with no attributes config, every + high-cardinality key the fixture carries is still stamped on the + metric, so existing customers who rely on them are unaffected.""" + reader = self._record(None) + expected = set(self.HIGH_CARDINALITY_KEYS) + + for metric_name in (self.DURATION_METRIC, self.TOKEN_METRIC): + keysets = self._keysets(reader, metric_name) + self.assertTrue(keysets, f"{metric_name} was not recorded") + for keys in keysets: + self.assertTrue( + expected.issubset(keys), + f"{metric_name} dropped {expected - keys} by default", + ) + self.assertIn(self.RETAINED_LOW_CARDINALITY_KEY, keys) + + def test_proxy_callback_settings_attributes_applied_without_kwarg(self): + """Regression for the proxy path: the OpenTelemetry logger is constructed + before the proxy populates litellm.callback_settings['otel']['attributes'], + and without the attributes kwarg, so the filter must be resolved at record + time rather than at __init__. Otherwise metrics ship at full cardinality + (the bug the live proxy surfaced; constructing with the kwarg, or with + callback_settings already set, hid it).""" + previous = litellm.callback_settings + litellm.callback_settings = {} # not yet populated when the logger is built + try: + metric_reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[metric_reader]) + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + SimpleSpanProcessor(InMemorySpanExporter()) + ) + otel = OpenTelemetry( + config=OpenTelemetryConfig(exporter="console", enable_metrics=True), + tracer_provider=tracer_provider, + meter_provider=meter_provider, + ) + otel.tracer = tracer_provider.get_tracer(__name__) + # The proxy sets this only after the logger already exists. + litellm.callback_settings = { + "otel": { + "attributes": {"exclude_list": list(self.HIGH_CARDINALITY_KEYS)} + } + } + kwargs, response_obj = self._load_fixtures() + start = datetime.utcnow() + otel._handle_success( + kwargs, response_obj, start, start + timedelta(seconds=1) + ) + finally: + litellm.callback_settings = previous + + excluded = set(self.HIGH_CARDINALITY_KEYS) + for metric_name in (self.DURATION_METRIC, self.TOKEN_METRIC): + keysets = self._keysets(metric_reader, metric_name) + self.assertTrue(keysets, f"{metric_name} was not recorded") + for keys in keysets: + self.assertTrue( + excluded.isdisjoint(keys), + f"{metric_name} leaked {excluded & keys} via callback_settings", + ) + self.assertIn(self.RETAINED_LOW_CARDINALITY_KEY, keys) + + def test_callback_settings_validation_failure_is_not_sticky(self): + """On the lazy callback_settings path a validation failure must not cache + the bad config. Once the operator corrects + callback_settings['otel']['attributes'], the next record resolves the + fixed filter instead of re-raising the stale error until a restart.""" + previous = litellm.callback_settings + litellm.callback_settings = { + "otel": { + "attributes": { + "include_list": ["gen_ai.system"], + "exclude_list": ["hidden_params"], + } + } + } + try: + otel = OpenTelemetry(config=OpenTelemetryConfig(exporter="console")) + attrs = {"gen_ai.system": "openai", "hidden_params": "{}"} + + with self.assertRaises(ValueError): + otel._filter_metric_attributes(attrs) + + litellm.callback_settings = { + "otel": {"attributes": {"exclude_list": ["hidden_params"]}} + } + filtered = otel._filter_metric_attributes(attrs) + finally: + litellm.callback_settings = previous + + self.assertEqual(filtered, {"gen_ai.system": "openai"}) + + def test_include_and_exclude_together_raise_value_error(self): + with self.assertRaises(ValueError): + OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", + attributes=OTELMetricAttributeFilter( + include_list=["gen_ai.system"], + exclude_list=["hidden_params"], + ), + ) + ) + + def test_unknown_include_name_raises_value_error(self): + with self.assertRaises(ValueError): + OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", + attributes=OTELMetricAttributeFilter( + include_list=["not.a.real.attribute"] + ), + ) + ) + + def test_unknown_exclude_name_raises_value_error(self): + with self.assertRaises(ValueError): + OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", + attributes=OTELMetricAttributeFilter( + exclude_list=["metadata.does_not_exist"] + ), + ) + ) + + def test_dict_attributes_kwarg_path_validates(self): + """The YAML/kwargs entry point (a plain dict) flows through + _build_metric_attribute_filter and hits the same validation.""" + with self.assertRaises(ValueError): + OpenTelemetry( + attributes={ + "include_list": ["gen_ai.system"], + "exclude_list": ["hidden_params"], + } + ) + + def test_no_filter_returns_attrs_object_unchanged(self): + """The no-config path is a hot-path no-op: it returns the same dict + object, so default emission pays zero copy cost. Locking identity makes + a future refactor that always copies/filters trip here.""" + otel = OpenTelemetry(config=OpenTelemetryConfig(exporter="console")) + attrs = {"gen_ai.request.model": "m", "hidden_params": "{}"} + self.assertIs(otel._filter_metric_attributes(attrs), attrs) + + def test_token_type_discriminator_rejected_from_either_list(self): + """gen_ai.token.type is a structural discriminator stamped onto the + input/output token series after filtering; it cannot be filtered without + collapsing the two series into one. Listing it in include_list or + exclude_list is rejected loudly at startup rather than silently ignored, + so an operator gets an error instead of a no-op.""" + for attributes in ( + OTELMetricAttributeFilter(exclude_list=["gen_ai.token.type"]), + OTELMetricAttributeFilter(include_list=["gen_ai.token.type"]), + ): + with self.assertRaises(ValueError): + OpenTelemetry( + config=OpenTelemetryConfig( + exporter="console", attributes=attributes + ) + )