diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a9bd93e6420..2aa7d041f20 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -190,6 +190,32 @@ METRIC_METADATA_KEYS: Final[tuple[str, ...]] = ( "vector_store_request_metadata", ) +# The only ``hidden_params`` field that becomes part of the ``hidden_params`` label. +# The object as a whole is per-request by construction -- ``response_cost``, +# ``litellm_overhead_time_ms``, ``cache_key``, ``usage_object`` and the provider's +# ``additional_headers`` rate-limit counters all move on every call -- so dumping it +# whole made one series per request out of every instrument. +# +# ``model_id`` is the router's own deployment id, so it is bounded by the deployment +# list and is what a per-deployment panel joins on. ``api_base`` is deliberately NOT +# here even though it names the same thing: it is a documented per-call parameter, so +# in SDK use it is chosen by the caller rather than provisioned by the operator, and a +# caller varying it would put the per-request cardinality straight back. +BOUNDED_HIDDEN_PARAM_KEYS: Final[tuple[str, ...]] = ("model_id",) + + +def bounded_hidden_params_label(std_log: object) -> str | None: + hidden_params: Final = getattr(std_log, "hidden_params", None) or ( + std_log.get("hidden_params") if isinstance(std_log, Mapping) else None + ) + bounded: Final = { + key: hidden_params[key] + for key in BOUNDED_HIDDEN_PARAM_KEYS + if isinstance(hidden_params, Mapping) and hidden_params.get(key) is not None + } + return safe_dumps(bounded) if bounded else None + + TOKEN_TYPE_ATTRIBUTE: Final[str] = "gen_ai.token.type" VALID_METRIC_ATTRIBUTE_NAMES: Final[frozenset[str]] = frozenset( @@ -1726,10 +1752,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): else: common_attrs[f"metadata.{key}"] = str(value) - # get hidden params - hidden_params: Final = getattr(std_log, "hidden_params", None) or (std_log or {}).get("hidden_params", {}) - if hidden_params: - common_attrs["hidden_params"] = safe_dumps(hidden_params) + if (hidden_params_label := bounded_hidden_params_label(std_log)) is not None: + common_attrs["hidden_params"] = hidden_params_label common_attrs = self._filter_metric_attributes(common_attrs) diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 1197ae055d0..f92a01792e3 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -21,6 +21,7 @@ from litellm._logging import verbose_logger from litellm.integrations.opentelemetry import ( METRIC_METADATA_KEYS, TOKEN_TYPE_ATTRIBUTE, + bounded_hidden_params_label, build_metric_attribute_filter, resolve_metric_attribute_filter, ) @@ -139,19 +140,6 @@ METRIC_ATTRIBUTE_CEILING: Final[frozenset[str]] = frozenset( ) ) -# The only ``hidden_params`` field that becomes part of the ``hidden_params`` label. -# The object as a whole is per-request by construction -- ``response_cost``, -# ``litellm_overhead_time_ms``, ``cache_key``, ``usage_object`` and the provider's -# ``additional_headers`` rate-limit counters all move on every call -- so dumping it -# whole made one series per request out of every instrument. -# -# ``model_id`` is the router's own deployment id, so it is bounded by the deployment -# list and is what a per-deployment panel joins on. ``api_base`` is deliberately NOT -# here even though it names the same thing: it is a documented per-call parameter, so -# in SDK use it is chosen by the caller rather than provisioned by the operator, and a -# caller varying it would put the per-request cardinality straight back. -BOUNDED_HIDDEN_PARAM_KEYS: Final[tuple[str, ...]] = ("model_id",) - class _TokenUsage(TypedDict, total=False): """The token counts a response's ``usage`` carries, as the recorder reads them.""" @@ -300,14 +288,8 @@ class GenAIMetricRecorder: else: common_attrs[f"metadata.{key}"] = str(value) - hidden_params: Final = getattr(std_log, "hidden_params", None) or (std_log or {}).get("hidden_params", {}) - bounded_hidden_params: Final = { - key: hidden_params[key] - for key in BOUNDED_HIDDEN_PARAM_KEYS - if isinstance(hidden_params, Mapping) and hidden_params.get(key) is not None - } - if bounded_hidden_params: - common_attrs["hidden_params"] = safe_dumps(bounded_hidden_params) + if (hidden_params_label := bounded_hidden_params_label(std_log)) is not None: + common_attrs["hidden_params"] = hidden_params_label return common_attrs diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 5f749e951b1..c82c3cea149 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -6232,9 +6232,11 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): DURATION_METRIC = "gen_ai.client.operation.duration" TOKEN_METRIC = "gen_ai.client.token.usage" - # High-cardinality attributes the captured fixture emits by default. Each is - # a member of VALID_METRIC_ATTRIBUTE_NAMES and is present on the recorded - # metric when no filter is configured (verified by the backward-compat test). + # Attributes the captured fixture emits by default and an operator may want + # off metrics. Each is a member of VALID_METRIC_ATTRIBUTE_NAMES and is present + # on the recorded metric when no filter is configured (verified by the + # backward-compat test). The metadata ones vary per caller or per request; + # hidden_params carries only model_id, so it varies per deployment. HIGH_CARDINALITY_KEYS = ( "hidden_params", "metadata.user_api_key_hash", @@ -6333,9 +6335,9 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): self.assertEqual(keys - {"gen_ai.token.type"}, allowed) def test_no_filter_preserves_high_cardinality_keys(self): - """Backward compatibility: with no attributes config, every - high-cardinality key the fixture carries is still stamped on the - metric, so existing customers who rely on them are unaffected.""" + """Backward compatibility: with no attributes config, every key in + HIGH_CARDINALITY_KEYS the fixture carries is still stamped on the metric, + so existing customers who rely on them are unaffected.""" reader = self._record(None) expected = set(self.HIGH_CARDINALITY_KEYS) @@ -6396,6 +6398,66 @@ class TestOpenTelemetryMetricAttributeFiltering(unittest.TestCase): ) self.assertIn(self.RETAINED_LOW_CARDINALITY_KEY, keys) + def _duration_points(self, *mutations): + """Run one success hook per mutation of the fixture kwargs and return the + duration histogram's data points.""" + metric_reader = InMemoryMetricReader() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(InMemorySpanExporter())) + otel = OpenTelemetry( + config=OpenTelemetryConfig(exporter="console", enable_metrics=True), + tracer_provider=tracer_provider, + meter_provider=MeterProvider(metric_readers=[metric_reader]), + ) + otel.tracer = tracer_provider.get_tracer(__name__) + + start = datetime.utcnow() + for mutate in mutations: + kwargs, response_obj = self._load_fixtures() + mutate(kwargs) + otel._handle_success(kwargs, response_obj, start, start + timedelta(seconds=1)) + + return [ + dp + for rm in metric_reader.get_metrics_data().resource_metrics + for sm in rm.scope_metrics + for m in sm.metrics + if m.name == self.DURATION_METRIC + for dp in m.data.data_points + ] + + def test_hidden_params_label_is_one_series_across_calls_with_different_cost(self): + """Two calls differing only in per-request cost must land in one series + whose hidden_params label carries the deployment's model_id alone.""" + + def with_cost(cost): + def mutate(kwargs): + kwargs["response_cost"] = cost + kwargs["standard_logging_object"]["hidden_params"]["response_cost"] = cost + + return mutate + + data_points = self._duration_points(with_cost(0.001), with_cost(0.002)) + + model_id = self._load_fixtures()[0]["standard_logging_object"]["hidden_params"]["model_id"] + self.assertEqual(len(data_points), 1) + self.assertEqual(data_points[0].count, 2) + self.assertEqual(json.loads(data_points[0].attributes["hidden_params"]), {"model_id": model_id}) + + def test_hidden_params_label_is_omitted_without_model_id(self): + """A call with no model_id still records the metric, just without the + hidden_params label.""" + + def drop_model_id(kwargs): + del kwargs["standard_logging_object"]["hidden_params"]["model_id"] + + data_points = self._duration_points(drop_model_id) + + self.assertEqual(len(data_points), 1) + self.assertEqual(data_points[0].count, 1) + self.assertNotIn("hidden_params", data_points[0].attributes) + self.assertIn(self.RETAINED_LOW_CARDINALITY_KEY, data_points[0].attributes) + def test_callback_settings_validation_failure_is_not_sticky(self): """On the lazy callback_settings path a validation failure must not cache the bad config. Once the operator corrects