From 07ea4b3e145314a6670ec57c32d319bbcbaecd90 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 13 Jul 2026 18:51:13 -0700 Subject: [PATCH] feat(prometheus): expose video duration and image count consumption metrics (#33138) --- litellm/integrations/prometheus.py | 79 +++++++- litellm/litellm_core_utils/litellm_logging.py | 7 +- litellm/types/integrations/prometheus.py | 7 + ...est_prometheus_media_generation_metrics.py | 180 ++++++++++++++++++ .../test_litellm_logging.py | 66 +++++++ 5 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_media_generation_metrics.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 743287c52e4..64d4dd578b2 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -239,6 +239,18 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_output_audio_tokens_metric"), ) + self.litellm_video_duration_seconds_metric = self._counter_factory( + "litellm_video_duration_seconds_metric", + "Seconds of video generated, from usage.duration_seconds on video generation calls", + labelnames=self.get_labels_for_metric("litellm_video_duration_seconds_metric"), + ) + + self.litellm_images_generated_metric = self._counter_factory( + "litellm_images_generated_metric", + "Number of images generated, from the image generation response", + labelnames=self.get_labels_for_metric("litellm_images_generated_metric"), + ) + # Remaining Budget for Team self.litellm_remaining_team_budget_metric = self._gauge_factory( "litellm_remaining_team_budget_metric", @@ -1336,6 +1348,12 @@ class PrometheusLogger(CustomLogger): label_context=label_context, ) + self._increment_media_generation_metrics( + standard_logging_payload=standard_logging_payload, + enum_values=enum_values, + label_context=label_context, + ) + # MCP tool call metrics self._increment_mcp_tool_call_metrics( standard_logging_payload=standard_logging_payload, @@ -1459,8 +1477,65 @@ class PrometheusLogger(CustomLogger): ), ] - for counter, metric_name, value in detail_metrics: - if not isinstance(value, (int, float)) or value <= 0: + PrometheusLogger._inc_sparse_usage_counters( + self, + detail_metrics, + enum_values=enum_values, + label_context=label_context, + ) + + def _increment_media_generation_metrics( + self, + standard_logging_payload: StandardLoggingPayload, + enum_values: UserAPIKeyLabelValues, + label_context: PrometheusLabelFactoryContext | None = None, + ) -> None: + """ + Increment video-seconds and images-generated counters from + ``standard_logging_payload["metadata"]["usage_object"]``. Video + providers report ``duration_seconds`` there; image generation calls + report ``output_image_count``. Both are sparse: only emitted when the + value is present and > 0, so token-only call types are unaffected. + """ + metadata = standard_logging_payload.get("metadata") or {} + usage_object = metadata.get("usage_object") if isinstance(metadata, dict) else None + if not isinstance(usage_object, dict): + return + + media_metrics: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]] = [ + ( + self.litellm_video_duration_seconds_metric, + "litellm_video_duration_seconds_metric", + usage_object.get("duration_seconds"), + ), + ( + self.litellm_images_generated_metric, + "litellm_images_generated_metric", + usage_object.get("output_image_count"), + ), + ] + + PrometheusLogger._inc_sparse_usage_counters( + self, + media_metrics, + enum_values=enum_values, + label_context=label_context, + ) + + def _inc_sparse_usage_counters( + self, + counters_with_values: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]], + enum_values: UserAPIKeyLabelValues, + label_context: PrometheusLabelFactoryContext | None = None, + ) -> None: + """ + Increment each ``(counter, metric_name, value)`` entry whose value is + a positive number. Non-numeric values (including booleans from + malformed provider usage dicts) and values <= 0 are skipped, keeping + scrape output sparse. + """ + for counter, metric_name, value in counters_with_values: + if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0: continue PrometheusLogger._inc_labeled_counter( self, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index fc04c65449a..461ab62b815 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5212,10 +5212,15 @@ def get_standard_logging_object_payload( call_type = kwargs.get("call_type") cache_hit = kwargs.get("cache_hit", False) # Extract usage as a plain dict, avoiding Pydantic round-trip - usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict( + raw_usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict( response_obj=response_obj, combined_usage_object=cast(Optional[Usage], kwargs.get("combined_usage_object")), ) + usage_dict = ( + {**raw_usage_dict, "output_image_count": len(init_response_obj.data)} + if isinstance(init_response_obj, ImageResponse) and init_response_obj.data + else raw_usage_dict + ) id = response_obj.get("id", kwargs.get("litellm_call_id")) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index 69bccff701f..318ba1f5956 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -213,6 +213,8 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_input_audio_tokens_metric", "litellm_output_reasoning_tokens_metric", "litellm_output_audio_tokens_metric", + "litellm_video_duration_seconds_metric", + "litellm_images_generated_metric", "litellm_deployment_successful_fallbacks", "litellm_deployment_failed_fallbacks", "litellm_remaining_team_budget_metric", @@ -506,6 +508,9 @@ class PrometheusMetricLabels: litellm_output_reasoning_tokens_metric = litellm_output_tokens_metric litellm_output_audio_tokens_metric = litellm_output_tokens_metric + litellm_video_duration_seconds_metric = litellm_output_tokens_metric + litellm_images_generated_metric = litellm_output_tokens_metric + litellm_deployment_state = [ UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, @@ -717,6 +722,8 @@ class PrometheusMetricLabels: "litellm_input_tokens_metric", "litellm_total_tokens_metric", "litellm_output_tokens_metric", + "litellm_video_duration_seconds_metric", + "litellm_images_generated_metric", } ) # Managed batch metrics diff --git a/tests/test_litellm/integrations/test_prometheus_media_generation_metrics.py b/tests/test_litellm/integrations/test_prometheus_media_generation_metrics.py new file mode 100644 index 00000000000..2ce92f08ef0 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_media_generation_metrics.py @@ -0,0 +1,180 @@ +""" +Unit tests for the video-seconds and images-generated Prometheus counters (LIT-4254). + +Video providers report ``duration_seconds`` inside the usage object that lands +on ``standard_logging_payload["metadata"]["usage_object"]``; image generation +calls report ``output_image_count`` there. Both counters are sparse: only +incremented when the value is present and > 0. +""" + +from typing import get_args +from unittest.mock import MagicMock + +import pytest + +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.integrations.prometheus import ( + DEFINED_PROMETHEUS_METRICS, + PrometheusMetricLabels, + UserAPIKeyLabelValues, +) + +MEDIA_GENERATION_METRICS = [ + "litellm_video_duration_seconds_metric", + "litellm_images_generated_metric", +] + + +@pytest.fixture +def sample_enum_values(): + return UserAPIKeyLabelValues( + end_user="test-end-user", + hashed_api_key="test-key-hash", + api_key_alias="test-key-alias", + team="test-team", + team_alias="test-team-alias", + user="test-user", + model="sora-2", + ) + + +def _make_mock_logger(): + logger = MagicMock() + for name in MEDIA_GENERATION_METRICS: + setattr(logger, name, MagicMock()) + logger.get_labels_for_metric = MagicMock( + return_value=[ + "model", + "hashed_api_key", + "api_key_alias", + "team", + "team_alias", + "end_user", + "user", + ] + ) + return logger + + +class TestMediaGenerationMetricsRegistration: + def test_metrics_in_defined_prometheus_metrics(self): + defined = get_args(DEFINED_PROMETHEUS_METRICS) + for name in MEDIA_GENERATION_METRICS: + assert name in defined, f"{name} missing from DEFINED_PROMETHEUS_METRICS" + + def test_metric_labels_defined(self): + for name in MEDIA_GENERATION_METRICS: + assert hasattr(PrometheusMetricLabels, name), f"{name} missing from PrometheusMetricLabels" + + def test_metrics_share_output_token_label_set(self): + assert ( + PrometheusMetricLabels.litellm_video_duration_seconds_metric + == PrometheusMetricLabels.litellm_output_tokens_metric + ) + assert ( + PrometheusMetricLabels.litellm_images_generated_metric + == PrometheusMetricLabels.litellm_output_tokens_metric + ) + + def test_runtime_label_set_matches_output_tokens_metric(self): + """Full parity with litellm_output_tokens_metric, including the org labels + appended via _org_label_metrics, so existing token dashboards can be cloned.""" + expected = PrometheusMetricLabels.get_labels("litellm_output_tokens_metric") + for name in MEDIA_GENERATION_METRICS: + assert PrometheusMetricLabels.get_labels(name) == expected + + +class TestIncrementMediaGenerationMetrics: + def test_video_duration_incremented(self, sample_enum_values): + logger = _make_mock_logger() + payload = {"metadata": {"usage_object": {"duration_seconds": 8.0}}} + + PrometheusLogger._increment_media_generation_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + logger.litellm_video_duration_seconds_metric.labels().inc.assert_called_once_with(8.0) + logger.litellm_images_generated_metric.labels.assert_not_called() + + def test_image_count_incremented(self, sample_enum_values): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens": 18, + "completion_tokens": 391, + "total_tokens": 409, + "output_image_count": 2, + } + } + } + + PrometheusLogger._increment_media_generation_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + logger.litellm_images_generated_metric.labels().inc.assert_called_once_with(2.0) + logger.litellm_video_duration_seconds_metric.labels.assert_not_called() + + def test_token_only_usage_is_a_noop(self, sample_enum_values): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + } + } + } + + PrometheusLogger._increment_media_generation_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + for name in MEDIA_GENERATION_METRICS: + getattr(logger, name).labels.assert_not_called() + + @pytest.mark.parametrize("bad_value", [0, 0.0, None, -4.0, "4", True]) + def test_non_positive_or_non_numeric_values_are_ignored(self, sample_enum_values, bad_value): + logger = _make_mock_logger() + payload = { + "metadata": { + "usage_object": { + "duration_seconds": bad_value, + "output_image_count": bad_value, + } + } + } + + PrometheusLogger._increment_media_generation_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + for name in MEDIA_GENERATION_METRICS: + getattr(logger, name).labels.assert_not_called() + + def test_missing_usage_object_is_a_noop(self, sample_enum_values): + logger = _make_mock_logger() + + for payload in ({"metadata": {}}, {"metadata": None}, {"metadata": {"usage_object": "redacted"}}): + PrometheusLogger._increment_media_generation_metrics( + logger, + standard_logging_payload=payload, + enum_values=sample_enum_values, + ) + + for name in MEDIA_GENERATION_METRICS: + getattr(logger, name).labels.assert_not_called() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 0523ed7ecb1..ade2c677745 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -3707,3 +3707,69 @@ def test_set_cost_breakdown_stores_reasoning_cost(): cost_for_built_in_tools_cost_usd_dollar=0.0, ) assert "reasoning_cost" not in no_reasoning.cost_breakdown + + +def _build_payload_for_media_response(logging_obj, init_response_obj, kwargs=None): + import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, + ) + + now = datetime.datetime.now() + return get_standard_logging_object_payload( + kwargs=kwargs or {"litellm_call_id": "media-call-id", "model": "test-model", "messages": []}, + init_response_obj=init_response_obj, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + +def test_image_response_sets_output_image_count_on_usage_object(logging_obj): + """Generated-image count must land on metadata.usage_object for callbacks (e.g. Prometheus).""" + from litellm.types.utils import ImageResponse + + response = ImageResponse(created=1, data=[{"url": "https://img/1"}, {"url": "https://img/2"}]) + + payload = _build_payload_for_media_response(logging_obj, response) + + assert payload is not None + assert payload["metadata"]["usage_object"]["output_image_count"] == 2 + + +def test_output_image_count_survives_message_redaction(logging_obj, monkeypatch): + """Redaction replaces the ImageResponse body, so the count must be captured pre-redaction.""" + import litellm + from litellm.types.utils import ImageResponse + + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + response = ImageResponse(created=1, data=[{"url": "https://img/1"}]) + + payload = _build_payload_for_media_response(logging_obj, response) + + assert payload is not None + assert payload["response"] == {"text": "redacted-by-litellm"} + assert payload["metadata"]["usage_object"]["output_image_count"] == 1 + + +def test_non_image_response_has_no_output_image_count(logging_obj): + payload = _build_payload_for_media_response( + logging_obj, {"id": "chatcmpl-1", "usage": {"prompt_tokens": 1, "completion_tokens": 2}} + ) + + assert payload is not None + assert "output_image_count" not in payload["metadata"]["usage_object"] + + +def test_zero_token_video_usage_preserves_duration_seconds(logging_obj): + """Video usage bills by duration; the payload must keep duration_seconds even with zero tokens.""" + payload = _build_payload_for_media_response( + logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}} + ) + + assert payload is not None + assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0 + assert payload["total_tokens"] == 0 + assert payload["completion_tokens"] == 0