scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

feat(prometheus): expose video duration and image count consumption metrics (#33138)

Browse source
This commit is contained in:
yucheng-berri 2026-07-13 18:51:13 -07:00 • committed by GitHub
parent 397c84678b
commit 07ea4b3e14
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 336 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
litellm/integrations/prometheus.py
View file

@ -239,6 +239,18 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_output_audio_tokens_metric"),
)
self.litellm_video_duration_seconds_metric = self._counter_factory(
"litellm_video_duration_seconds_metric",
"Seconds of video generated, from usage.duration_seconds on video generation calls",
labelnames=self.get_labels_for_metric("litellm_video_duration_seconds_metric"),
)
self.litellm_images_generated_metric = self._counter_factory(
"litellm_images_generated_metric",
"Number of images generated, from the image generation response",
labelnames=self.get_labels_for_metric("litellm_images_generated_metric"),
)
# Remaining Budget for Team
self.litellm_remaining_team_budget_metric = self._gauge_factory(
"litellm_remaining_team_budget_metric",
@ -1336,6 +1348,12 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
self._increment_media_generation_metrics(
standard_logging_payload=standard_logging_payload,
enum_values=enum_values,
label_context=label_context,
)
# MCP tool call metrics
self._increment_mcp_tool_call_metrics(
standard_logging_payload=standard_logging_payload,
@ -1459,8 +1477,65 @@ class PrometheusLogger(CustomLogger):
),
]
for counter, metric_name, value in detail_metrics:
if not isinstance(value, (int, float)) or value <= 0:
PrometheusLogger._inc_sparse_usage_counters(
self,
detail_metrics,
enum_values=enum_values,
label_context=label_context,
)
def _increment_media_generation_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
label_context: PrometheusLabelFactoryContext | None = None,
) -> None:
"""
Increment video-seconds and images-generated counters from
``standard_logging_payload["metadata"]["usage_object"]``. Video
providers report ``duration_seconds`` there; image generation calls
report ``output_image_count``. Both are sparse: only emitted when the
value is present and > 0, so token-only call types are unaffected.
"""
metadata = standard_logging_payload.get("metadata") or {}
usage_object = metadata.get("usage_object") if isinstance(metadata, dict) else None
if not isinstance(usage_object, dict):
return
media_metrics: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]] = [
(
self.litellm_video_duration_seconds_metric,
"litellm_video_duration_seconds_metric",
usage_object.get("duration_seconds"),
),
(
self.litellm_images_generated_metric,
"litellm_images_generated_metric",
usage_object.get("output_image_count"),
),
]
PrometheusLogger._inc_sparse_usage_counters(
self,
media_metrics,
enum_values=enum_values,
label_context=label_context,
)
def _inc_sparse_usage_counters(
self,
counters_with_values: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]],
enum_values: UserAPIKeyLabelValues,
label_context: PrometheusLabelFactoryContext | None = None,
) -> None:
"""
Increment each ``(counter, metric_name, value)`` entry whose value is
a positive number. Non-numeric values (including booleans from
malformed provider usage dicts) and values <= 0 are skipped, keeping
scrape output sparse.
"""
for counter, metric_name, value in counters_with_values:
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
continue
PrometheusLogger._inc_labeled_counter(
self,

7
litellm/litellm_core_utils/litellm_logging.py
View file

@ -5212,10 +5212,15 @@ def get_standard_logging_object_payload(
call_type = kwargs.get("call_type")
cache_hit = kwargs.get("cache_hit", False)
# Extract usage as a plain dict, avoiding Pydantic round-trip
usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict(
raw_usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=response_obj,
combined_usage_object=cast(Optional[Usage], kwargs.get("combined_usage_object")),
)
usage_dict = (
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
if isinstance(init_response_obj, ImageResponse) and init_response_obj.data
else raw_usage_dict
)
id = response_obj.get("id", kwargs.get("litellm_call_id"))

7
litellm/types/integrations/prometheus.py
View file

@ -213,6 +213,8 @@ DEFINED_PROMETHEUS_METRICS = Literal[
"litellm_input_audio_tokens_metric",
"litellm_output_reasoning_tokens_metric",
"litellm_output_audio_tokens_metric",
"litellm_video_duration_seconds_metric",
"litellm_images_generated_metric",
"litellm_deployment_successful_fallbacks",
"litellm_deployment_failed_fallbacks",
"litellm_remaining_team_budget_metric",
@ -506,6 +508,9 @@ class PrometheusMetricLabels:
litellm_output_reasoning_tokens_metric = litellm_output_tokens_metric
litellm_output_audio_tokens_metric = litellm_output_tokens_metric
litellm_video_duration_seconds_metric = litellm_output_tokens_metric
litellm_images_generated_metric = litellm_output_tokens_metric
litellm_deployment_state = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
@ -717,6 +722,8 @@ class PrometheusMetricLabels:
"litellm_input_tokens_metric",
"litellm_total_tokens_metric",
"litellm_output_tokens_metric",
"litellm_video_duration_seconds_metric",
"litellm_images_generated_metric",
}
)
# Managed batch metrics

180
tests/test_litellm/integrations/test_prometheus_media_generation_metrics.py Normal file
View file

@ -0,0 +1,180 @@
"""
Unit tests for the video-seconds and images-generated Prometheus counters (LIT-4254).
Video providers report ``duration_seconds`` inside the usage object that lands
on ``standard_logging_payload["metadata"]["usage_object"]``; image generation
calls report ``output_image_count`` there. Both counters are sparse: only
incremented when the value is present and > 0.
"""
from typing import get_args
from unittest.mock import MagicMock
import pytest
from litellm.integrations.prometheus import PrometheusLogger
from litellm.types.integrations.prometheus import (
DEFINED_PROMETHEUS_METRICS,
PrometheusMetricLabels,
UserAPIKeyLabelValues,
)
MEDIA_GENERATION_METRICS = [
"litellm_video_duration_seconds_metric",
"litellm_images_generated_metric",
]
@pytest.fixture
def sample_enum_values():
return UserAPIKeyLabelValues(
end_user="test-end-user",
hashed_api_key="test-key-hash",
api_key_alias="test-key-alias",
team="test-team",
team_alias="test-team-alias",
user="test-user",
model="sora-2",
)
def _make_mock_logger():
logger = MagicMock()
for name in MEDIA_GENERATION_METRICS:
setattr(logger, name, MagicMock())
logger.get_labels_for_metric = MagicMock(
return_value=[
"model",
"hashed_api_key",
"api_key_alias",
"team",
"team_alias",
"end_user",
"user",
]
)
return logger
class TestMediaGenerationMetricsRegistration:
def test_metrics_in_defined_prometheus_metrics(self):
defined = get_args(DEFINED_PROMETHEUS_METRICS)
for name in MEDIA_GENERATION_METRICS:
assert name in defined, f"{name} missing from DEFINED_PROMETHEUS_METRICS"
def test_metric_labels_defined(self):
for name in MEDIA_GENERATION_METRICS:
assert hasattr(PrometheusMetricLabels, name), f"{name} missing from PrometheusMetricLabels"
def test_metrics_share_output_token_label_set(self):
assert (
PrometheusMetricLabels.litellm_video_duration_seconds_metric
== PrometheusMetricLabels.litellm_output_tokens_metric
)
assert (
PrometheusMetricLabels.litellm_images_generated_metric
== PrometheusMetricLabels.litellm_output_tokens_metric
)
def test_runtime_label_set_matches_output_tokens_metric(self):
"""Full parity with litellm_output_tokens_metric, including the org labels
appended via _org_label_metrics, so existing token dashboards can be cloned."""
expected = PrometheusMetricLabels.get_labels("litellm_output_tokens_metric")
for name in MEDIA_GENERATION_METRICS:
assert PrometheusMetricLabels.get_labels(name) == expected
class TestIncrementMediaGenerationMetrics:
def test_video_duration_incremented(self, sample_enum_values):
logger = _make_mock_logger()
payload = {"metadata": {"usage_object": {"duration_seconds": 8.0}}}
PrometheusLogger._increment_media_generation_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
logger.litellm_video_duration_seconds_metric.labels().inc.assert_called_once_with(8.0)
logger.litellm_images_generated_metric.labels.assert_not_called()
def test_image_count_incremented(self, sample_enum_values):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens": 18,
"completion_tokens": 391,
"total_tokens": 409,
"output_image_count": 2,
}
}
}
PrometheusLogger._increment_media_generation_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
logger.litellm_images_generated_metric.labels().inc.assert_called_once_with(2.0)
logger.litellm_video_duration_seconds_metric.labels.assert_not_called()
def test_token_only_usage_is_a_noop(self, sample_enum_values):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30,
}
}
}
PrometheusLogger._increment_media_generation_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
for name in MEDIA_GENERATION_METRICS:
getattr(logger, name).labels.assert_not_called()
@pytest.mark.parametrize("bad_value", [0, 0.0, None, -4.0, "4", True])
def test_non_positive_or_non_numeric_values_are_ignored(self, sample_enum_values, bad_value):
logger = _make_mock_logger()
payload = {
"metadata": {
"usage_object": {
"duration_seconds": bad_value,
"output_image_count": bad_value,
}
}
}
PrometheusLogger._increment_media_generation_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
for name in MEDIA_GENERATION_METRICS:
getattr(logger, name).labels.assert_not_called()
def test_missing_usage_object_is_a_noop(self, sample_enum_values):
logger = _make_mock_logger()
for payload in ({"metadata": {}}, {"metadata": None}, {"metadata": {"usage_object": "redacted"}}):
PrometheusLogger._increment_media_generation_metrics(
logger,
standard_logging_payload=payload,
enum_values=sample_enum_values,
)
for name in MEDIA_GENERATION_METRICS:
getattr(logger, name).labels.assert_not_called()
if __name__ == "__main__":
pytest.main([__file__, "-v"])

66
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -3707,3 +3707,69 @@ def test_set_cost_breakdown_stores_reasoning_cost():
cost_for_built_in_tools_cost_usd_dollar=0.0,
)
assert "reasoning_cost" not in no_reasoning.cost_breakdown
def _build_payload_for_media_response(logging_obj, init_response_obj, kwargs=None):
import datetime
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload,
)
now = datetime.datetime.now()
return get_standard_logging_object_payload(
kwargs=kwargs or {"litellm_call_id": "media-call-id", "model": "test-model", "messages": []},
init_response_obj=init_response_obj,
start_time=now,
end_time=now,
logging_obj=logging_obj,
status="success",
)
def test_image_response_sets_output_image_count_on_usage_object(logging_obj):
"""Generated-image count must land on metadata.usage_object for callbacks (e.g. Prometheus)."""
from litellm.types.utils import ImageResponse
response = ImageResponse(created=1, data=[{"url": "https://img/1"}, {"url": "https://img/2"}])
payload = _build_payload_for_media_response(logging_obj, response)
assert payload is not None
assert payload["metadata"]["usage_object"]["output_image_count"] == 2
def test_output_image_count_survives_message_redaction(logging_obj, monkeypatch):
"""Redaction replaces the ImageResponse body, so the count must be captured pre-redaction."""
import litellm
from litellm.types.utils import ImageResponse
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
response = ImageResponse(created=1, data=[{"url": "https://img/1"}])
payload = _build_payload_for_media_response(logging_obj, response)
assert payload is not None
assert payload["response"] == {"text": "redacted-by-litellm"}
assert payload["metadata"]["usage_object"]["output_image_count"] == 1
def test_non_image_response_has_no_output_image_count(logging_obj):
payload = _build_payload_for_media_response(
logging_obj, {"id": "chatcmpl-1", "usage": {"prompt_tokens": 1, "completion_tokens": 2}}
)
assert payload is not None
assert "output_image_count" not in payload["metadata"]["usage_object"]
def test_zero_token_video_usage_preserves_duration_seconds(logging_obj):
"""Video usage bills by duration; the payload must keep duration_seconds even with zero tokens."""
payload = _build_payload_for_media_response(
logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}}
)
assert payload is not None
assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0
assert payload["total_tokens"] == 0
assert payload["completion_tokens"] == 0