fix(prometheus): add model_group label to deployment request and rate limit metrics (#42966)

* fix(prometheus): add model_group label to deployment request and rate limit metrics

litellm_deployment_total_requests, litellm_deployment_success_responses,
litellm_deployment_failure_responses, litellm_deployment_tpm_limit and
litellm_deployment_rpm_limit had no way to identify which model_group a
pooled deployment belongs to, only requested_model, litellm_model_name and
model_id, none of which name the alias a model_name resolves through when
it fans out to more than one deployment.

model_group was already resolved onto enum_values for every request in
async_log_success_event, so this is a label-list addition for the metrics
built directly from that enum_values (the two request counters). The
failure counter builds its own UserAPIKeyLabelValues locally and had a
model_group variable already in scope that it never passed through, and
the tpm/rpm limit gauges are set from a helper that took no model_group
parameter at all even though its only caller already had it on
enum_values. Both now thread the value through.

* test(prometheus): expect model_group in deployment success/total request labels

test_set_llm_deployment_success_metrics asserts the exact label set passed
to litellm_deployment_success_responses.labels() and
litellm_deployment_total_requests.labels(), which now includes model_group
since it was added to those metrics' label list.

* fix(prometheus): bound model_group on deployment failure metrics

On a pre-routing reject (no deployment selected), model_group is
caller-supplied via litellm_params.metadata and was passed through
unbounded, letting an unrecognized value mint unlimited label series on
litellm_deployment_failure_responses / litellm_deployment_total_requests.
Bound it with the same _bounded_requested_model_label used for
requested_model on this path. When a deployment is actually selected,
model_group is router-resolved and passed through as-is.

Also documents the model_group parameter on
_set_deployment_tpm_rpm_limit_metrics and the bounding behavior on
set_llm_deployment_failure_metrics.

---------

Co-authored-by: ahamedshaik16 <24526479+ahamedshaik16@users.noreply.github.com>

litellm/integrations/prometheus.py

@@ -2778,6 +2778,13 @@ class PrometheusLogger(CustomLogger):
         - increment deployment failure responses metric
         - increment deployment total requests metric
 
+        Both counters also carry a model_group label. When a deployment was
+        actually selected, model_group is the router-resolved value and is
+        trusted as-is. On a pre-routing reject (no deployment selected), it
+        is caller-supplied via litellm_params.metadata and is bounded with
+        _bounded_requested_model_label the same way requested_model is, so an
+        unrecognized value cannot mint unbounded label series.
+
         Args:
             request_kwargs: dict
 
@@ -2844,6 +2851,7 @@ class PrometheusLogger(CustomLogger):
                 label_api_base = api_base
                 label_api_provider = llm_provider
                 label_requested_model = model_group or litellm_model_name
+                label_model_group = model_group
             else:
                 label_litellm_model_name = ""
                 label_model_id = ""
@@ -2852,6 +2860,7 @@ class PrometheusLogger(CustomLogger):
                 label_requested_model = (
                     _bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or ""
                 )
+                label_model_group = _bounded_requested_model_label(model_group, router_originated=True)
 
             enum_values: Final = UserAPIKeyLabelValues(
                 litellm_model_name=label_litellm_model_name,
@@ -2861,6 +2870,7 @@ class PrometheusLogger(CustomLogger):
                 exception_status=exception_status,
                 exception_class=(self._get_exception_class_name(exception) if exception else None),
                 requested_model=label_requested_model,
+                model_group=label_model_group,
                 hashed_api_key=hashed_api_key,
                 api_key_alias=api_key_alias,
                 user_email=user_email,
@@ -2912,9 +2922,21 @@ class PrometheusLogger(CustomLogger):
         model_id: str | None,
         api_base: str | None,
         llm_provider: str | None,
+        model_group: str | None,
     ):
         """
         Set the deployment TPM and RPM limits metrics
+
+        Args:
+            model_info: the deployment's static model_info config (id, tpm, rpm, etc.)
+            litellm_params: the deployment's litellm_params, as a tpm/rpm fallback source
+            litellm_model_name: the resolved deployment model name
+            model_id: the deployment's model_id
+            api_base: the deployment's api_base
+            llm_provider: the deployment's custom_llm_provider
+            model_group: the router-resolved model_group the deployment belongs to,
+                from the caller's already-resolved enum_values.model_group (trusted,
+                not caller-supplied at this call site)
         """
         tpm: Final = model_info.get("tpm") or litellm_params.get("tpm")
         rpm: Final = model_info.get("rpm") or litellm_params.get("rpm")
@@ -2927,6 +2949,7 @@ class PrometheusLogger(CustomLogger):
                     model_id=model_id,
                     api_base=api_base,
                     api_provider=llm_provider,
+                    model_group=model_group,
                 ),
             )
             self.litellm_deployment_tpm_limit.labels(**_labels).set(tpm)
@@ -2939,6 +2962,7 @@ class PrometheusLogger(CustomLogger):
                     model_id=model_id,
                     api_base=api_base,
                     api_provider=llm_provider,
+                    model_group=model_group,
                 ),
             )
             self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
@@ -3058,6 +3082,7 @@ class PrometheusLogger(CustomLogger):
                     model_id=model_id,
                     api_base=api_base,
                     llm_provider=llm_provider,
+                    model_group=enum_values.model_group,
                 )
 
             remaining_requests: int | None = None

litellm/types/integrations/prometheus.py

@@ -664,6 +664,7 @@ class PrometheusMetricLabels:
     ]
 
     litellm_deployment_tpm_limit = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.MODEL_ID.value,
         UserAPIKeyLabelNames.API_BASE.value,
@@ -770,6 +771,7 @@ class PrometheusMetricLabels:
 
     # Add deployment metrics
     litellm_deployment_failure_responses = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.REQUESTED_MODEL.value,
         UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.MODEL_ID.value,
@@ -786,6 +788,7 @@ class PrometheusMetricLabels:
     ]
 
     litellm_deployment_total_requests = [
+        UserAPIKeyLabelNames.MODEL_GROUP.value,
         UserAPIKeyLabelNames.REQUESTED_MODEL.value,
         UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.MODEL_ID.value,

tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py

@@ -1031,6 +1031,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
         api_base="https://api.openai.com",
         api_provider="openai",
         requested_model="my_custom_model_group",
+        model_group="my_custom_model_group",
         hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
         api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
         team=standard_logging_payload["metadata"]["user_api_key_team_id"],
@@ -1047,6 +1048,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
         api_base="https://api.openai.com",
         api_provider="openai",
         requested_model="my_custom_model_group",
+        model_group="my_custom_model_group",
         hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
         api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
         team=standard_logging_payload["metadata"]["user_api_key_team_id"],

tests/test_litellm/integrations/test_prometheus_labels.py

@@ -787,6 +787,176 @@ async def test_failure_hook_prefers_request_data_provider_over_exception_provide
     ) == ["azure"]
 
 
+def test_model_group_in_deployment_metrics():
+    """
+    Test that model_group label is present on the deployment-scoped metrics
+    needed to build model-group dashboards (request counts, success/failure
+    counts, tpm/rpm limits). These metrics previously only carried
+    requested_model, litellm_model_name and model_id, none of which identify
+    the model_group a pooled deployment belongs to.
+    """
+    model_group_label = UserAPIKeyLabelNames.MODEL_GROUP.value
+
+    metrics_with_model_group = [
+        "litellm_deployment_total_requests",
+        "litellm_deployment_success_responses",
+        "litellm_deployment_failure_responses",
+        "litellm_deployment_tpm_limit",
+        "litellm_deployment_rpm_limit",
+    ]
+
+    for metric_name in metrics_with_model_group:
+        labels = PrometheusMetricLabels.get_labels(metric_name)
+        assert (
+            model_group_label in labels
+        ), f"Metric {metric_name} should contain model_group label"
+        print(f"✅ {metric_name} contains model_group label")
+
+
+def test_model_group_value_flows_through_deployment_metrics_label_factory():
+    """
+    The label being in the allow-list is necessary but not sufficient: the
+    factory must also carry the value from the enum through to the emitted
+    label. This would fail if the label were dropped from a metric's list or
+    if the value plumbing regressed, which the allow-list assertion above
+    cannot catch on its own.
+    """
+    from unittest.mock import MagicMock
+
+    from litellm.integrations.prometheus import (
+        PrometheusLogger,
+        UserAPIKeyLabelValues,
+        prometheus_label_factory,
+    )
+
+    prometheus_logger = MagicMock()
+    prometheus_logger._cached_metric_labels = {}
+    prometheus_logger.label_filters = {}
+    prometheus_logger.get_labels_for_metric = (
+        PrometheusLogger.get_labels_for_metric.__get__(prometheus_logger)
+    )
+
+    enum_values = UserAPIKeyLabelValues(
+        model_group="example-model-group",
+        litellm_model_name="gpt-4o-mini",
+        requested_model="example-model-group",
+        status_code="200",
+    )
+
+    for metric_name in [
+        "litellm_deployment_total_requests",
+        "litellm_deployment_success_responses",
+        "litellm_deployment_failure_responses",
+        "litellm_deployment_tpm_limit",
+        "litellm_deployment_rpm_limit",
+    ]:
+        labels = prometheus_label_factory(
+            supported_enum_labels=prometheus_logger.get_labels_for_metric(
+                metric_name=metric_name
+            ),
+            enum_values=enum_values,
+        )
+        assert (
+            labels.get("model_group") == "example-model-group"
+        ), f"{metric_name} should emit model_group=example-model-group, got {labels.get('model_group')!r}"
+
+
+def test_deployment_failure_metrics_emit_model_group_from_standard_logging_payload():
+    """
+    End-to-end emit wiring for the failure path.
+
+    The label-list and factory tests above prove the label exists and that
+    the factory carries a value handed to it, but neither drives the real
+    set_llm_deployment_failure_metrics code path, so deleting the production
+    model_group=model_group assignment there would still pass them. This
+    calls it directly with a standard_logging_object carrying model_group and
+    asserts the real litellm_deployment_failure_responses / _total_requests
+    Counter series actually carry it.
+    """
+    from litellm.integrations.prometheus import PrometheusLogger
+
+    _clear_prometheus_registry()
+    try:
+        logger = PrometheusLogger()
+        logger.set_llm_deployment_failure_metrics(
+            request_kwargs={
+                "model": "gpt-4o-mini",
+                "litellm_params": {"metadata": {}},
+                "standard_logging_object": {
+                    "model_group": "example-model-group",
+                    "model_id": "model-123",
+                    "api_base": "https://api.openai.com",
+                    "request_tags": [],
+                },
+                "exception": Exception("boom"),
+            }
+        )
+
+        for metric in (
+            logger.litellm_deployment_failure_responses,
+            logger.litellm_deployment_total_requests,
+        ):
+            index = metric._labelnames.index("model_group")
+            values = {sample_key[index] for sample_key in metric._metrics}
+            assert values == {"example-model-group"}, (
+                f"expected model_group=example-model-group on {metric._name}, got {values}"
+            )
+    finally:
+        _clear_prometheus_registry()
+
+
+def test_deployment_tpm_rpm_limit_metrics_emit_model_group_from_enum_values():
+    """
+    End-to-end emit wiring for the tpm/rpm limit gauges.
+
+    _set_deployment_tpm_rpm_limit_metrics used to build its own
+    UserAPIKeyLabelValues with no model_group parameter at all, dropping the
+    value even though its only caller (set_llm_deployment_success_metrics)
+    already had it on enum_values. This drives set_llm_deployment_success_metrics
+    directly with a deployment that has tpm/rpm configured and asserts the real
+    litellm_deployment_tpm_limit / litellm_deployment_rpm_limit Gauge series
+    carry model_group; it fails if that plumbing is removed.
+    """
+    import datetime
+
+    from litellm.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues
+
+    _clear_prometheus_registry()
+    try:
+        logger = PrometheusLogger()
+        now = datetime.datetime.now()
+        enum_values = UserAPIKeyLabelValues(
+            model_group="example-model-group",
+            litellm_model_name="gpt-4o-mini",
+            requested_model="example-model-group",
+            status_code="200",
+        )
+        logger.set_llm_deployment_success_metrics(
+            request_kwargs={
+                "model": "gpt-4o-mini",
+                "litellm_params": {"metadata": {"model_info": {"id": "model-123", "tpm": 1000, "rpm": 10}}},
+                "standard_logging_object": {
+                    "model_group": "example-model-group",
+                    "model_id": "model-123",
+                    "api_base": "https://api.openai.com",
+                    "hidden_params": {"additional_headers": None, "litellm_overhead_time_ms": None},
+                },
+            },
+            start_time=now,
+            end_time=now,
+            enum_values=enum_values,
+        )
+
+        for metric in (logger.litellm_deployment_tpm_limit, logger.litellm_deployment_rpm_limit):
+            index = metric._labelnames.index("model_group")
+            values = {sample_key[index] for sample_key in metric._metrics}
+            assert values == {"example-model-group"}, (
+                f"expected model_group=example-model-group on {metric._name}, got {values}"
+            )
+    finally:
+        _clear_prometheus_registry()
+
+
 if __name__ == "__main__":
     test_user_email_in_required_metrics()
     test_user_email_label_exists()