From f39a56b004c83f4242f7f95c48504235f1af2180 Mon Sep 17 00:00:00 2001 From: ahamedshaik16 Date: Fri, 25 Sep 2026 00:45:01 +0530 Subject: [PATCH] fix(prometheus): add model_group label to deployment request and rate limit metrics (#42966) * fix(prometheus): add model_group label to deployment request and rate limit metrics litellm_deployment_total_requests, litellm_deployment_success_responses, litellm_deployment_failure_responses, litellm_deployment_tpm_limit and litellm_deployment_rpm_limit had no way to identify which model_group a pooled deployment belongs to, only requested_model, litellm_model_name and model_id, none of which name the alias a model_name resolves through when it fans out to more than one deployment. model_group was already resolved onto enum_values for every request in async_log_success_event, so this is a label-list addition for the metrics built directly from that enum_values (the two request counters). The failure counter builds its own UserAPIKeyLabelValues locally and had a model_group variable already in scope that it never passed through, and the tpm/rpm limit gauges are set from a helper that took no model_group parameter at all even though its only caller already had it on enum_values. Both now thread the value through. * test(prometheus): expect model_group in deployment success/total request labels test_set_llm_deployment_success_metrics asserts the exact label set passed to litellm_deployment_success_responses.labels() and litellm_deployment_total_requests.labels(), which now includes model_group since it was added to those metrics' label list. * fix(prometheus): bound model_group on deployment failure metrics On a pre-routing reject (no deployment selected), model_group is caller-supplied via litellm_params.metadata and was passed through unbounded, letting an unrecognized value mint unlimited label series on litellm_deployment_failure_responses / litellm_deployment_total_requests. Bound it with the same _bounded_requested_model_label used for requested_model on this path. When a deployment is actually selected, model_group is router-resolved and passed through as-is. Also documents the model_group parameter on _set_deployment_tpm_rpm_limit_metrics and the bounding behavior on set_llm_deployment_failure_metrics. --------- Co-authored-by: ahamedshaik16 <24526479+ahamedshaik16@users.noreply.github.com> --- litellm/integrations/prometheus.py | 25 +++ litellm/types/integrations/prometheus.py | 3 + .../test_prometheus_logging_callbacks.py | 2 + .../integrations/test_prometheus_labels.py | 170 ++++++++++++++++++ 4 files changed, 200 insertions(+) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2fdcb8ef745..fb010ab5886 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2778,6 +2778,13 @@ class PrometheusLogger(CustomLogger): - increment deployment failure responses metric - increment deployment total requests metric + Both counters also carry a model_group label. When a deployment was + actually selected, model_group is the router-resolved value and is + trusted as-is. On a pre-routing reject (no deployment selected), it + is caller-supplied via litellm_params.metadata and is bounded with + _bounded_requested_model_label the same way requested_model is, so an + unrecognized value cannot mint unbounded label series. + Args: request_kwargs: dict @@ -2844,6 +2851,7 @@ class PrometheusLogger(CustomLogger): label_api_base = api_base label_api_provider = llm_provider label_requested_model = model_group or litellm_model_name + label_model_group = model_group else: label_litellm_model_name = "" label_model_id = "" @@ -2852,6 +2860,7 @@ class PrometheusLogger(CustomLogger): label_requested_model = ( _bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or "" ) + label_model_group = _bounded_requested_model_label(model_group, router_originated=True) enum_values: Final = UserAPIKeyLabelValues( litellm_model_name=label_litellm_model_name, @@ -2861,6 +2870,7 @@ class PrometheusLogger(CustomLogger): exception_status=exception_status, exception_class=(self._get_exception_class_name(exception) if exception else None), requested_model=label_requested_model, + model_group=label_model_group, hashed_api_key=hashed_api_key, api_key_alias=api_key_alias, user_email=user_email, @@ -2912,9 +2922,21 @@ class PrometheusLogger(CustomLogger): model_id: str | None, api_base: str | None, llm_provider: str | None, + model_group: str | None, ): """ Set the deployment TPM and RPM limits metrics + + Args: + model_info: the deployment's static model_info config (id, tpm, rpm, etc.) + litellm_params: the deployment's litellm_params, as a tpm/rpm fallback source + litellm_model_name: the resolved deployment model name + model_id: the deployment's model_id + api_base: the deployment's api_base + llm_provider: the deployment's custom_llm_provider + model_group: the router-resolved model_group the deployment belongs to, + from the caller's already-resolved enum_values.model_group (trusted, + not caller-supplied at this call site) """ tpm: Final = model_info.get("tpm") or litellm_params.get("tpm") rpm: Final = model_info.get("rpm") or litellm_params.get("rpm") @@ -2927,6 +2949,7 @@ class PrometheusLogger(CustomLogger): model_id=model_id, api_base=api_base, api_provider=llm_provider, + model_group=model_group, ), ) self.litellm_deployment_tpm_limit.labels(**_labels).set(tpm) @@ -2939,6 +2962,7 @@ class PrometheusLogger(CustomLogger): model_id=model_id, api_base=api_base, api_provider=llm_provider, + model_group=model_group, ), ) self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm) @@ -3058,6 +3082,7 @@ class PrometheusLogger(CustomLogger): model_id=model_id, api_base=api_base, llm_provider=llm_provider, + model_group=enum_values.model_group, ) remaining_requests: int | None = None diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index c929ee2ee79..239fc7f2779 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -664,6 +664,7 @@ class PrometheusMetricLabels: ] litellm_deployment_tpm_limit = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, UserAPIKeyLabelNames.API_BASE.value, @@ -770,6 +771,7 @@ class PrometheusMetricLabels: # Add deployment metrics litellm_deployment_failure_responses = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, @@ -786,6 +788,7 @@ class PrometheusMetricLabels: ] litellm_deployment_total_requests = [ + UserAPIKeyLabelNames.MODEL_GROUP.value, UserAPIKeyLabelNames.REQUESTED_MODEL.value, UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value, UserAPIKeyLabelNames.MODEL_ID.value, diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 58cde4c8103..92ff3d5813c 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1031,6 +1031,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_base="https://api.openai.com", api_provider="openai", requested_model="my_custom_model_group", + model_group="my_custom_model_group", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], team=standard_logging_payload["metadata"]["user_api_key_team_id"], @@ -1047,6 +1048,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_base="https://api.openai.com", api_provider="openai", requested_model="my_custom_model_group", + model_group="my_custom_model_group", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], team=standard_logging_payload["metadata"]["user_api_key_team_id"], diff --git a/tests/test_litellm/integrations/test_prometheus_labels.py b/tests/test_litellm/integrations/test_prometheus_labels.py index 200f8e65add..41d0c44ff89 100644 --- a/tests/test_litellm/integrations/test_prometheus_labels.py +++ b/tests/test_litellm/integrations/test_prometheus_labels.py @@ -787,6 +787,176 @@ async def test_failure_hook_prefers_request_data_provider_over_exception_provide ) == ["azure"] +def test_model_group_in_deployment_metrics(): + """ + Test that model_group label is present on the deployment-scoped metrics + needed to build model-group dashboards (request counts, success/failure + counts, tpm/rpm limits). These metrics previously only carried + requested_model, litellm_model_name and model_id, none of which identify + the model_group a pooled deployment belongs to. + """ + model_group_label = UserAPIKeyLabelNames.MODEL_GROUP.value + + metrics_with_model_group = [ + "litellm_deployment_total_requests", + "litellm_deployment_success_responses", + "litellm_deployment_failure_responses", + "litellm_deployment_tpm_limit", + "litellm_deployment_rpm_limit", + ] + + for metric_name in metrics_with_model_group: + labels = PrometheusMetricLabels.get_labels(metric_name) + assert ( + model_group_label in labels + ), f"Metric {metric_name} should contain model_group label" + print(f"✅ {metric_name} contains model_group label") + + +def test_model_group_value_flows_through_deployment_metrics_label_factory(): + """ + The label being in the allow-list is necessary but not sufficient: the + factory must also carry the value from the enum through to the emitted + label. This would fail if the label were dropped from a metric's list or + if the value plumbing regressed, which the allow-list assertion above + cannot catch on its own. + """ + from unittest.mock import MagicMock + + from litellm.integrations.prometheus import ( + PrometheusLogger, + UserAPIKeyLabelValues, + prometheus_label_factory, + ) + + prometheus_logger = MagicMock() + prometheus_logger._cached_metric_labels = {} + prometheus_logger.label_filters = {} + prometheus_logger.get_labels_for_metric = ( + PrometheusLogger.get_labels_for_metric.__get__(prometheus_logger) + ) + + enum_values = UserAPIKeyLabelValues( + model_group="example-model-group", + litellm_model_name="gpt-4o-mini", + requested_model="example-model-group", + status_code="200", + ) + + for metric_name in [ + "litellm_deployment_total_requests", + "litellm_deployment_success_responses", + "litellm_deployment_failure_responses", + "litellm_deployment_tpm_limit", + "litellm_deployment_rpm_limit", + ]: + labels = prometheus_label_factory( + supported_enum_labels=prometheus_logger.get_labels_for_metric( + metric_name=metric_name + ), + enum_values=enum_values, + ) + assert ( + labels.get("model_group") == "example-model-group" + ), f"{metric_name} should emit model_group=example-model-group, got {labels.get('model_group')!r}" + + +def test_deployment_failure_metrics_emit_model_group_from_standard_logging_payload(): + """ + End-to-end emit wiring for the failure path. + + The label-list and factory tests above prove the label exists and that + the factory carries a value handed to it, but neither drives the real + set_llm_deployment_failure_metrics code path, so deleting the production + model_group=model_group assignment there would still pass them. This + calls it directly with a standard_logging_object carrying model_group and + asserts the real litellm_deployment_failure_responses / _total_requests + Counter series actually carry it. + """ + from litellm.integrations.prometheus import PrometheusLogger + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "model_group": "example-model-group", + "model_id": "model-123", + "api_base": "https://api.openai.com", + "request_tags": [], + }, + "exception": Exception("boom"), + } + ) + + for metric in ( + logger.litellm_deployment_failure_responses, + logger.litellm_deployment_total_requests, + ): + index = metric._labelnames.index("model_group") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {"example-model-group"}, ( + f"expected model_group=example-model-group on {metric._name}, got {values}" + ) + finally: + _clear_prometheus_registry() + + +def test_deployment_tpm_rpm_limit_metrics_emit_model_group_from_enum_values(): + """ + End-to-end emit wiring for the tpm/rpm limit gauges. + + _set_deployment_tpm_rpm_limit_metrics used to build its own + UserAPIKeyLabelValues with no model_group parameter at all, dropping the + value even though its only caller (set_llm_deployment_success_metrics) + already had it on enum_values. This drives set_llm_deployment_success_metrics + directly with a deployment that has tpm/rpm configured and asserts the real + litellm_deployment_tpm_limit / litellm_deployment_rpm_limit Gauge series + carry model_group; it fails if that plumbing is removed. + """ + import datetime + + from litellm.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues + + _clear_prometheus_registry() + try: + logger = PrometheusLogger() + now = datetime.datetime.now() + enum_values = UserAPIKeyLabelValues( + model_group="example-model-group", + litellm_model_name="gpt-4o-mini", + requested_model="example-model-group", + status_code="200", + ) + logger.set_llm_deployment_success_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {"model_info": {"id": "model-123", "tpm": 1000, "rpm": 10}}}, + "standard_logging_object": { + "model_group": "example-model-group", + "model_id": "model-123", + "api_base": "https://api.openai.com", + "hidden_params": {"additional_headers": None, "litellm_overhead_time_ms": None}, + }, + }, + start_time=now, + end_time=now, + enum_values=enum_values, + ) + + for metric in (logger.litellm_deployment_tpm_limit, logger.litellm_deployment_rpm_limit): + index = metric._labelnames.index("model_group") + values = {sample_key[index] for sample_key in metric._metrics} + assert values == {"example-model-group"}, ( + f"expected model_group=example-model-group on {metric._name}, got {values}" + ) + finally: + _clear_prometheus_registry() + + if __name__ == "__main__": test_user_email_in_required_metrics() test_user_email_label_exists()