From d756f14b59e2ca348a96ff57e782bd63e2c4b52a Mon Sep 17 00:00:00 2001 From: Denel John Date: Fri, 16 Oct 2026 14:29:50 -0400 Subject: [PATCH] fix(prometheus): bound the model label on failed LLM API requests When a request is rejected before any deployment is picked, for example a model the key is not allowed to call, the legacy `model` label on litellm_llm_api_failed_requests_metric carried the raw name the caller sent. Every distinct name created a new series that stayed until the worker restarted. Run that value through the same bounding used for `requested_model` when no deployment served the call, so unrecognized names share the single `other` series. Calls that reached a deployment keep their model label. Fixes #46909 Tests drafted with AI assistance (Cursor agent). Signed-off-by: Denel John --- litellm/integrations/prometheus.py | 22 +++++- ..._prometheus_requested_model_cardinality.py | 67 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fa5496004e3..32f9b97d4bf 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2601,6 +2601,23 @@ class PrometheusLogger(CustomLogger): _labels, ) + @staticmethod + def _bounded_failed_request_model_label( + model: object, standard_logging_payload: dict | StandardLoggingPayload + ) -> object: + """ + Bound the legacy ``model`` label on the failed-request metric. + + When no deployment served the call (no ``model_id``), ``model`` is the + raw name the caller sent, e.g. a model the key is not allowed to use. + Collapse unrecognized names into the shared ``other`` bucket so callers + cannot mint unlimited series. Names of real deployments keep their label, + and with no proxy router (plain SDK use) nothing is collapsed. + """ + if standard_logging_payload.get("model_id"): + return model + return _bounded_requested_model_label(model, router_originated=True) + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "prometheus Logging - Enters failure logging function (kwargs keys: %s)", @@ -2612,7 +2629,10 @@ class PrometheusLogger(CustomLogger): if self._should_skip_metrics_for_invalid_key(kwargs=kwargs, standard_logging_payload=standard_logging_payload): return - model: Final = kwargs.get("model", "") + model: Final = self._bounded_failed_request_model_label( + model=kwargs.get("model", ""), + standard_logging_payload=standard_logging_payload, + ) litellm_params: Final = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking() diff --git a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py index f86f4460b28..de08ff556ec 100644 --- a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py @@ -293,3 +293,70 @@ async def test_fallback_event_requested_model_is_bounded(router): assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == { UNRECOGNIZED_REQUESTED_MODEL_LABEL } + + +def _model_label_values(metric) -> set[str]: + index = metric._labelnames.index("model") + return {sample_key[index] for sample_key in metric._metrics} + + +async def _fire_llm_api_failure(logger: PrometheusLogger, model: str, model_id: str = "") -> None: + await logger.async_log_failure_event( + kwargs={ + "model": model, + "litellm_params": {"metadata": {}}, + "exception": _ClientSideError("model not allowed for this key"), + "standard_logging_object": { + "model_id": model_id, + "model_group": model, + "metadata": { + "user_api_key_user_id": "user-1", + "user_api_key_hash": "hashed-key-1", + "user_api_key_alias": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + }, + }, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + +@pytest.mark.asyncio +async def test_denied_model_names_collapse_to_one_series_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + await _fire_llm_api_failure(logger, f"denied-model-{index}") + + metric = logger.litellm_llm_api_failed_requests_metric + assert _model_label_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _series_count(metric) == 1 + assert _total_value(metric) == 25 + + +@pytest.mark.asyncio +async def test_deployment_model_names_keep_their_label_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "openai/gpt-4o-mini", model_id="deployment-1") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"openai/gpt-4o-mini"} + + +@pytest.mark.asyncio +async def test_sdk_failures_without_proxy_router_keep_their_model_label_on_llm_api_failed_requests_metric(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "gpt-4o-mini") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"gpt-4o-mini"}