From d756f14b59e2ca348a96ff57e782bd63e2c4b52a Mon Sep 17 00:00:00 2001 From: Denel John Date: Fri, 16 Oct 2026 14:29:50 -0400 Subject: [PATCH 1/4] fix(prometheus): bound the model label on failed LLM API requests When a request is rejected before any deployment is picked, for example a model the key is not allowed to call, the legacy `model` label on litellm_llm_api_failed_requests_metric carried the raw name the caller sent. Every distinct name created a new series that stayed until the worker restarted. Run that value through the same bounding used for `requested_model` when no deployment served the call, so unrecognized names share the single `other` series. Calls that reached a deployment keep their model label. Fixes #46909 Tests drafted with AI assistance (Cursor agent). Signed-off-by: Denel John --- litellm/integrations/prometheus.py | 22 +++++- ..._prometheus_requested_model_cardinality.py | 67 +++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fa5496004e3..32f9b97d4bf 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2601,6 +2601,23 @@ class PrometheusLogger(CustomLogger): _labels, ) + @staticmethod + def _bounded_failed_request_model_label( + model: object, standard_logging_payload: dict | StandardLoggingPayload + ) -> object: + """ + Bound the legacy ``model`` label on the failed-request metric. + + When no deployment served the call (no ``model_id``), ``model`` is the + raw name the caller sent, e.g. a model the key is not allowed to use. + Collapse unrecognized names into the shared ``other`` bucket so callers + cannot mint unlimited series. Names of real deployments keep their label, + and with no proxy router (plain SDK use) nothing is collapsed. + """ + if standard_logging_payload.get("model_id"): + return model + return _bounded_requested_model_label(model, router_originated=True) + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "prometheus Logging - Enters failure logging function (kwargs keys: %s)", @@ -2612,7 +2629,10 @@ class PrometheusLogger(CustomLogger): if self._should_skip_metrics_for_invalid_key(kwargs=kwargs, standard_logging_payload=standard_logging_payload): return - model: Final = kwargs.get("model", "") + model: Final = self._bounded_failed_request_model_label( + model=kwargs.get("model", ""), + standard_logging_payload=standard_logging_payload, + ) litellm_params: Final = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking() diff --git a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py index f86f4460b28..de08ff556ec 100644 --- a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py @@ -293,3 +293,70 @@ async def test_fallback_event_requested_model_is_bounded(router): assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == { UNRECOGNIZED_REQUESTED_MODEL_LABEL } + + +def _model_label_values(metric) -> set[str]: + index = metric._labelnames.index("model") + return {sample_key[index] for sample_key in metric._metrics} + + +async def _fire_llm_api_failure(logger: PrometheusLogger, model: str, model_id: str = "") -> None: + await logger.async_log_failure_event( + kwargs={ + "model": model, + "litellm_params": {"metadata": {}}, + "exception": _ClientSideError("model not allowed for this key"), + "standard_logging_object": { + "model_id": model_id, + "model_group": model, + "metadata": { + "user_api_key_user_id": "user-1", + "user_api_key_hash": "hashed-key-1", + "user_api_key_alias": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + }, + }, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + +@pytest.mark.asyncio +async def test_denied_model_names_collapse_to_one_series_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + await _fire_llm_api_failure(logger, f"denied-model-{index}") + + metric = logger.litellm_llm_api_failed_requests_metric + assert _model_label_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _series_count(metric) == 1 + assert _total_value(metric) == 25 + + +@pytest.mark.asyncio +async def test_deployment_model_names_keep_their_label_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "openai/gpt-4o-mini", model_id="deployment-1") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"openai/gpt-4o-mini"} + + +@pytest.mark.asyncio +async def test_sdk_failures_without_proxy_router_keep_their_model_label_on_llm_api_failed_requests_metric(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "gpt-4o-mini") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"gpt-4o-mini"} From a42a175f142ded6aeaf3409572cf2d09a3f3a9e4 Mon Sep 17 00:00:00 2001 From: Denel John Date: Fri, 16 Oct 2026 14:56:50 -0400 Subject: [PATCH 2/4] fix(prometheus): use a read-only payload type in the failed model label helper The type discipline gate counted the dict | StandardLoggingPayload annotation as a new mutable collection. Signed-off-by: Denel John --- litellm/integrations/prometheus.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 32f9b97d4bf..9d49c4c761d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2602,9 +2602,7 @@ class PrometheusLogger(CustomLogger): ) @staticmethod - def _bounded_failed_request_model_label( - model: object, standard_logging_payload: dict | StandardLoggingPayload - ) -> object: + def _bounded_failed_request_model_label(model: object, standard_logging_payload: StandardLoggingPayload) -> object: """ Bound the legacy ``model`` label on the failed-request metric. From 8c297d232bb5a9f0414647c6a7f1efc769812a00 Mon Sep 17 00:00:00 2001 From: Denel John Date: Fri, 16 Oct 2026 15:39:40 -0400 Subject: [PATCH 3/4] fix(prometheus): keep the failed model label change type-check clean The basedpyright gate counted the helper call as new unknown argument errors. Inline the bounding in async_log_failure_event so the error count matches the merge-base. Signed-off-by: Denel John --- litellm/integrations/prometheus.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 9d49c4c761d..2af5faf0757 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2601,21 +2601,6 @@ class PrometheusLogger(CustomLogger): _labels, ) - @staticmethod - def _bounded_failed_request_model_label(model: object, standard_logging_payload: StandardLoggingPayload) -> object: - """ - Bound the legacy ``model`` label on the failed-request metric. - - When no deployment served the call (no ``model_id``), ``model`` is the - raw name the caller sent, e.g. a model the key is not allowed to use. - Collapse unrecognized names into the shared ``other`` bucket so callers - cannot mint unlimited series. Names of real deployments keep their label, - and with no proxy router (plain SDK use) nothing is collapsed. - """ - if standard_logging_payload.get("model_id"): - return model - return _bounded_requested_model_label(model, router_originated=True) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "prometheus Logging - Enters failure logging function (kwargs keys: %s)", @@ -2627,9 +2612,14 @@ class PrometheusLogger(CustomLogger): if self._should_skip_metrics_for_invalid_key(kwargs=kwargs, standard_logging_payload=standard_logging_payload): return - model: Final = self._bounded_failed_request_model_label( - model=kwargs.get("model", ""), - standard_logging_payload=standard_logging_payload, + deployment_model_id: Final = standard_logging_payload.get("model_id", "") # pyright: ignore[reportUnknownVariableType] # StandardLoggingPayload.get is partially unknown + requested_name: Final = kwargs.get("model", "") # pyright: ignore[reportUnknownVariableType] # kwargs is an untyped callback argument + # With no deployment behind the call, the name is whatever the caller sent (for example a model the + # key may not use). Bound it like ``requested_model`` so unknown names share one series. + model: Final = ( + _bounded_requested_model_label(requested_name, router_originated=True) + if not deployment_model_id and isinstance(requested_name, str) + else requested_name ) litellm_params: Final = kwargs.get("litellm_params", {}) or {} @@ -2652,7 +2642,7 @@ class PrometheusLogger(CustomLogger): team=user_api_team, team_alias=user_api_team_alias, user=user_id, - model_id=standard_logging_payload.get("model_id", ""), + model_id=deployment_model_id, requested_model=standard_logging_payload.get("model_group"), api_provider=standard_logging_payload.get("custom_llm_provider"), custom_metadata_labels=get_custom_labels_from_metadata( From 32c15b5478691adbdf3075dd88ae5ea55e8ca918 Mon Sep 17 00:00:00 2001 From: Denel John Date: Fri, 16 Oct 2026 16:22:24 -0400 Subject: [PATCH 4/4] fix(prometheus): bound non-string model values on failed requests A numeric, list or dict model with no deployment behind the call skipped the bounding and became its own label value. Send every such value through the same helper so they collapse into the shared other series. Signed-off-by: Denel John --- litellm/integrations/prometheus.py | 6 +++--- .../test_prometheus_requested_model_cardinality.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 2af5faf0757..e513d2c655e 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2617,9 +2617,9 @@ class PrometheusLogger(CustomLogger): # With no deployment behind the call, the name is whatever the caller sent (for example a model the # key may not use). Bound it like ``requested_model`` so unknown names share one series. model: Final = ( - _bounded_requested_model_label(requested_name, router_originated=True) - if not deployment_model_id and isinstance(requested_name, str) - else requested_name + requested_name + if deployment_model_id + else _bounded_requested_model_label(requested_name, router_originated=True) # pyright: ignore[reportUnknownArgumentType] # caller-supplied value, the helper type-checks it ) litellm_params: Final = kwargs.get("litellm_params", {}) or {} diff --git a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py index de08ff556ec..f4f0d3607e4 100644 --- a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py @@ -360,3 +360,14 @@ async def test_sdk_failures_without_proxy_router_keep_their_model_label_on_llm_a await _fire_llm_api_failure(logger, "gpt-4o-mini") assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"gpt-4o-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [123, 4.5, ["gpt-4o-mini"], {"name": "gpt-4o-mini"}]) +async def test_non_string_denied_models_collapse_to_other_on_llm_api_failed_requests_metric(router, model: object): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, model) # type: ignore[arg-type] + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}