Merge 32c15b5478 into e0982d6248

litellm/integrations/prometheus.py

@@ -2660,7 +2660,15 @@ class PrometheusLogger(CustomLogger):
         if self._should_skip_metrics_for_invalid_key(kwargs=kwargs, standard_logging_payload=standard_logging_payload):
             return
 
-        model: Final = kwargs.get("model", "")
+        deployment_model_id: Final = standard_logging_payload.get("model_id", "")  # pyright: ignore[reportUnknownVariableType]  # StandardLoggingPayload.get is partially unknown
+        requested_name: Final = kwargs.get("model", "")  # pyright: ignore[reportUnknownVariableType]  # kwargs is an untyped callback argument
+        # With no deployment behind the call, the name is whatever the caller sent (for example a model the
+        # key may not use). Bound it like ``requested_model`` so unknown names share one series.
+        model: Final = (
+            requested_name
+            if deployment_model_id
+            else _bounded_requested_model_label(requested_name, router_originated=True)  # pyright: ignore[reportUnknownArgumentType]  # caller-supplied value, the helper type-checks it
+        )
 
         litellm_params: Final = kwargs.get("litellm_params", {}) or {}
         get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking()
@@ -2682,7 +2690,7 @@ class PrometheusLogger(CustomLogger):
                 team=user_api_team,
                 team_alias=user_api_team_alias,
                 user=user_id,
-                model_id=standard_logging_payload.get("model_id", ""),
+                model_id=deployment_model_id,
                 requested_model=standard_logging_payload.get("model_group"),
                 api_provider=standard_logging_payload.get("custom_llm_provider"),
                 custom_metadata_labels=get_custom_labels_from_metadata(

tests/unit/integrations/test_prometheus_requested_model_cardinality.py

@@ -293,3 +293,81 @@ async def test_fallback_event_requested_model_is_bounded(router):
     assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == {
         UNRECOGNIZED_REQUESTED_MODEL_LABEL
     }
+
+
+def _model_label_values(metric) -> set[str]:
+    index = metric._labelnames.index("model")
+    return {sample_key[index] for sample_key in metric._metrics}
+
+
+async def _fire_llm_api_failure(logger: PrometheusLogger, model: str, model_id: str = "") -> None:
+    await logger.async_log_failure_event(
+        kwargs={
+            "model": model,
+            "litellm_params": {"metadata": {}},
+            "exception": _ClientSideError("model not allowed for this key"),
+            "standard_logging_object": {
+                "model_id": model_id,
+                "model_group": model,
+                "metadata": {
+                    "user_api_key_user_id": "user-1",
+                    "user_api_key_hash": "hashed-key-1",
+                    "user_api_key_alias": None,
+                    "user_api_key_team_id": None,
+                    "user_api_key_team_alias": None,
+                },
+            },
+        },
+        response_obj=None,
+        start_time=None,
+        end_time=None,
+    )
+
+
+@pytest.mark.asyncio
+async def test_denied_model_names_collapse_to_one_series_on_llm_api_failed_requests_metric(router):
+    logger = PrometheusLogger()
+
+    with patch(
+        "litellm.proxy.proxy_server.llm_router", router, create=True
+    ):  # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
+        for index in range(25):
+            await _fire_llm_api_failure(logger, f"denied-model-{index}")
+
+    metric = logger.litellm_llm_api_failed_requests_metric
+    assert _model_label_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}
+    assert _series_count(metric) == 1
+    assert _total_value(metric) == 25
+
+
+@pytest.mark.asyncio
+async def test_deployment_model_names_keep_their_label_on_llm_api_failed_requests_metric(router):
+    logger = PrometheusLogger()
+
+    with patch(
+        "litellm.proxy.proxy_server.llm_router", router, create=True
+    ):  # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
+        await _fire_llm_api_failure(logger, "openai/gpt-4o-mini", model_id="deployment-1")
+
+    assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"openai/gpt-4o-mini"}
+
+
+@pytest.mark.asyncio
+async def test_sdk_failures_without_proxy_router_keep_their_model_label_on_llm_api_failed_requests_metric():
+    logger = PrometheusLogger()
+
+    with patch("litellm.proxy.proxy_server.llm_router", None, create=True):  # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
+        await _fire_llm_api_failure(logger, "gpt-4o-mini")
+
+    assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"gpt-4o-mini"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model", [123, 4.5, ["gpt-4o-mini"], {"name": "gpt-4o-mini"}])
+async def test_non_string_denied_models_collapse_to_other_on_llm_api_failed_requests_metric(router, model: object):
+    logger = PrometheusLogger()
+
+    with patch("litellm.proxy.proxy_server.llm_router", router, create=True):  # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
+        await _fire_llm_api_failure(logger, model)  # type: ignore[arg-type]
+
+    assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}