Merge 57629e7383 into 9071ca503e

litellm/integrations/prometheus.py

@@ -2340,6 +2340,34 @@ class PrometheusLogger(CustomLogger):
                 _labels,
             )
 
+    @staticmethod
+    def _get_deployment_failure_model_id(
+        request_kwargs: Mapping[str, object], standard_logging_payload: StandardLoggingPayload
+    ) -> str | None:
+        exception: Final = request_kwargs.get("exception")
+        failed_deployment_id: Final = getattr(exception, "failed_deployment_id", None)
+        if isinstance(failed_deployment_id, str) and failed_deployment_id:
+            return failed_deployment_id
+
+        standard_model_id: Final = standard_logging_payload.get("model_id")
+        if standard_model_id:
+            return standard_model_id
+
+        litellm_params: Final = request_kwargs.get("litellm_params")
+        if not isinstance(litellm_params, Mapping):
+            return None
+        for metadata_key in ("litellm_metadata", "metadata"):
+            metadata = litellm_params.get(metadata_key)
+            if not isinstance(metadata, Mapping):
+                continue
+            model_info = metadata.get("model_info")
+            if not isinstance(model_info, Mapping):
+                continue
+            model_id = model_info.get("id")
+            if isinstance(model_id, str) and model_id:
+                return model_id
+        return None
+
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         verbose_logger.debug(
             "prometheus Logging - Enters failure logging function (kwargs keys: %s)",
@@ -2363,6 +2391,13 @@ class PrometheusLogger(CustomLogger):
         user_api_team: Final = standard_logging_payload["metadata"]["user_api_key_team_id"]
         user_api_team_alias: Final = standard_logging_payload["metadata"]["user_api_key_team_alias"]
         user_api_key_org_id: Final = standard_logging_payload["metadata"].get("user_api_key_org_id")
+        model_id: Final = (
+            self._get_deployment_failure_model_id(
+                request_kwargs=kwargs,
+                standard_logging_payload=standard_logging_payload,
+            )
+            or ""
+        )
 
         try:
             enum_values: Final = UserAPIKeyLabelValues(
@@ -2373,7 +2408,7 @@ class PrometheusLogger(CustomLogger):
                 team=user_api_team,
                 team_alias=user_api_team_alias,
                 user=user_id,
-                model_id=standard_logging_payload.get("model_id", ""),
+                model_id=model_id,
                 custom_metadata_labels=get_custom_labels_from_metadata(
                     metadata=_get_combined_custom_metadata_from_standard_logging_payload(
                         standard_logging_payload=standard_logging_payload
@@ -2773,17 +2808,11 @@ class PrometheusLogger(CustomLogger):
             litellm_model_name: Final = request_kwargs.get("model", None)
             model_group = standard_logging_payload.get("model_group", None)
             api_base: Final = standard_logging_payload.get("api_base", None)
-            model_id = standard_logging_payload.get("model_id", None)
             exception: Final = request_kwargs.get("exception", None)
-
-            # Fallback: model_id from litellm_metadata.model_info
-            if model_id is None:
-                _model_info: Final = (
-                    (_litellm_params.get("litellm_metadata") or {}).get("model_info")
-                    or (_litellm_params.get("metadata") or {}).get("model_info")
-                    or {}
-                )
-                model_id = _model_info.get("id")
+            model_id: Final = self._get_deployment_failure_model_id(
+                request_kwargs=request_kwargs,
+                standard_logging_payload=standard_logging_payload,
+            )
 
             # Fallback: model_group from litellm_metadata
             if model_group is None:

litellm/router.py

@@ -8599,6 +8599,7 @@ class Router:
                 try:
                     await _callback.async_pre_call_check(deployment, parent_otel_span)
                 except litellm.RateLimitError as e:
+                    self._set_failed_deployment_id_on_exception(e, deployment)
                     ## LOG FAILURE EVENT
                     if logging_obj is not None:
                         asyncio.create_task(
@@ -8617,6 +8618,7 @@ class Router:
                     )
                     raise e
                 except Exception as e:
+                    self._set_failed_deployment_id_on_exception(e, deployment)
                     ## LOG FAILURE EVENT
                     if logging_obj is not None:
                         asyncio.create_task(

tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py

@@ -83,6 +83,11 @@ def _requested_model_values(metric) -> set[str]:
     return {sample_key[index] for sample_key in metric._metrics}
 
 
+def _model_id_values(metric) -> set[str]:
+    index = metric._labelnames.index("model_id")
+    return {sample_key[index] for sample_key in metric._metrics}
+
+
 def _series_count(metric) -> int:
     return len(metric._metrics)
 
@@ -194,6 +199,85 @@ async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router():
     assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"}
 
 
+def test_deployment_failure_prefers_stamped_failed_deployment_id_over_mutated_metadata():
+    logger = PrometheusLogger()
+    exception = _ClientSideError("deployment-a exceeded its TPM limit")
+    exception.failed_deployment_id = "deployment-a"
+
+    logger.set_llm_deployment_failure_metrics(
+        request_kwargs={
+            "model": "model-group",
+            "litellm_params": {"metadata": {"model_info": {"id": "deployment-b"}}},
+            "standard_logging_object": {"model_id": "deployment-b"},
+            "exception": exception,
+        }
+    )
+
+    assert _model_id_values(logger.litellm_deployment_failure_responses) == {"deployment-a"}
+
+
+@pytest.mark.asyncio
+async def test_async_failure_metrics_prefer_stamped_failed_deployment_id():
+    logger = PrometheusLogger()
+    exception = _ClientSideError("deployment-a exceeded its TPM limit")
+    exception.failed_deployment_id = "deployment-a"
+
+    await logger.async_log_failure_event(
+        kwargs={
+            "model": "model-group",
+            "litellm_params": {"metadata": {"model_info": {"id": "deployment-b"}}},
+            "standard_logging_object": {
+                "model_id": "deployment-b",
+                "model_group": "model-group",
+                "metadata": {
+                    "user_api_key_user_id": "user",
+                    "user_api_key_hash": "hash",
+                    "user_api_key_alias": "alias",
+                    "user_api_key_team_id": "team",
+                    "user_api_key_team_alias": "team-alias",
+                },
+            },
+            "exception": exception,
+        },
+        response_obj=None,
+        start_time=None,
+        end_time=None,
+    )
+
+    assert _model_id_values(logger.litellm_llm_api_failed_requests_metric) == {"deployment-a"}
+    assert _model_id_values(logger.litellm_deployment_failure_responses) == {"deployment-a"}
+
+
+def test_deployment_failure_model_id_falls_back_to_nested_metadata():
+    logger = PrometheusLogger()
+
+    model_id = logger._get_deployment_failure_model_id(
+        request_kwargs={
+            "litellm_params": {"litellm_metadata": {"model_info": {"id": "deployment-a"}}},
+        },
+        standard_logging_payload={},
+    )
+
+    assert model_id == "deployment-a"
+
+
+@pytest.mark.parametrize(
+    "request_kwargs",
+    [
+        {},
+        {"litellm_params": {"metadata": {"model_info": {}}}},
+    ],
+)
+def test_deployment_failure_model_id_returns_none_without_a_model_id(request_kwargs):
+    assert (
+        PrometheusLogger._get_deployment_failure_model_id(
+            request_kwargs=request_kwargs,
+            standard_logging_payload={},
+        )
+        is None
+    )
+
+
 @pytest.mark.asyncio
 async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch):
     logger = PrometheusLogger()

tests/test_litellm/test_router.py

@@ -15279,6 +15279,8 @@ async def test_async_routing_strategy_pre_call_checks_failure_logging_is_coordin
             type(hook_error),
         )
 
+    assert hook_error.failed_deployment_id == deployment["model_info"]["id"]
+
 
 @pytest.mark.asyncio
 async def test_async_callback_filter_deployments_failure_logging_is_coordinated():

tests/test_litellm/test_router/test_enforce_model_rate_limits.py

@@ -6,13 +6,14 @@ regardless of the routing strategy being used.
 """
 
 import asyncio
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
 import litellm
 from litellm import Router
 from litellm.caching.dual_cache import DualCache
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
     ModelRateLimitingCheck,
 )
@@ -353,3 +354,37 @@ class TestModelRateLimitConcurrency:
 
         assert len(successes) == 2, f"Expected 2 successes, got {len(successes)}"
         assert len(failures) == 2, f"Expected 2 rate limit errors, got {len(failures)}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "hook_error",
+    [
+        litellm.RateLimitError(message="rpm exceeded", llm_provider="openai", model="gpt-5.6"),
+        RuntimeError("pre call check blew up"),
+    ],
+)
+async def test_router_async_pre_call_checks_stamp_the_refusing_deployment(hook_error):
+    class _RaisingPreCallCheck(CustomLogger):
+        async def async_pre_call_check(self, deployment, parent_otel_span):
+            raise hook_error
+
+    router = Router(
+        model_list=[
+            {
+                "model_name": "gpt-5.6",
+                "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"},
+                "model_info": {"id": "deployment-a"},
+            }
+        ]
+    )
+    deployment = router.model_list[0]
+
+    with patch.object(litellm, "callbacks", [_RaisingPreCallCheck()]):  # test-quality-ok: router reads this global
+        with pytest.raises(type(hook_error)):
+            await router.async_routing_strategy_pre_call_checks(
+                deployment=deployment,
+                parent_otel_span=None,
+            )
+
+    assert hook_error.failed_deployment_id == "deployment-a"