diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 6766d246894..840f3e6c5e1 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2295,6 +2295,34 @@ class PrometheusLogger(CustomLogger): _labels, ) + @staticmethod + def _get_deployment_failure_model_id( + request_kwargs: Mapping[str, object], standard_logging_payload: StandardLoggingPayload + ) -> str | None: + exception: Final = request_kwargs.get("exception") + failed_deployment_id: Final = getattr(exception, "failed_deployment_id", None) + if isinstance(failed_deployment_id, str) and failed_deployment_id: + return failed_deployment_id + + standard_model_id: Final = standard_logging_payload.get("model_id") + if standard_model_id: + return standard_model_id + + litellm_params: Final = request_kwargs.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return None + for metadata_key in ("litellm_metadata", "metadata"): + metadata = litellm_params.get(metadata_key) + if not isinstance(metadata, Mapping): + continue + model_info = metadata.get("model_info") + if not isinstance(model_info, Mapping): + continue + model_id = model_info.get("id") + if isinstance(model_id, str) and model_id: + return model_id + return None + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( "prometheus Logging - Enters failure logging function (kwargs keys: %s)", @@ -2318,6 +2346,13 @@ class PrometheusLogger(CustomLogger): user_api_team: Final = standard_logging_payload["metadata"]["user_api_key_team_id"] user_api_team_alias: Final = standard_logging_payload["metadata"]["user_api_key_team_alias"] user_api_key_org_id: Final = standard_logging_payload["metadata"].get("user_api_key_org_id") + model_id: Final = ( + self._get_deployment_failure_model_id( + request_kwargs=kwargs, + standard_logging_payload=standard_logging_payload, + ) + or "" + ) try: enum_values: Final = UserAPIKeyLabelValues( @@ -2328,7 +2363,7 @@ class PrometheusLogger(CustomLogger): team=user_api_team, team_alias=user_api_team_alias, user=user_id, - model_id=standard_logging_payload.get("model_id", ""), + model_id=model_id, custom_metadata_labels=get_custom_labels_from_metadata( metadata=_get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload=standard_logging_payload @@ -2728,17 +2763,11 @@ class PrometheusLogger(CustomLogger): litellm_model_name: Final = request_kwargs.get("model", None) model_group = standard_logging_payload.get("model_group", None) api_base: Final = standard_logging_payload.get("api_base", None) - model_id = standard_logging_payload.get("model_id", None) exception: Final = request_kwargs.get("exception", None) - - # Fallback: model_id from litellm_metadata.model_info - if model_id is None: - _model_info: Final = ( - (_litellm_params.get("litellm_metadata") or {}).get("model_info") - or (_litellm_params.get("metadata") or {}).get("model_info") - or {} - ) - model_id = _model_info.get("id") + model_id: Final = self._get_deployment_failure_model_id( + request_kwargs=request_kwargs, + standard_logging_payload=standard_logging_payload, + ) # Fallback: model_group from litellm_metadata if model_group is None: diff --git a/litellm/router.py b/litellm/router.py index 934a4ac86a9..442c66ecdc0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8555,6 +8555,7 @@ class Router: try: await _callback.async_pre_call_check(deployment, parent_otel_span) except litellm.RateLimitError as e: + self._set_failed_deployment_id_on_exception(e, deployment) ## LOG FAILURE EVENT if logging_obj is not None: asyncio.create_task( @@ -8573,6 +8574,7 @@ class Router: ) raise e except Exception as e: + self._set_failed_deployment_id_on_exception(e, deployment) ## LOG FAILURE EVENT if logging_obj is not None: asyncio.create_task( diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py index 519a13751f1..d239764b84f 100644 --- a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -83,6 +83,11 @@ def _requested_model_values(metric) -> set[str]: return {sample_key[index] for sample_key in metric._metrics} +def _model_id_values(metric) -> set[str]: + index = metric._labelnames.index("model_id") + return {sample_key[index] for sample_key in metric._metrics} + + def _series_count(metric) -> int: return len(metric._metrics) @@ -194,6 +199,55 @@ async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router(): assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"} +def test_deployment_failure_prefers_stamped_failed_deployment_id_over_mutated_metadata(): + logger = PrometheusLogger() + exception = _ClientSideError("deployment-a exceeded its TPM limit") + exception.failed_deployment_id = "deployment-a" + + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "model-group", + "litellm_params": {"metadata": {"model_info": {"id": "deployment-b"}}}, + "standard_logging_object": {"model_id": "deployment-b"}, + "exception": exception, + } + ) + + assert _model_id_values(logger.litellm_deployment_failure_responses) == {"deployment-a"} + + +@pytest.mark.asyncio +async def test_async_failure_metrics_prefer_stamped_failed_deployment_id(): + logger = PrometheusLogger() + exception = _ClientSideError("deployment-a exceeded its TPM limit") + exception.failed_deployment_id = "deployment-a" + + await logger.async_log_failure_event( + kwargs={ + "model": "model-group", + "litellm_params": {"metadata": {"model_info": {"id": "deployment-b"}}}, + "standard_logging_object": { + "model_id": "deployment-b", + "model_group": "model-group", + "metadata": { + "user_api_key_user_id": "user", + "user_api_key_hash": "hash", + "user_api_key_alias": "alias", + "user_api_key_team_id": "team", + "user_api_key_team_alias": "team-alias", + }, + }, + "exception": exception, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert _model_id_values(logger.litellm_llm_api_failed_requests_metric) == {"deployment-a"} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {"deployment-a"} + + @pytest.mark.asyncio async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch): logger = PrometheusLogger() diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index bc79c5f6589..c3009c13bab 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -14460,6 +14460,8 @@ async def test_async_routing_strategy_pre_call_checks_failure_logging_is_coordin type(hook_error), ) + assert hook_error.failed_deployment_id == deployment["model_info"]["id"] + @pytest.mark.asyncio async def test_async_callback_filter_deployments_failure_logging_is_coordinated():