From 2adae6b4757aaaa9677785195ed68c8996a5d915 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:14:27 -0700 Subject: [PATCH] fix(prometheus): pass through router-originated labels when no proxy router exists --- litellm/integrations/prometheus.py | 20 +++++---- ..._prometheus_requested_model_cardinality.py | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 651f9c5d392..add91033ff3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -161,25 +161,27 @@ def _get_budget_metrics_per_request_timeout() -> float: def _get_proxy_llm_router() -> Router | None: try: from litellm.proxy.proxy_server import llm_router - except ImportError: + except Exception: return None return llm_router -def _bounded_requested_model_label(requested_model: str | None) -> str | None: +def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None: """ Bound ``requested_model`` label cardinality: names the router recognizes (model names, deployment ids, aliases, routing groups, team public model names) or matches via a global or team wildcard/pattern route keep their own label value; any other client-supplied string collapses into the - single ``other`` bucket. With no router to vouch for the string, it also - collapses to ``other``. + single ``other`` bucket. With no proxy router to vouch for the string, + client-supplied values collapse to ``other`` while ``router_originated`` + values (emitted by an SDK ``Router``'s own deployment failure and + fallback events, where the proxy router never exists) pass through. """ if not requested_model: return requested_model llm_router: Final = _get_proxy_llm_router() if llm_router is None: - return UNRECOGNIZED_REQUESTED_MODEL_LABEL + return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL if llm_router.is_recognized_model(requested_model): return requested_model if requested_model in llm_router.team_public_model_names: @@ -2667,7 +2669,9 @@ class PrometheusLogger(CustomLogger): label_model_id = "" label_api_base = "" label_api_provider = "" - label_requested_model = _bounded_requested_model_label(litellm_model_name or model_group) or "" + label_requested_model = ( + _bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or "" + ) enum_values: Final = UserAPIKeyLabelValues( litellm_model_name=label_litellm_model_name, @@ -3226,7 +3230,7 @@ class PrometheusLogger(CustomLogger): _tags: Final = cast(list[str], kwargs.get("tags") or []) enum_values: Final = UserAPIKeyLabelValues( - requested_model=_bounded_requested_model_label(original_model_group), + requested_model=_bounded_requested_model_label(original_model_group, router_originated=True), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], @@ -3267,7 +3271,7 @@ class PrometheusLogger(CustomLogger): ) enum_values: Final = UserAPIKeyLabelValues( - requested_model=_bounded_requested_model_label(original_model_group), + requested_model=_bounded_requested_model_label(original_model_group, router_originated=True), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py index 8d803eeab18..519a13751f1 100644 --- a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -7,6 +7,8 @@ pattern matches) into the single ``other`` label bucket, while recognized names, aliases, and wildcard-matched names keep their own label values. """ +import sys +import types from unittest.mock import patch import pytest @@ -169,6 +171,49 @@ async def test_unknown_models_collapse_to_other_when_router_is_unavailable(): } +@pytest.mark.asyncio +async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "sdk-deployment-group", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("model does not exist"), + } + ) + await logger.log_failure_fallback_event( + original_model_group="sdk-fallback-group", + kwargs={"model": "sdk-fallback-group", "metadata": {}}, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failure_responses) == {"sdk-deployment-group"} + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"} + + +@pytest.mark.asyncio +async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch): + logger = PrometheusLogger() + broken_proxy_module = types.ModuleType("litellm.proxy.proxy_server") + + def _raise_value_error(_name: str): + raise ValueError("bad proxy env var") + + broken_proxy_module.__getattr__ = _raise_value_error # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", broken_proxy_module) # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam + + await logger.log_failure_fallback_event( + original_model_group="sdk-fallback-group", + kwargs={"model": "sdk-fallback-group", "metadata": {}}, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"} + + def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router): logger = PrometheusLogger()