From 3b3099d78dedea9bd576bdcb2dfecabece8c5099 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:34:58 -0700 Subject: [PATCH] fix(prometheus): bound requested_model label cardinality on client failure paths --- litellm/integrations/prometheus.py | 40 +++- .../test_prometheus_logging_callbacks.py | 26 ++- ..._prometheus_requested_model_cardinality.py | 196 ++++++++++++++++++ 3 files changed, 254 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 467ec72dc4a..29b7419a91b 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -59,6 +59,8 @@ from litellm.types.utils import ( if TYPE_CHECKING: from apscheduler.schedulers.asyncio import AsyncIOScheduler from prometheus_client.metrics import MetricWrapperBase + + from litellm.router import Router else: AsyncIOScheduler = Any @@ -67,6 +69,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel) _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0 +UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other" + _NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset( ( "guardrail_name", @@ -154,6 +158,34 @@ def _get_budget_metrics_per_request_timeout() -> float: return parsed +def _get_proxy_llm_router() -> Router | None: + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return None + return llm_router + + +def _bounded_requested_model_label(requested_model: str | None) -> str | None: + """ + Bound ``requested_model`` label cardinality: names the router recognizes + (model names, deployment ids, aliases, routing groups) or matches via a + wildcard/pattern route keep their own label value; any other + client-supplied string collapses into the single ``other`` bucket. With no + router to vouch for the string, it also collapses to ``other``. + """ + if not requested_model: + return requested_model + llm_router: Final = _get_proxy_llm_router() + if llm_router is None: + return UNRECOGNIZED_REQUESTED_MODEL_LABEL + if llm_router.is_recognized_model(requested_model): + return requested_model + if llm_router.pattern_router.route(requested_model) is not None: + return requested_model + return UNRECOGNIZED_REQUESTED_MODEL_LABEL + + class PrometheusLogger(CustomLogger): # Class variables or attributes @@ -2407,7 +2439,7 @@ class PrometheusLogger(CustomLogger): team_alias=user_api_key_dict.team_alias, org_id=user_api_key_dict.org_id, org_alias=user_api_key_dict.organization_alias, - requested_model=request_data.get("model", ""), + requested_model=_bounded_requested_model_label(request_data.get("model", "")), status_code=str(status_code), exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), @@ -2627,7 +2659,7 @@ class PrometheusLogger(CustomLogger): label_model_id = "" label_api_base = "" label_api_provider = "" - label_requested_model = litellm_model_name or model_group or "" + label_requested_model = _bounded_requested_model_label(litellm_model_name or model_group) or "" enum_values: Final = UserAPIKeyLabelValues( litellm_model_name=label_litellm_model_name, @@ -3186,7 +3218,7 @@ class PrometheusLogger(CustomLogger): _tags: Final = cast(list[str], kwargs.get("tags") or []) enum_values: Final = UserAPIKeyLabelValues( - requested_model=original_model_group, + requested_model=_bounded_requested_model_label(original_model_group), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], @@ -3227,7 +3259,7 @@ class PrometheusLogger(CustomLogger): ) enum_values: Final = UserAPIKeyLabelValues( - requested_model=original_model_group, + requested_model=_bounded_requested_model_label(original_model_group), fallback_model=_new_model, hashed_api_key=standard_metadata["user_api_key_hash"], api_key_alias=standard_metadata["user_api_key_alias"], diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 05886e4b7f6..58cde4c8103 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -40,6 +40,24 @@ def prometheus_logger() -> PrometheusLogger: return PrometheusLogger() +@pytest.fixture +def known_model_router(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + }, + { + "model_name": "us/azure/openai/gpt-5-mini", + "litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + }, + ] + ) + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + yield router + + def create_standard_logging_payload() -> StandardLoggingPayload: return StandardLoggingPayload( id="test_id", @@ -741,7 +759,7 @@ async def test_async_log_failure_event(prometheus_logger): @pytest.mark.asyncio -async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger): +async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger, known_model_router): """LiteLLM-side reject (no deployment picked) routes the requested model into `requested_model` and skips the partial-outage flag.""" standard_logging_object = create_standard_logging_payload() @@ -786,7 +804,7 @@ async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger @pytest.mark.asyncio -async def test_async_post_call_failure_hook(prometheus_logger): +async def test_async_post_call_failure_hook(prometheus_logger, known_model_router): """ Test for the async_post_call_failure_hook method @@ -1069,7 +1087,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): @pytest.mark.asyncio -async def test_log_success_fallback_event(prometheus_logger): +async def test_log_success_fallback_event(prometheus_logger, known_model_router): prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock() original_model_group = "gpt-5-mini" @@ -1107,7 +1125,7 @@ async def test_log_success_fallback_event(prometheus_logger): @pytest.mark.asyncio -async def test_log_failure_fallback_event(prometheus_logger): +async def test_log_failure_fallback_event(prometheus_logger, known_model_router): prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock() original_model_group = "gpt-5-mini" diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py new file mode 100644 index 00000000000..2343384e762 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -0,0 +1,196 @@ +""" +LIT-6611: every unique client-supplied model name that fails routing used to +mint permanent Prometheus series carrying ``requested_model=""`` on the +proxy request metrics and the deployment metrics, with no eviction. The fix +collapses any requested model the router does not recognize (and no wildcard +pattern matches) into the single ``other`` label bucket, while recognized +names, aliases, and wildcard-matched names keep their own label values. +""" + +from unittest.mock import patch + +import pytest +from prometheus_client import REGISTRY + +import litellm +from litellm.integrations.prometheus import ( + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + PrometheusLogger, +) +from litellm.proxy._types import UserAPIKeyAuth + + +class _ClientSideError(Exception): + status_code = 400 + + +@pytest.fixture(autouse=True) +def cleanup_prometheus_registry(): + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + yield + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +@pytest.fixture +def router(): + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake-key"}, + }, + ], + model_group_alias={"gpt4o-alias": "gpt-4o-mini"}, + ) + + +def _requested_model_values(metric) -> set[str]: + index = metric._labelnames.index("requested_model") + return {sample_key[index] for sample_key in metric._metrics} + + +def _series_count(metric) -> int: + return len(metric._metrics) + + +def _total_value(metric) -> float: + return sum(child._value.get() for child in metric._metrics.values()) + + +async def _fire_proxy_failure(logger: PrometheusLogger, model: str) -> None: + await logger.async_post_call_failure_hook( + request_data={"model": model, "metadata": {}, "proxy_server_request": {}}, + original_exception=_ClientSideError(f"model {model} does not exist"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"), + ) + + +@pytest.mark.asyncio +async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + await _fire_proxy_failure(logger, f"agent-typo-{index}") + + for metric in ( + logger.litellm_proxy_failed_requests_metric, + logger.litellm_proxy_total_requests_metric, + ): + assert _requested_model_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _series_count(metric) == 1 + assert _total_value(metric) == 25 + + +@pytest.mark.asyncio +async def test_known_alias_and_wildcard_models_keep_their_own_labels(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_proxy_failure(logger, "gpt-4o-mini") + await _fire_proxy_failure(logger, "gpt4o-alias") + await _fire_proxy_failure(logger, "openai/gpt-4o-audio-preview") + await _fire_proxy_failure(logger, "agent-typo-hallucinated") + + for metric in ( + logger.litellm_proxy_failed_requests_metric, + logger.litellm_proxy_total_requests_metric, + ): + assert _requested_model_values(metric) == { + "gpt-4o-mini", + "gpt4o-alias", + "openai/gpt-4o-audio-preview", + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + } + + +@pytest.mark.asyncio +async def test_unknown_models_collapse_to_other_when_router_is_unavailable(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_proxy_failure(logger, "agent-typo-no-router") + await _fire_proxy_failure(logger, "gpt-4o-mini") + + assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL + } + + +def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": f"agent-typo-{index}", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("model does not exist"), + } + ) + logger.set_llm_deployment_failure_metrics( + request_kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {}, + "exception": _ClientSideError("all deployments cooling down"), + } + ) + + for metric in ( + logger.litellm_deployment_failure_responses, + logger.litellm_deployment_total_requests, + ): + assert _requested_model_values(metric) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + "gpt-4o-mini", + } + assert _series_count(metric) == 2 + assert _total_value(metric) == 26 + + +@pytest.mark.asyncio +async def test_fallback_event_requested_model_is_bounded(router): + logger = PrometheusLogger() + kwargs = {"model": "gpt-4o-mini", "metadata": {}} + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await logger.log_failure_fallback_event( + original_model_group="agent-typo-hallucinated", + kwargs=kwargs, + original_exception=_ClientSideError("model does not exist"), + ) + await logger.log_success_fallback_event( + original_model_group="agent-typo-hallucinated", + kwargs=kwargs, + original_exception=_ClientSideError("model does not exist"), + ) + await logger.log_failure_fallback_event( + original_model_group="gpt-4o-mini", + kwargs=kwargs, + original_exception=_ClientSideError("upstream unavailable"), + ) + + assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL, + "gpt-4o-mini", + } + assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == { + UNRECOGNIZED_REQUESTED_MODEL_LABEL + }