diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 19d161b8aa4..d2fc588a567 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2660,7 +2660,15 @@ class PrometheusLogger(CustomLogger): if self._should_skip_metrics_for_invalid_key(kwargs=kwargs, standard_logging_payload=standard_logging_payload): return - model: Final = kwargs.get("model", "") + deployment_model_id: Final = standard_logging_payload.get("model_id", "") # pyright: ignore[reportUnknownVariableType] # StandardLoggingPayload.get is partially unknown + requested_name: Final = kwargs.get("model", "") # pyright: ignore[reportUnknownVariableType] # kwargs is an untyped callback argument + # With no deployment behind the call, the name is whatever the caller sent (for example a model the + # key may not use). Bound it like ``requested_model`` so unknown names share one series. + model: Final = ( + requested_name + if deployment_model_id + else _bounded_requested_model_label(requested_name, router_originated=True) # pyright: ignore[reportUnknownArgumentType] # caller-supplied value, the helper type-checks it + ) litellm_params: Final = kwargs.get("litellm_params", {}) or {} get_end_user_id_for_cost_tracking: Final = get_cached_end_user_id_for_cost_tracking() @@ -2682,7 +2690,7 @@ class PrometheusLogger(CustomLogger): team=user_api_team, team_alias=user_api_team_alias, user=user_id, - model_id=standard_logging_payload.get("model_id", ""), + model_id=deployment_model_id, requested_model=standard_logging_payload.get("model_group"), api_provider=standard_logging_payload.get("custom_llm_provider"), custom_metadata_labels=get_custom_labels_from_metadata( diff --git a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py index f86f4460b28..f4f0d3607e4 100644 --- a/tests/unit/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/unit/integrations/test_prometheus_requested_model_cardinality.py @@ -293,3 +293,81 @@ async def test_fallback_event_requested_model_is_bounded(router): assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == { UNRECOGNIZED_REQUESTED_MODEL_LABEL } + + +def _model_label_values(metric) -> set[str]: + index = metric._labelnames.index("model") + return {sample_key[index] for sample_key in metric._metrics} + + +async def _fire_llm_api_failure(logger: PrometheusLogger, model: str, model_id: str = "") -> None: + await logger.async_log_failure_event( + kwargs={ + "model": model, + "litellm_params": {"metadata": {}}, + "exception": _ClientSideError("model not allowed for this key"), + "standard_logging_object": { + "model_id": model_id, + "model_group": model, + "metadata": { + "user_api_key_user_id": "user-1", + "user_api_key_hash": "hashed-key-1", + "user_api_key_alias": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + }, + }, + }, + response_obj=None, + start_time=None, + end_time=None, + ) + + +@pytest.mark.asyncio +async def test_denied_model_names_collapse_to_one_series_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + for index in range(25): + await _fire_llm_api_failure(logger, f"denied-model-{index}") + + metric = logger.litellm_llm_api_failed_requests_metric + assert _model_label_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _series_count(metric) == 1 + assert _total_value(metric) == 25 + + +@pytest.mark.asyncio +async def test_deployment_model_names_keep_their_label_on_llm_api_failed_requests_metric(router): + logger = PrometheusLogger() + + with patch( + "litellm.proxy.proxy_server.llm_router", router, create=True + ): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "openai/gpt-4o-mini", model_id="deployment-1") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"openai/gpt-4o-mini"} + + +@pytest.mark.asyncio +async def test_sdk_failures_without_proxy_router_keep_their_model_label_on_llm_api_failed_requests_metric(): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, "gpt-4o-mini") + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {"gpt-4o-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [123, 4.5, ["gpt-4o-mini"], {"name": "gpt-4o-mini"}]) +async def test_non_string_denied_models_collapse_to_other_on_llm_api_failed_requests_metric(router, model: object): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await _fire_llm_api_failure(logger, model) # type: ignore[arg-type] + + assert _model_label_values(logger.litellm_llm_api_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}