scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 136

fix(prometheus): bound requested_model label cardinality on client failure paths

Browse source
This commit is contained in:
mateo-berri 2026-09-01 10:34:58 -07:00
parent 30bf592aaf
commit 3b3099d78d
3 changed files with 254 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
litellm/integrations/prometheus.py
View file

@ -59,6 +59,8 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from prometheus_client.metrics import MetricWrapperBase
from litellm.router import Router
else:
AsyncIOScheduler = Any
@ -67,6 +69,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel)
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0
UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other"
_NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset(
(
"guardrail_name",
@ -154,6 +158,34 @@ def _get_budget_metrics_per_request_timeout() -> float:
return parsed
def _get_proxy_llm_router() -> Router | None:
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
return None
return llm_router
def _bounded_requested_model_label(requested_model: str | None) -> str | None:
"""
Bound ``requested_model`` label cardinality: names the router recognizes
(model names, deployment ids, aliases, routing groups) or matches via a
wildcard/pattern route keep their own label value; any other
client-supplied string collapses into the single ``other`` bucket. With no
router to vouch for the string, it also collapses to ``other``.
"""
if not requested_model:
return requested_model
llm_router: Final = _get_proxy_llm_router()
if llm_router is None:
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
if llm_router.is_recognized_model(requested_model):
return requested_model
if llm_router.pattern_router.route(requested_model) is not None:
return requested_model
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -2407,7 +2439,7 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=request_data.get("model", ""),
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
@ -2627,7 +2659,7 @@ class PrometheusLogger(CustomLogger):
label_model_id = ""
label_api_base = ""
label_api_provider = ""
label_requested_model = litellm_model_name or model_group or ""
label_requested_model = _bounded_requested_model_label(litellm_model_name or model_group) or ""
enum_values: Final = UserAPIKeyLabelValues(
litellm_model_name=label_litellm_model_name,
@ -3186,7 +3218,7 @@ class PrometheusLogger(CustomLogger):
_tags: Final = cast(list[str], kwargs.get("tags") or [])
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],
@ -3227,7 +3259,7 @@ class PrometheusLogger(CustomLogger):
)
enum_values: Final = UserAPIKeyLabelValues(
requested_model=original_model_group,
requested_model=_bounded_requested_model_label(original_model_group),
fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"],
api_key_alias=standard_metadata["user_api_key_alias"],

26
tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file

@ -40,6 +40,24 @@ def prometheus_logger() -> PrometheusLogger:
return PrometheusLogger()
@pytest.fixture
def known_model_router():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-5-mini",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
},
{
"model_name": "us/azure/openai/gpt-5-mini",
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
},
]
)
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
yield router
def create_standard_logging_payload() -> StandardLoggingPayload:
return StandardLoggingPayload(
id="test_id",
@ -741,7 +759,7 @@ async def test_async_log_failure_event(prometheus_logger):
@pytest.mark.asyncio
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger):
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger, known_model_router):
"""LiteLLM-side reject (no deployment picked) routes the requested model
into `requested_model` and skips the partial-outage flag."""
standard_logging_object = create_standard_logging_payload()
@ -786,7 +804,7 @@ async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger
@pytest.mark.asyncio
async def test_async_post_call_failure_hook(prometheus_logger):
async def test_async_post_call_failure_hook(prometheus_logger, known_model_router):
"""
Test for the async_post_call_failure_hook method
@ -1069,7 +1087,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
@pytest.mark.asyncio
async def test_log_success_fallback_event(prometheus_logger):
async def test_log_success_fallback_event(prometheus_logger, known_model_router):
prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock()
original_model_group = "gpt-5-mini"
@ -1107,7 +1125,7 @@ async def test_log_success_fallback_event(prometheus_logger):
@pytest.mark.asyncio
async def test_log_failure_fallback_event(prometheus_logger):
async def test_log_failure_fallback_event(prometheus_logger, known_model_router):
prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock()
original_model_group = "gpt-5-mini"

196
tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py Normal file
View file

@ -0,0 +1,196 @@
"""
LIT-6611: every unique client-supplied model name that fails routing used to
mint permanent Prometheus series carrying ``requested_model="<junk>"`` on the
proxy request metrics and the deployment metrics, with no eviction. The fix
collapses any requested model the router does not recognize (and no wildcard
pattern matches) into the single ``other`` label bucket, while recognized
names, aliases, and wildcard-matched names keep their own label values.
"""
from unittest.mock import patch
import pytest
from prometheus_client import REGISTRY
import litellm
from litellm.integrations.prometheus import (
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
PrometheusLogger,
)
from litellm.proxy._types import UserAPIKeyAuth
class _ClientSideError(Exception):
status_code = 400
@pytest.fixture(autouse=True)
def cleanup_prometheus_registry():
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
yield
for collector in list(REGISTRY._collector_to_names.keys()):
try:
REGISTRY.unregister(collector)
except Exception:
pass
@pytest.fixture
def router():
return litellm.Router(
model_list=[
{
"model_name": "gpt-4o-mini",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"},
},
{
"model_name": "openai/*",
"litellm_params": {"model": "openai/*", "api_key": "fake-key"},
},
],
model_group_alias={"gpt4o-alias": "gpt-4o-mini"},
)
def _requested_model_values(metric) -> set[str]:
index = metric._labelnames.index("requested_model")
return {sample_key[index] for sample_key in metric._metrics}
def _series_count(metric) -> int:
return len(metric._metrics)
def _total_value(metric) -> float:
return sum(child._value.get() for child in metric._metrics.values())
async def _fire_proxy_failure(logger: PrometheusLogger, model: str) -> None:
await logger.async_post_call_failure_hook(
request_data={"model": model, "metadata": {}, "proxy_server_request": {}},
original_exception=_ClientSideError(f"model {model} does not exist"),
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"),
)
@pytest.mark.asyncio
async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
for index in range(25):
await _fire_proxy_failure(logger, f"agent-typo-{index}")
for metric in (
logger.litellm_proxy_failed_requests_metric,
logger.litellm_proxy_total_requests_metric,
):
assert _requested_model_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}
assert _series_count(metric) == 1
assert _total_value(metric) == 25
@pytest.mark.asyncio
async def test_known_alias_and_wildcard_models_keep_their_own_labels(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await _fire_proxy_failure(logger, "gpt-4o-mini")
await _fire_proxy_failure(logger, "gpt4o-alias")
await _fire_proxy_failure(logger, "openai/gpt-4o-audio-preview")
await _fire_proxy_failure(logger, "agent-typo-hallucinated")
for metric in (
logger.litellm_proxy_failed_requests_metric,
logger.litellm_proxy_total_requests_metric,
):
assert _requested_model_values(metric) == {
"gpt-4o-mini",
"gpt4o-alias",
"openai/gpt-4o-audio-preview",
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
}
@pytest.mark.asyncio
async def test_unknown_models_collapse_to_other_when_router_is_unavailable():
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await _fire_proxy_failure(logger, "agent-typo-no-router")
await _fire_proxy_failure(logger, "gpt-4o-mini")
assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL
}
def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router):
logger = PrometheusLogger()
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
for index in range(25):
logger.set_llm_deployment_failure_metrics(
request_kwargs={
"model": f"agent-typo-{index}",
"litellm_params": {"metadata": {}},
"standard_logging_object": {},
"exception": _ClientSideError("model does not exist"),
}
)
logger.set_llm_deployment_failure_metrics(
request_kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {}},
"standard_logging_object": {},
"exception": _ClientSideError("all deployments cooling down"),
}
)
for metric in (
logger.litellm_deployment_failure_responses,
logger.litellm_deployment_total_requests,
):
assert _requested_model_values(metric) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
"gpt-4o-mini",
}
assert _series_count(metric) == 2
assert _total_value(metric) == 26
@pytest.mark.asyncio
async def test_fallback_event_requested_model_is_bounded(router):
logger = PrometheusLogger()
kwargs = {"model": "gpt-4o-mini", "metadata": {}}
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
await logger.log_failure_fallback_event(
original_model_group="agent-typo-hallucinated",
kwargs=kwargs,
original_exception=_ClientSideError("model does not exist"),
)
await logger.log_success_fallback_event(
original_model_group="agent-typo-hallucinated",
kwargs=kwargs,
original_exception=_ClientSideError("model does not exist"),
)
await logger.log_failure_fallback_event(
original_model_group="gpt-4o-mini",
kwargs=kwargs,
original_exception=_ClientSideError("upstream unavailable"),
)
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
"gpt-4o-mini",
}
assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == {
UNRECOGNIZED_REQUESTED_MODEL_LABEL
}