mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
Merge pull request #39136 from BerriAI/litellm_lit6611_requested_model_label_cap
fix(prometheus): bound requested_model label cardinality on client failure paths
This commit is contained in:
Mateo Wang
GitHub
commit
44b595f0bb
3 changed files with 349 additions and 8 deletions
|
|
@ -59,6 +59,8 @@ from litellm.types.utils import (
|
|||
if TYPE_CHECKING:
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from prometheus_client.metrics import MetricWrapperBase
|
||||
|
||||
from litellm.router import Router
|
||||
else:
|
||||
AsyncIOScheduler = Any
|
||||
|
||||
|
|
@ -67,6 +69,8 @@ _TableRowT: Final = TypeVar("_TableRowT", bound=BaseModel)
|
|||
|
||||
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT: Final = 5.0
|
||||
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL: Final = "other"
|
||||
|
||||
_NON_ENUM_METRIC_LABELS: Final[frozenset[str]] = frozenset(
|
||||
(
|
||||
"guardrail_name",
|
||||
|
|
@ -154,6 +158,44 @@ def _get_budget_metrics_per_request_timeout() -> float:
|
|||
return parsed
|
||||
|
||||
|
||||
def _get_proxy_llm_router() -> Router | None:
|
||||
try:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
except Exception:
|
||||
return None
|
||||
return llm_router
|
||||
|
||||
|
||||
def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None:
|
||||
"""
|
||||
Bound ``requested_model`` label cardinality: names the router recognizes
|
||||
(model names, deployment ids, aliases, routing groups, team public model
|
||||
names) or matches via a global or team wildcard/pattern route keep their
|
||||
own label value; any other client-supplied string collapses into the
|
||||
single ``other`` bucket. With no proxy router to vouch for the string,
|
||||
client-supplied values collapse to ``other`` while ``router_originated``
|
||||
values (emitted by an SDK ``Router``'s own deployment failure and
|
||||
fallback events, where the proxy router never exists) pass through.
|
||||
"""
|
||||
if not requested_model:
|
||||
return requested_model
|
||||
llm_router: Final = _get_proxy_llm_router()
|
||||
if llm_router is None:
|
||||
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
if llm_router.is_recognized_model(requested_model):
|
||||
return requested_model
|
||||
if requested_model in llm_router.team_public_model_names:
|
||||
return requested_model
|
||||
if llm_router.pattern_router.route(requested_model) is not None:
|
||||
return requested_model
|
||||
if any(
|
||||
team_pattern_router.route(requested_model) is not None
|
||||
for team_pattern_router in llm_router.team_pattern_routers.values()
|
||||
):
|
||||
return requested_model
|
||||
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
|
||||
|
||||
class PrometheusLogger(CustomLogger):
|
||||
# Class variables or attributes
|
||||
|
||||
|
|
@ -2407,7 +2449,7 @@ class PrometheusLogger(CustomLogger):
|
|||
team_alias=user_api_key_dict.team_alias,
|
||||
org_id=user_api_key_dict.org_id,
|
||||
org_alias=user_api_key_dict.organization_alias,
|
||||
requested_model=request_data.get("model", ""),
|
||||
requested_model=_bounded_requested_model_label(request_data.get("model", "")),
|
||||
status_code=str(status_code),
|
||||
exception_status=str(status_code),
|
||||
exception_class=self._get_exception_class_name(original_exception),
|
||||
|
|
@ -2627,7 +2669,9 @@ class PrometheusLogger(CustomLogger):
|
|||
label_model_id = ""
|
||||
label_api_base = ""
|
||||
label_api_provider = ""
|
||||
label_requested_model = litellm_model_name or model_group or ""
|
||||
label_requested_model = (
|
||||
_bounded_requested_model_label(litellm_model_name or model_group, router_originated=True) or ""
|
||||
)
|
||||
|
||||
enum_values: Final = UserAPIKeyLabelValues(
|
||||
litellm_model_name=label_litellm_model_name,
|
||||
|
|
@ -3186,7 +3230,7 @@ class PrometheusLogger(CustomLogger):
|
|||
_tags: Final = cast(list[str], kwargs.get("tags") or [])
|
||||
|
||||
enum_values: Final = UserAPIKeyLabelValues(
|
||||
requested_model=original_model_group,
|
||||
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
|
||||
fallback_model=_new_model,
|
||||
hashed_api_key=standard_metadata["user_api_key_hash"],
|
||||
api_key_alias=standard_metadata["user_api_key_alias"],
|
||||
|
|
@ -3227,7 +3271,7 @@ class PrometheusLogger(CustomLogger):
|
|||
)
|
||||
|
||||
enum_values: Final = UserAPIKeyLabelValues(
|
||||
requested_model=original_model_group,
|
||||
requested_model=_bounded_requested_model_label(original_model_group, router_originated=True),
|
||||
fallback_model=_new_model,
|
||||
hashed_api_key=standard_metadata["user_api_key_hash"],
|
||||
api_key_alias=standard_metadata["user_api_key_alias"],
|
||||
|
|
|
|||
|
|
@ -40,6 +40,24 @@ def prometheus_logger() -> PrometheusLogger:
|
|||
return PrometheusLogger()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def known_model_router():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5-mini",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
|
||||
},
|
||||
{
|
||||
"model_name": "us/azure/openai/gpt-5-mini",
|
||||
"litellm_params": {"model": "openai/gpt-5-mini", "api_key": "fake-key"},
|
||||
},
|
||||
]
|
||||
)
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
yield router
|
||||
|
||||
|
||||
def create_standard_logging_payload() -> StandardLoggingPayload:
|
||||
return StandardLoggingPayload(
|
||||
id="test_id",
|
||||
|
|
@ -741,7 +759,7 @@ async def test_async_log_failure_event(prometheus_logger):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger):
|
||||
async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger, known_model_router):
|
||||
"""LiteLLM-side reject (no deployment picked) routes the requested model
|
||||
into `requested_model` and skips the partial-outage flag."""
|
||||
standard_logging_object = create_standard_logging_payload()
|
||||
|
|
@ -786,7 +804,7 @@ async def test_async_log_failure_event_litellm_side_rate_limit(prometheus_logger
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook(prometheus_logger):
|
||||
async def test_async_post_call_failure_hook(prometheus_logger, known_model_router):
|
||||
"""
|
||||
Test for the async_post_call_failure_hook method
|
||||
|
||||
|
|
@ -1069,7 +1087,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_success_fallback_event(prometheus_logger):
|
||||
async def test_log_success_fallback_event(prometheus_logger, known_model_router):
|
||||
prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock()
|
||||
|
||||
original_model_group = "gpt-5-mini"
|
||||
|
|
@ -1107,7 +1125,7 @@ async def test_log_success_fallback_event(prometheus_logger):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_failure_fallback_event(prometheus_logger):
|
||||
async def test_log_failure_fallback_event(prometheus_logger, known_model_router):
|
||||
prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock()
|
||||
|
||||
original_model_group = "gpt-5-mini"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,279 @@
|
|||
"""
|
||||
LIT-6611: every unique client-supplied model name that fails routing used to
|
||||
mint permanent Prometheus series carrying ``requested_model="<junk>"`` on the
|
||||
proxy request metrics and the deployment metrics, with no eviction. The fix
|
||||
collapses any requested model the router does not recognize (and no wildcard
|
||||
pattern matches) into the single ``other`` label bucket, while recognized
|
||||
names, aliases, and wildcard-matched names keep their own label values.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.prometheus import (
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
|
||||
PrometheusLogger,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
||||
class _ClientSideError(Exception):
|
||||
status_code = 400
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_prometheus_registry():
|
||||
for collector in list(REGISTRY._collector_to_names.keys()):
|
||||
try:
|
||||
REGISTRY.unregister(collector)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
yield
|
||||
|
||||
for collector in list(REGISTRY._collector_to_names.keys()):
|
||||
try:
|
||||
REGISTRY.unregister(collector)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def router():
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"},
|
||||
},
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "fake-key"},
|
||||
},
|
||||
],
|
||||
model_group_alias={"gpt4o-alias": "gpt-4o-mini"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def team_router():
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "team-internal-gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake-key"},
|
||||
"model_info": {"team_id": "team-1", "team_public_model_name": "team-alias-gpt"},
|
||||
},
|
||||
{
|
||||
"model_name": "team-internal-bedrock",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "fake-key"},
|
||||
"model_info": {"team_id": "team-1", "team_public_model_name": "team-models/*"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _requested_model_values(metric) -> set[str]:
|
||||
index = metric._labelnames.index("requested_model")
|
||||
return {sample_key[index] for sample_key in metric._metrics}
|
||||
|
||||
|
||||
def _series_count(metric) -> int:
|
||||
return len(metric._metrics)
|
||||
|
||||
|
||||
def _total_value(metric) -> float:
|
||||
return sum(child._value.get() for child in metric._metrics.values())
|
||||
|
||||
|
||||
async def _fire_proxy_failure(logger: PrometheusLogger, model: str) -> None:
|
||||
await logger.async_post_call_failure_hook(
|
||||
request_data={"model": model, "metadata": {}, "proxy_server_request": {}},
|
||||
original_exception=_ClientSideError(f"model {model} does not exist"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(router):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
for index in range(25):
|
||||
await _fire_proxy_failure(logger, f"agent-typo-{index}")
|
||||
|
||||
for metric in (
|
||||
logger.litellm_proxy_failed_requests_metric,
|
||||
logger.litellm_proxy_total_requests_metric,
|
||||
):
|
||||
assert _requested_model_values(metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL}
|
||||
assert _series_count(metric) == 1
|
||||
assert _total_value(metric) == 25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_known_alias_and_wildcard_models_keep_their_own_labels(router):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
await _fire_proxy_failure(logger, "gpt-4o-mini")
|
||||
await _fire_proxy_failure(logger, "gpt4o-alias")
|
||||
await _fire_proxy_failure(logger, "openai/gpt-4o-audio-preview")
|
||||
await _fire_proxy_failure(logger, "agent-typo-hallucinated")
|
||||
|
||||
for metric in (
|
||||
logger.litellm_proxy_failed_requests_metric,
|
||||
logger.litellm_proxy_total_requests_metric,
|
||||
):
|
||||
assert _requested_model_values(metric) == {
|
||||
"gpt-4o-mini",
|
||||
"gpt4o-alias",
|
||||
"openai/gpt-4o-audio-preview",
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_alias_and_team_wildcard_models_keep_their_own_labels(team_router):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", team_router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
await _fire_proxy_failure(logger, "team-alias-gpt")
|
||||
await _fire_proxy_failure(logger, "team-models/gpt-4o-audio-preview")
|
||||
await _fire_proxy_failure(logger, "agent-typo-hallucinated")
|
||||
|
||||
for metric in (
|
||||
logger.litellm_proxy_failed_requests_metric,
|
||||
logger.litellm_proxy_total_requests_metric,
|
||||
):
|
||||
assert _requested_model_values(metric) == {
|
||||
"team-alias-gpt",
|
||||
"team-models/gpt-4o-audio-preview",
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_models_collapse_to_other_when_router_is_unavailable():
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
await _fire_proxy_failure(logger, "agent-typo-no-router")
|
||||
await _fire_proxy_failure(logger, "gpt-4o-mini")
|
||||
|
||||
assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_router_originated_metrics_keep_labels_without_proxy_router():
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", None, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
logger.set_llm_deployment_failure_metrics(
|
||||
request_kwargs={
|
||||
"model": "sdk-deployment-group",
|
||||
"litellm_params": {"metadata": {}},
|
||||
"standard_logging_object": {},
|
||||
"exception": _ClientSideError("model does not exist"),
|
||||
}
|
||||
)
|
||||
await logger.log_failure_fallback_event(
|
||||
original_model_group="sdk-fallback-group",
|
||||
kwargs={"model": "sdk-fallback-group", "metadata": {}},
|
||||
original_exception=_ClientSideError("upstream unavailable"),
|
||||
)
|
||||
|
||||
assert _requested_model_values(logger.litellm_deployment_failure_responses) == {"sdk-deployment-group"}
|
||||
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_fallback_labels_survive_non_import_errors_from_proxy_module(monkeypatch):
|
||||
logger = PrometheusLogger()
|
||||
broken_proxy_module = types.ModuleType("litellm.proxy.proxy_server")
|
||||
|
||||
def _raise_value_error(_name: str):
|
||||
raise ValueError("bad proxy env var")
|
||||
|
||||
broken_proxy_module.__getattr__ = _raise_value_error # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam
|
||||
monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", broken_proxy_module) # test-quality-ok: reproduces a proxy_server import raising non-ImportError, no injection seam
|
||||
|
||||
await logger.log_failure_fallback_event(
|
||||
original_model_group="sdk-fallback-group",
|
||||
kwargs={"model": "sdk-fallback-group", "metadata": {}},
|
||||
original_exception=_ClientSideError("upstream unavailable"),
|
||||
)
|
||||
|
||||
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {"sdk-fallback-group"}
|
||||
|
||||
|
||||
def test_unknown_models_collapse_to_one_series_on_deployment_metrics(router):
|
||||
logger = PrometheusLogger()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
for index in range(25):
|
||||
logger.set_llm_deployment_failure_metrics(
|
||||
request_kwargs={
|
||||
"model": f"agent-typo-{index}",
|
||||
"litellm_params": {"metadata": {}},
|
||||
"standard_logging_object": {},
|
||||
"exception": _ClientSideError("model does not exist"),
|
||||
}
|
||||
)
|
||||
logger.set_llm_deployment_failure_metrics(
|
||||
request_kwargs={
|
||||
"model": "gpt-4o-mini",
|
||||
"litellm_params": {"metadata": {}},
|
||||
"standard_logging_object": {},
|
||||
"exception": _ClientSideError("all deployments cooling down"),
|
||||
}
|
||||
)
|
||||
|
||||
for metric in (
|
||||
logger.litellm_deployment_failure_responses,
|
||||
logger.litellm_deployment_total_requests,
|
||||
):
|
||||
assert _requested_model_values(metric) == {
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
|
||||
"gpt-4o-mini",
|
||||
}
|
||||
assert _series_count(metric) == 2
|
||||
assert _total_value(metric) == 26
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_event_requested_model_is_bounded(router):
|
||||
logger = PrometheusLogger()
|
||||
kwargs = {"model": "gpt-4o-mini", "metadata": {}}
|
||||
|
||||
with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam
|
||||
await logger.log_failure_fallback_event(
|
||||
original_model_group="agent-typo-hallucinated",
|
||||
kwargs=kwargs,
|
||||
original_exception=_ClientSideError("model does not exist"),
|
||||
)
|
||||
await logger.log_success_fallback_event(
|
||||
original_model_group="agent-typo-hallucinated",
|
||||
kwargs=kwargs,
|
||||
original_exception=_ClientSideError("model does not exist"),
|
||||
)
|
||||
await logger.log_failure_fallback_event(
|
||||
original_model_group="gpt-4o-mini",
|
||||
kwargs=kwargs,
|
||||
original_exception=_ClientSideError("upstream unavailable"),
|
||||
)
|
||||
|
||||
assert _requested_model_values(logger.litellm_deployment_failed_fallbacks) == {
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL,
|
||||
"gpt-4o-mini",
|
||||
}
|
||||
assert _requested_model_values(logger.litellm_deployment_successful_fallbacks) == {
|
||||
UNRECOGNIZED_REQUESTED_MODEL_LABEL
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue