fix(router): enforce deployment TPM on streaming requests (#41610)

In ModelRateLimitingCheck.async_log_success_event and log_success_event, fall back to resolving model name when hidden_params.litellm_model_name is None, and populate clean_hidden_params.litellm_model_name in standard logging payload for streaming responses. Fixes #41610
This commit is contained in:
Ritik Yadav 2026-09-18 00:33:58 +05:30
parent ab03850666
commit 23e4c2c766
3 changed files with 68 additions and 2 deletions

View file

@ -6270,6 +6270,14 @@ def get_standard_logging_object_payload(
getattr(logging_obj, "response_timing_metrics", None) or {} # mutable-ok: empty fallback
)
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
if clean_hidden_params["litellm_model_name"] is None:
clean_hidden_params["litellm_model_name"] = (
getattr(logging_obj, "model", None)
or litellm_params.get("model")
or metadata.get("deployment")
or kwargs.get("model")
)
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,

View file

@ -338,7 +338,12 @@ class ModelRateLimitingCheck(CustomLogger):
return
total_tokens: Final = standard_logging_object.get("total_tokens", 0)
model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
model: Final = (
standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
or standard_logging_object.get("model")
or (kwargs.get("litellm_params") or {}).get("model")
or kwargs.get("model")
)
verbose_router_logger.debug(
"[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model
@ -402,7 +407,12 @@ class ModelRateLimitingCheck(CustomLogger):
return
total_tokens: Final = standard_logging_object.get("total_tokens", 0)
model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
model: Final = (
standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
or standard_logging_object.get("model")
or (kwargs.get("litellm_params") or {}).get("model")
or kwargs.get("model")
)
if not model or not total_tokens:
return

View file

@ -165,6 +165,29 @@ class TestModelRateLimitingCheck:
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
assert kwarg_params["value"] == 50
def test_log_success_event_streaming_fallback_model_name(self):
"""Test that log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming)."""
mock_cache = MagicMock()
check = ModelRateLimitingCheck(dual_cache=mock_cache)
kwargs = {
"standard_logging_object": {
"model_id": "test-id",
"model": "gpt-4",
"total_tokens": 50,
"hidden_params": {"litellm_model_name": None},
},
"litellm_params": {"model": "gpt-4"},
}
check.log_success_event(kwargs, None, None, None)
mock_cache.increment_cache.assert_called_once()
_, kwarg_params = mock_cache.increment_cache.call_args
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
assert kwarg_params["value"] == 50
class TestModelRateLimitingCheckAsync:
"""Test async methods of ModelRateLimitingCheck."""
@ -268,6 +291,31 @@ class TestModelRateLimitingCheckAsync:
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
assert kwarg_params["value"] == 50
@pytest.mark.asyncio
async def test_async_log_success_event_streaming_fallback_model_name(self):
"""Test that async_log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming)."""
mock_cache = MagicMock()
mock_cache.async_increment_cache = AsyncMock()
check = ModelRateLimitingCheck(dual_cache=mock_cache)
kwargs = {
"standard_logging_object": {
"model_id": "test-id",
"model": "gpt-4",
"total_tokens": 75,
"hidden_params": {"litellm_model_name": None},
},
"model": "gpt-4",
}
await check.async_log_success_event(kwargs, None, None, None)
mock_cache.async_increment_cache.assert_called_once()
_, kwarg_params = mock_cache.async_increment_cache.call_args
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
assert kwarg_params["value"] == 75
class TestRouterWithEnforceModelRateLimits:
"""Test Router integration with enforce_model_rate_limits."""