mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): enforce deployment TPM on streaming requests (#41610)
In ModelRateLimitingCheck.async_log_success_event and log_success_event, fall back to resolving model name when hidden_params.litellm_model_name is None, and populate clean_hidden_params.litellm_model_name in standard logging payload for streaming responses. Fixes #41610
This commit is contained in:
parent
ab03850666
commit
23e4c2c766
3 changed files with 68 additions and 2 deletions
|
|
@ -6270,6 +6270,14 @@ def get_standard_logging_object_payload(
|
|||
getattr(logging_obj, "response_timing_metrics", None) or {} # mutable-ok: empty fallback
|
||||
)
|
||||
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
|
||||
if clean_hidden_params["litellm_model_name"] is None:
|
||||
clean_hidden_params["litellm_model_name"] = (
|
||||
getattr(logging_obj, "model", None)
|
||||
or litellm_params.get("model")
|
||||
or metadata.get("deployment")
|
||||
or kwargs.get("model")
|
||||
)
|
||||
|
||||
|
||||
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
|
||||
base_model=base_model,
|
||||
|
|
|
|||
|
|
@ -338,7 +338,12 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
return
|
||||
|
||||
total_tokens: Final = standard_logging_object.get("total_tokens", 0)
|
||||
model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
|
||||
model: Final = (
|
||||
standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
|
||||
or standard_logging_object.get("model")
|
||||
or (kwargs.get("litellm_params") or {}).get("model")
|
||||
or kwargs.get("model")
|
||||
)
|
||||
|
||||
verbose_router_logger.debug(
|
||||
"[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model
|
||||
|
|
@ -402,7 +407,12 @@ class ModelRateLimitingCheck(CustomLogger):
|
|||
return
|
||||
|
||||
total_tokens: Final = standard_logging_object.get("total_tokens", 0)
|
||||
model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
|
||||
model: Final = (
|
||||
standard_logging_object.get("hidden_params", {}).get("litellm_model_name")
|
||||
or standard_logging_object.get("model")
|
||||
or (kwargs.get("litellm_params") or {}).get("model")
|
||||
or kwargs.get("model")
|
||||
)
|
||||
|
||||
if not model or not total_tokens:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -165,6 +165,29 @@ class TestModelRateLimitingCheck:
|
|||
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
|
||||
assert kwarg_params["value"] == 50
|
||||
|
||||
def test_log_success_event_streaming_fallback_model_name(self):
|
||||
"""Test that log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming)."""
|
||||
mock_cache = MagicMock()
|
||||
check = ModelRateLimitingCheck(dual_cache=mock_cache)
|
||||
|
||||
kwargs = {
|
||||
"standard_logging_object": {
|
||||
"model_id": "test-id",
|
||||
"model": "gpt-4",
|
||||
"total_tokens": 50,
|
||||
"hidden_params": {"litellm_model_name": None},
|
||||
},
|
||||
"litellm_params": {"model": "gpt-4"},
|
||||
}
|
||||
|
||||
check.log_success_event(kwargs, None, None, None)
|
||||
|
||||
mock_cache.increment_cache.assert_called_once()
|
||||
_, kwarg_params = mock_cache.increment_cache.call_args
|
||||
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
|
||||
assert kwarg_params["value"] == 50
|
||||
|
||||
|
||||
|
||||
class TestModelRateLimitingCheckAsync:
|
||||
"""Test async methods of ModelRateLimitingCheck."""
|
||||
|
|
@ -268,6 +291,31 @@ class TestModelRateLimitingCheckAsync:
|
|||
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
|
||||
assert kwarg_params["value"] == 50
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_log_success_event_streaming_fallback_model_name(self):
|
||||
"""Test that async_log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming)."""
|
||||
mock_cache = MagicMock()
|
||||
mock_cache.async_increment_cache = AsyncMock()
|
||||
check = ModelRateLimitingCheck(dual_cache=mock_cache)
|
||||
|
||||
kwargs = {
|
||||
"standard_logging_object": {
|
||||
"model_id": "test-id",
|
||||
"model": "gpt-4",
|
||||
"total_tokens": 75,
|
||||
"hidden_params": {"litellm_model_name": None},
|
||||
},
|
||||
"model": "gpt-4",
|
||||
}
|
||||
|
||||
await check.async_log_success_event(kwargs, None, None, None)
|
||||
|
||||
mock_cache.async_increment_cache.assert_called_once()
|
||||
_, kwarg_params = mock_cache.async_increment_cache.call_args
|
||||
assert "test-id:gpt-4:tpm:" in kwarg_params["key"]
|
||||
assert kwarg_params["value"] == 75
|
||||
|
||||
|
||||
|
||||
class TestRouterWithEnforceModelRateLimits:
|
||||
"""Test Router integration with enforce_model_rate_limits."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue