diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 40621a2f68d..d663551efbb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6270,6 +6270,14 @@ def get_standard_logging_object_payload( getattr(logging_obj, "response_timing_metrics", None) or {} # mutable-ok: empty fallback ) clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms") + if clean_hidden_params["litellm_model_name"] is None: + clean_hidden_params["litellm_model_name"] = ( + getattr(logging_obj, "model", None) + or litellm_params.get("model") + or metadata.get("deployment") + or kwargs.get("model") + ) + model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information( base_model=base_model, diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index af3d7ddfac7..cdb1ced5fac 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -338,7 +338,12 @@ class ModelRateLimitingCheck(CustomLogger): return total_tokens: Final = standard_logging_object.get("total_tokens", 0) - model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + model: Final = ( + standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + or standard_logging_object.get("model") + or (kwargs.get("litellm_params") or {}).get("model") + or kwargs.get("model") + ) verbose_router_logger.debug( "[TPM TRACKING] model_id=%s, total_tokens=%s, model=%s", model_id, total_tokens, model @@ -402,7 +407,12 @@ class ModelRateLimitingCheck(CustomLogger): return total_tokens: Final = standard_logging_object.get("total_tokens", 0) - model: Final = standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + model: Final = ( + standard_logging_object.get("hidden_params", {}).get("litellm_model_name") + or standard_logging_object.get("model") + or (kwargs.get("litellm_params") or {}).get("model") + or kwargs.get("model") + ) if not model or not total_tokens: return diff --git a/tests/test_litellm/test_router/test_enforce_model_rate_limits.py b/tests/test_litellm/test_router/test_enforce_model_rate_limits.py index 1def253ac93..74adcd340a3 100644 --- a/tests/test_litellm/test_router/test_enforce_model_rate_limits.py +++ b/tests/test_litellm/test_router/test_enforce_model_rate_limits.py @@ -165,6 +165,29 @@ class TestModelRateLimitingCheck: assert "test-id:gpt-4:tpm:" in kwarg_params["key"] assert kwarg_params["value"] == 50 + def test_log_success_event_streaming_fallback_model_name(self): + """Test that log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming).""" + mock_cache = MagicMock() + check = ModelRateLimitingCheck(dual_cache=mock_cache) + + kwargs = { + "standard_logging_object": { + "model_id": "test-id", + "model": "gpt-4", + "total_tokens": 50, + "hidden_params": {"litellm_model_name": None}, + }, + "litellm_params": {"model": "gpt-4"}, + } + + check.log_success_event(kwargs, None, None, None) + + mock_cache.increment_cache.assert_called_once() + _, kwarg_params = mock_cache.increment_cache.call_args + assert "test-id:gpt-4:tpm:" in kwarg_params["key"] + assert kwarg_params["value"] == 50 + + class TestModelRateLimitingCheckAsync: """Test async methods of ModelRateLimitingCheck.""" @@ -268,6 +291,31 @@ class TestModelRateLimitingCheckAsync: assert "test-id:gpt-4:tpm:" in kwarg_params["key"] assert kwarg_params["value"] == 50 + @pytest.mark.asyncio + async def test_async_log_success_event_streaming_fallback_model_name(self): + """Test that async_log_success_event correctly increments the cache when litellm_model_name is None in hidden_params (e.g. streaming).""" + mock_cache = MagicMock() + mock_cache.async_increment_cache = AsyncMock() + check = ModelRateLimitingCheck(dual_cache=mock_cache) + + kwargs = { + "standard_logging_object": { + "model_id": "test-id", + "model": "gpt-4", + "total_tokens": 75, + "hidden_params": {"litellm_model_name": None}, + }, + "model": "gpt-4", + } + + await check.async_log_success_event(kwargs, None, None, None) + + mock_cache.async_increment_cache.assert_called_once() + _, kwarg_params = mock_cache.async_increment_cache.call_args + assert "test-id:gpt-4:tpm:" in kwarg_params["key"] + assert kwarg_params["value"] == 75 + + class TestRouterWithEnforceModelRateLimits: """Test Router integration with enforce_model_rate_limits."""