diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py
index a00f491e621..23b6920972a 100644
--- a/litellm/integrations/SlackAlerting/slack_alerting.py
+++ b/litellm/integrations/SlackAlerting/slack_alerting.py
@@ -93,17 +93,12 @@ def _budget_alert_email_html(webhook_event: WebhookEvent, email_logo_url: str, e
Best,
The LiteLLM team
"""
- if webhook_event.event == "soft_budget_crossed":
- hard_limit: Final = (
- f" until your budget of ${webhook_event.max_budget} is reached"
- if webhook_event.max_budget is not None
- else ""
- )
+ if webhook_event.event == "soft_budget_crossed" and webhook_event.event_group == Litellm_EntityType.USER:
return f"""
Hi {user_id},
Your LLM API spend of ${webhook_event.spend} has reached your soft budget of ${webhook_event.soft_budget} .
- This is a warning only. API requests keep going through{hard_limit}.
+ This is a warning only. API requests keep going through.
If you have any questions, please send an email to {email_support_contact}
Best,
The LiteLLM team
diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py
index bb83ff6e7e8..d6166b9821a 100644
--- a/litellm/proxy/management_endpoints/internal_user_endpoints.py
+++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py
@@ -305,6 +305,11 @@ def _validate_user_soft_budget(soft_budget: float | None, max_budget: float | No
)
+def _validate_updated_user_soft_budget(updates: _UserBudgetRow, stored: _UserBudgetRow) -> None:
+ budgets: Final = stored.model_copy(update=updates.model_dump(exclude_unset=True))
+ _validate_user_soft_budget(soft_budget=budgets.soft_budget, max_budget=budgets.max_budget)
+
+
async def _check_duplicate_user_field(
field_name: str,
field_value: str | None,
@@ -1642,14 +1647,14 @@ async def _update_single_user_helper(
existing_user_row = await _user_table(prisma_client).find_first(where={"user_email": user_request.user_email})
_check_user_update_authz(user_request, user_api_key_dict, existing_user_row)
+ if "soft_budget" in non_default_values or "max_budget" in non_default_values:
+ _validate_updated_user_soft_budget(
+ _UserBudgetRow.model_validate(non_default_values),
+ _UserBudgetRow.model_validate(existing_user_row) if existing_user_row is not None else _UserBudgetRow(),
+ )
if existing_user_row is not None:
existing_user_row = LiteLLM_UserTable.model_validate(existing_user_row.model_dump(exclude_none=True))
- requested: Final = user_request.model_fields_set
- _validate_user_soft_budget(
- soft_budget=user_request.soft_budget if "soft_budget" in requested else existing_user_row.soft_budget,
- max_budget=user_request.max_budget if "max_budget" in requested else existing_user_row.max_budget,
- )
if (
"budget_reset_at" in non_default_values
and isinstance(non_default_values["budget_duration"], str)
@@ -2128,16 +2133,7 @@ async def bulk_user_update(
if "soft_budget" in non_default_values or "max_budget" in non_default_values:
for user_budget in _USER_BUDGET_ROWS.validate_python(all_users_in_db, from_attributes=True):
- _validate_user_soft_budget(
- soft_budget=(
- data.user_updates.soft_budget
- if "soft_budget" in non_default_values
- else user_budget.soft_budget
- ),
- max_budget=(
- data.user_updates.max_budget if "max_budget" in non_default_values else user_budget.max_budget
- ),
- )
+ _validate_updated_user_soft_budget(_UserBudgetRow.model_validate(non_default_values), user_budget)
shared_values: Final = MappingProxyType({k: v for k, v in non_default_values.items() if k != "budget_limits"})
diff --git a/tests/integration/observability/test_user_soft_budget_alert.py b/tests/integration/observability/test_user_soft_budget_alert.py
index 2d088dde52f..f76b2c0583a 100644
--- a/tests/integration/observability/test_user_soft_budget_alert.py
+++ b/tests/integration/observability/test_user_soft_budget_alert.py
@@ -103,7 +103,7 @@ def test_user_over_soft_budget_gets_alert_and_request_still_succeeds(
assert response["choices"], response
(alert, *_) = alerts_naming(alerting.alerts, user_id, seconds=ALERT_SECONDS)
assert "Soft Budget Crossed" in alert, alert
- assert "soft_budget: 0.5" in alert, alert
+ assert "*soft_budget:* `0.5`" in alert, alert
def test_user_under_soft_budget_gets_no_alert(alerting: AlertingProxy) -> None:
diff --git a/tests/unit/integrations/SlackAlerting/test_slack_alerting.py b/tests/unit/integrations/SlackAlerting/test_slack_alerting.py
index 89442bdd239..3396d25c3d0 100644
--- a/tests/unit/integrations/SlackAlerting/test_slack_alerting.py
+++ b/tests/unit/integrations/SlackAlerting/test_slack_alerting.py
@@ -621,8 +621,9 @@ async def test_daily_report_schedule_cache_calls_declare_their_key_family():
@pytest.fixture
def slack_alerting() -> SlackAlerting:
- return SlackAlerting(alerting_threshold=1, internal_usage_cache=DualCache(), alerting=["slack"])
-
+ return SlackAlerting(
+ alerting_threshold=1, internal_usage_cache=DualCache(), alerting=["slack"]
+ )
key_info: Final = CallInfo(
token="test_token",
@@ -664,7 +665,6 @@ key_no_max_budget_info: Final = CallInfo(
event_group=Litellm_EntityType.KEY,
)
-
@pytest.mark.parametrize(
"model, optional_params, expected_api_base",
[
@@ -701,10 +701,11 @@ async def test_response_taking_too_long_callback(slack_alerting):
end_time = start_time + timedelta(seconds=301)
kwargs = {"model": "test_model", "messages": "test_messages", "litellm_params": {}}
with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
- await slack_alerting.response_taking_too_long_callback(kwargs, None, start_time, end_time)
+ await slack_alerting.response_taking_too_long_callback(
+ kwargs, None, start_time, end_time
+ )
mock_send_alert.assert_awaited_once()
-
@pytest.mark.asyncio
async def test_alerting_metadata(slack_alerting):
"""
@@ -718,13 +719,15 @@ async def test_alerting_metadata(slack_alerting):
"litellm_params": {"metadata": {"alerting_metadata": {"hello": "world"}}},
}
with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
+
## RESPONSE TAKING TOO LONG
- await slack_alerting.response_taking_too_long_callback(kwargs, None, start_time, end_time)
+ await slack_alerting.response_taking_too_long_callback(
+ kwargs, None, start_time, end_time
+ )
mock_send_alert.assert_awaited_once()
assert "hello" in mock_send_alert.call_args[1]["alerting_metadata"]
-
@pytest.mark.asyncio
async def test_budget_alerts_crossed(slack_alerting):
user_max_budget = 100
@@ -741,7 +744,6 @@ async def test_budget_alerts_crossed(slack_alerting):
)
mock_send_alert.assert_awaited_once()
-
@pytest.mark.asyncio
async def test_budget_alerts_crossed_again(slack_alerting):
user_max_budget = 100
@@ -769,7 +771,6 @@ async def test_budget_alerts_crossed_again(slack_alerting):
)
mock_send_alert.assert_not_awaited()
-
@pytest.mark.asyncio
async def test_daily_reports_unit_test(slack_alerting):
with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
@@ -789,7 +790,9 @@ async def test_daily_reports_unit_test(slack_alerting):
updated_at=litellm.utils.get_utc_datetime(),
)
- updated_val = await slack_alerting.async_update_daily_reports(deployment_metrics=deployment_metrics)
+ updated_val = await slack_alerting.async_update_daily_reports(
+ deployment_metrics=deployment_metrics
+ )
assert updated_val == 1
@@ -797,7 +800,6 @@ async def test_daily_reports_unit_test(slack_alerting):
mock_send_alert.assert_awaited_once()
-
@pytest.mark.asyncio
async def test_send_daily_reports_ignores_zero_values():
router = MagicMock()
@@ -805,7 +807,9 @@ async def test_send_daily_reports_ignores_zero_values():
slack_alerting = SlackAlerting(internal_usage_cache=MagicMock())
# model1:failed=None, model2:failed=0, model3:failed=10, model1:latency=0; model2:latency=0; model3:latency=None
- slack_alerting.internal_usage_cache.async_batch_get_cache = AsyncMock(return_value=[None, 0, 10, 0, 0, None])
+ slack_alerting.internal_usage_cache.async_batch_get_cache = AsyncMock(
+ return_value=[None, 0, 10, 0, 0, None]
+ )
slack_alerting.internal_usage_cache.async_set_cache_pipeline = AsyncMock()
router.registry.return_value.deployment_dict.side_effect = lambda x: {"litellm_params": {"model": x}}
@@ -824,14 +828,15 @@ async def test_send_daily_reports_ignores_zero_values():
assert result == True
-
@pytest.mark.asyncio
async def test_send_daily_reports_all_zero_or_none():
router = MagicMock()
router.registry.return_value.names.return_value = ["model1", "model2", "model3"]
slack_alerting = SlackAlerting(internal_usage_cache=MagicMock())
- slack_alerting.internal_usage_cache.async_batch_get_cache = AsyncMock(return_value=[None, 0, None, 0, None, 0])
+ slack_alerting.internal_usage_cache.async_batch_get_cache = AsyncMock(
+ return_value=[None, 0, None, 0, None, 0]
+ )
with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
result = await slack_alerting.send_daily_reports(router)
@@ -841,7 +846,6 @@ async def test_send_daily_reports_all_zero_or_none():
assert result == False
-
@pytest.mark.parametrize(
"alerting_type",
[
@@ -879,7 +883,6 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
)
mock_send_alert.assert_awaited_once()
-
@pytest.mark.parametrize(
"alerting_type",
[
@@ -895,7 +898,9 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
async def test_webhook_alerting(alerting_type):
slack_alerting = SlackAlerting(alerting=["webhook"])
- with patch.object(slack_alerting, "send_webhook_alert", new=AsyncMock()) as mock_send_alert:
+ with patch.object(
+ slack_alerting, "send_webhook_alert", new=AsyncMock()
+ ) as mock_send_alert:
user_info = {
"token": "sk-test-mock-token-606",
"spend": 1,
@@ -916,7 +921,6 @@ async def test_webhook_alerting(alerting_type):
)
mock_send_alert.assert_awaited_once()
-
@pytest.mark.parametrize(
"model, api_base, llm_provider, vertex_project, vertex_location",
[
@@ -933,7 +937,9 @@ async def test_webhook_alerting(alerting_type):
)
@pytest.mark.parametrize("error_code", [500, 408, 400])
@pytest.mark.asyncio
-async def test_outage_alerting_called(model, api_base, llm_provider, vertex_project, vertex_location, error_code):
+async def test_outage_alerting_called(
+ model, api_base, llm_provider, vertex_project, vertex_location, error_code
+):
"""
If call fails, outage alert is called
@@ -954,7 +960,9 @@ async def test_outage_alerting_called(model, api_base, llm_provider, vertex_proj
)
elif error_code == 408:
print("RAISING 408 ERROR CODE")
- error_to_raise = litellm.Timeout(message="A timeout occurred", model=model, llm_provider=llm_provider)
+ error_to_raise = litellm.Timeout(
+ message="A timeout occurred", model=model, llm_provider=llm_provider
+ )
elif error_code == 500:
print("RAISING 500 ERROR CODE")
error_to_raise = litellm.ServiceUnavailableError(
@@ -988,7 +996,9 @@ async def test_outage_alerting_called(model, api_base, llm_provider, vertex_proj
)
slack_alerting.update_values(llm_router=router)
- with patch.object(slack_alerting, "outage_alerts", new=AsyncMock()) as mock_outage_alert:
+ with patch.object(
+ slack_alerting, "outage_alerts", new=AsyncMock()
+ ) as mock_outage_alert:
try:
await router.acompletion(
model=model,
@@ -1016,7 +1026,6 @@ async def test_outage_alerting_called(model, api_base, llm_provider, vertex_proj
else:
mock_send_alert.assert_not_called()
-
@pytest.mark.parametrize(
"model, api_base, llm_provider, vertex_project, vertex_location",
[
@@ -1041,7 +1050,9 @@ async def test_region_outage_alerting_called(
If multiple calls fail, outage alert is sent
"""
- slack_alerting = SlackAlerting(alerting=["webhook"], alert_types=[AlertType.region_outage_alerts])
+ slack_alerting = SlackAlerting(
+ alerting=["webhook"], alert_types=[AlertType.region_outage_alerts]
+ )
litellm.callbacks = [slack_alerting]
@@ -1056,7 +1067,9 @@ async def test_region_outage_alerting_called(
)
elif error_code == 408:
print("RAISING 408 ERROR CODE")
- error_to_raise = litellm.Timeout(message="A timeout occurred", model=model, llm_provider=llm_provider)
+ error_to_raise = litellm.Timeout(
+ message="A timeout occurred", model=model, llm_provider=llm_provider
+ )
elif error_code == 500:
print("RAISING 500 ERROR CODE")
error_to_raise = litellm.ServiceUnavailableError(
@@ -1109,15 +1122,13 @@ async def test_region_outage_alerting_called(
else:
deployment_id = "2"
await slack_alerting.region_outage_alerts(
- exception=error_to_raise,
- deployment_id=deployment_id, # type: ignore
+ exception=error_to_raise, deployment_id=deployment_id # type: ignore
)
if model == "gemini-3.8-flash" and (error_code == 500 or error_code == 408):
mock_send_alert.assert_called_once()
else:
mock_send_alert.assert_not_called()
-
@pytest.mark.asyncio
async def test_print_alerting_payload_warning():
"""
@@ -1147,7 +1158,9 @@ async def test_print_alerting_payload_warning():
test_payload = {"text": "Test alert message"}
# Send an alert
- with patch.object(slack_alerting.async_http_handler, "post", new=AsyncMock()) as mock_post:
+ with patch.object(
+ slack_alerting.async_http_handler, "post", new=AsyncMock()
+ ) as mock_post:
await send_to_webhook(
slackAlertingInstance=slack_alerting,
item={
@@ -1167,7 +1180,6 @@ async def test_print_alerting_payload_warning():
verbose_proxy_logger.removeHandler(handler)
log_stream.close()
-
@pytest.mark.asyncio
async def test_soft_budget_alerts():
"""
@@ -1211,7 +1223,6 @@ async def test_soft_budget_alerts():
)
assert alert_message == expected_message
-
@pytest.mark.parametrize(
"entity_info",
[
@@ -1276,10 +1287,29 @@ def test_soft_budget_crossed_email_names_spend_and_threshold() -> None:
assert "$52.5" in html
assert "soft budget of $50" in html
- assert "until your budget of $100" in html
assert "will be rejected" not in html
+@pytest.mark.parametrize("event_group", [Litellm_EntityType.KEY, Litellm_EntityType.TEAM])
+def test_soft_budget_crossed_email_for_keys_and_teams_keeps_generic_body(event_group: Litellm_EntityType) -> None:
+ from litellm.integrations.SlackAlerting.slack_alerting import _budget_alert_email_html
+ from litellm.proxy._types import WebhookEvent
+
+ event: Final = WebhookEvent(
+ spend=6,
+ soft_budget=5,
+ token="hashed-token",
+ event="soft_budget_crossed",
+ event_group=event_group,
+ event_message="Soft budget crossed",
+ )
+ html: Final = _budget_alert_email_html(
+ webhook_event=event, email_logo_url="https://logo", email_support_contact="support@example.com"
+ )
+
+ assert html == "Alert from LiteLLM Server"
+
+
def test_budget_crossed_email_keeps_the_max_budget_body() -> None:
from litellm.integrations.SlackAlerting.slack_alerting import _budget_alert_email_html
from litellm.proxy._types import Litellm_EntityType, WebhookEvent
diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py
index aa9b4366be7..00c09ea3fb7 100644
--- a/tests/unit/proxy/auth/test_auth_checks.py
+++ b/tests/unit/proxy/auth/test_auth_checks.py
@@ -163,15 +163,12 @@ async def test_team_require_trace_id_accepts_supported_trace_carriers(
@pytest.mark.parametrize("route", ["/v1/traces", "/v1/logs", "/v1/traces/query"])
@pytest.mark.asyncio
async def test_team_require_trace_id_accepts_trace_telemetry_routes(route: str) -> None:
- assert (
- await _run_team_trace_id_check(
- route=route,
- method="POST",
- request_body={},
- team_metadata={"require_trace_id": True},
- )
- is True
- )
+ assert await _run_team_trace_id_check(
+ route=route,
+ method="POST",
+ request_body={},
+ team_metadata={"require_trace_id": True},
+ ) is True
@pytest.mark.asyncio
@@ -203,14 +200,11 @@ async def test_team_require_trace_id_rejects_overridden_body_metadata_on_chat()
async def test_team_require_trace_id_accepts_effective_metadata_on_responses_route(
request_body: dict[str, object],
) -> None:
- assert (
- await _run_team_trace_id_check(
- route="/v1/responses",
- request_body=request_body,
- team_metadata={"require_trace_id": True},
- )
- is True
- )
+ assert await _run_team_trace_id_check(
+ route="/v1/responses",
+ request_body=request_body,
+ team_metadata={"require_trace_id": True},
+ ) is True
@pytest.mark.parametrize(
@@ -249,15 +243,12 @@ async def test_team_require_trace_id_rejects_fallback_carriers_when_litellm_trac
@pytest.mark.asyncio
async def test_team_require_trace_id_accepts_header_on_mcp_with_empty_body() -> None:
- assert (
- await _run_team_trace_id_check(
- route="/mcp/",
- request_body={},
- headers={"x-litellm-trace-id": "trace-mcp-123"},
- team_metadata={"require_trace_id": True},
- )
- is True
- )
+ assert await _run_team_trace_id_check(
+ route="/mcp/",
+ request_body={},
+ headers={"x-litellm-trace-id": "trace-mcp-123"},
+ team_metadata={"require_trace_id": True},
+ ) is True
@pytest.mark.parametrize(
@@ -290,15 +281,12 @@ async def test_team_require_trace_id_rejects_nonheader_carriers_on_pass_through_
@pytest.mark.asyncio
async def test_team_require_trace_id_accepts_header_on_pass_through_route() -> None:
- assert (
- await _run_team_trace_id_check(
- route="/anthropic/v1/messages",
- request_body={},
- headers={"x-litellm-trace-id": "trace-provider-123"},
- team_metadata={"require_trace_id": True},
- )
- is True
- )
+ assert await _run_team_trace_id_check(
+ route="/anthropic/v1/messages",
+ request_body={},
+ headers={"x-litellm-trace-id": "trace-provider-123"},
+ team_metadata={"require_trace_id": True},
+ ) is True
@pytest.mark.parametrize(
@@ -344,9 +332,7 @@ async def test_team_require_trace_id_rejects_nonheader_carriers_on_registered_pa
"jsonrpc": "2.0",
"id": "1",
"method": "message/send",
- "params": {
- "message": {"messageId": "msg-1", "role": "user", "parts": [{"type": "text", "text": "hi"}]}
- },
+ "params": {"message": {"messageId": "msg-1", "role": "user", "parts": [{"type": "text", "text": "hi"}]}},
},
),
],
@@ -387,9 +373,7 @@ async def test_team_require_trace_id_rejects_traceparent_on_mcp_and_agent_routes
"jsonrpc": "2.0",
"id": "1",
"method": "message/send",
- "params": {
- "message": {"messageId": "msg-1", "role": "user", "parts": [{"type": "text", "text": "hi"}]}
- },
+ "params": {"message": {"messageId": "msg-1", "role": "user", "parts": [{"type": "text", "text": "hi"}]}},
"metadata": {"trace_id": "trace-body-123"},
},
),
@@ -482,7 +466,7 @@ def test_lens_management_does_not_require_an_inference_user_parameter(route: str
async def test_get_end_user_object(customer_spend, customer_budget):
"""
Scenario 1: normal - get_end_user_object returns the cached user
- Scenario 2: user over budget - NOTE: budget enforcement now happens in
+ Scenario 2: user over budget - NOTE: budget enforcement now happens in
common_checks() via _check_end_user_budget(), not in get_end_user_object()
This test verifies that get_end_user_object correctly retrieves the end user
@@ -610,7 +594,7 @@ async def test_can_key_call_model(model, expect_to_work):
if expect_to_work:
await can_key_call_model(**args)
else:
- with pytest.raises(Exception, match="is not available for this API key") as e:
+ with pytest.raises(Exception, match='is not available for this API key') as e:
await can_key_call_model(**args)
print(e)
@@ -927,12 +911,16 @@ async def test_is_valid_fallback_model():
)
try:
- await is_valid_fallback_model(model="gpt-3.5-turbo", llm_router=router, user_model=None)
+ await is_valid_fallback_model(
+ model="gpt-3.5-turbo", llm_router=router, user_model=None
+ )
except Exception as e:
pytest.fail(f"Expected is_valid_fallback_model to work, got exception: {e}")
with pytest.raises(Exception, match="Invalid") as exc_info:
- await is_valid_fallback_model(model="gpt-4o", llm_router=router, user_model=None)
+ await is_valid_fallback_model(
+ model="gpt-4o", llm_router=router, user_model=None
+ )
e = exc_info.value
assert "Invalid" in str(e)
@@ -946,7 +934,9 @@ async def test_is_valid_fallback_model():
],
)
@pytest.mark.asyncio
-async def test_virtual_key_max_budget_check(token_spend, max_budget, expect_budget_error):
+async def test_virtual_key_max_budget_check(
+ token_spend, max_budget, expect_budget_error
+):
"""
Test if virtual key budget checks work as expected:
1. Triggers budget alert for all cases
@@ -1061,7 +1051,9 @@ async def test_can_team_access_model(model, team_models, expect_to_work):
team_model_aliases=None,
)
if not expect_to_work:
- pytest.fail(f"Expected model access check to fail for model={model}, team_models={team_models}")
+ pytest.fail(
+ f"Expected model access check to fail for model={model}, team_models={team_models}"
+ )
except Exception as e:
if expect_to_work:
pytest.fail(
@@ -1114,9 +1106,9 @@ async def test_virtual_key_soft_budget_check(spend, soft_budget, expect_alert):
await asyncio.sleep(0.1) # Allow time for the alert task to complete
- assert alert_triggered == expect_alert, (
- f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}"
- )
+ assert (
+ alert_triggered == expect_alert
+ ), f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}"
@pytest.mark.parametrize(
@@ -1169,7 +1161,9 @@ async def test_virtual_key_soft_budget_check(spend, soft_budget, expect_alert):
],
)
@pytest.mark.asyncio
-async def test_team_soft_budget_check(spend, soft_budget, expect_alert, metadata, expected_alert_emails):
+async def test_team_soft_budget_check(
+ spend, soft_budget, expect_alert, metadata, expected_alert_emails
+):
"""
Test cases for _team_soft_budget_check:
1. Spend over soft budget, no alert_emails configured - should NOT trigger alert (alerts only sent when alert_emails configured)
@@ -1217,9 +1211,9 @@ async def test_team_soft_budget_check(spend, soft_budget, expect_alert, metadata
await asyncio.sleep(0.1) # Allow time for the alert task to complete
- assert alert_triggered == expect_alert, (
- f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}"
- )
+ assert (
+ alert_triggered == expect_alert
+ ), f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}"
if expect_alert:
assert captured_call_info is not None
@@ -1231,7 +1225,10 @@ async def test_team_soft_budget_check(spend, soft_budget, expect_alert, metadata
if expected_alert_emails is not None:
assert captured_call_info.alert_emails == expected_alert_emails
else:
- assert captured_call_info.alert_emails is None or captured_call_info.alert_emails == []
+ assert (
+ captured_call_info.alert_emails is None
+ or captured_call_info.alert_emails == []
+ )
@pytest.mark.asyncio
@@ -1362,7 +1359,9 @@ async def test_get_fuzzy_user_object():
# Test 5: Only email provided (no SSO ID)
mock_prisma.db.litellm_usertable.find_first = AsyncMock(return_value=test_user)
- result = await _get_fuzzy_user_object(prisma_client=mock_prisma, user_email="test@example.com")
+ result = await _get_fuzzy_user_object(
+ prisma_client=mock_prisma, user_email="test@example.com"
+ )
assert result == test_user
mock_prisma.db.litellm_usertable.find_first.assert_called_with(
where={"user_email": {"equals": "test@example.com", "mode": "insensitive"}},
@@ -1371,7 +1370,9 @@ async def test_get_fuzzy_user_object():
# Test 6: Only SSO ID provided (no email)
mock_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=test_user)
- result = await _get_fuzzy_user_object(prisma_client=mock_prisma, sso_user_id="sso_123")
+ result = await _get_fuzzy_user_object(
+ prisma_client=mock_prisma, sso_user_id="sso_123"
+ )
assert result == test_user
mock_prisma.db.litellm_usertable.find_unique.assert_called_with(
where={"sso_user_id": "sso_123"}, include={"organization_memberships": True}
@@ -1418,7 +1419,7 @@ async def test_can_key_call_model_with_aliases(model, alias_map, expect_to_work)
llm_router=router,
)
else:
- with pytest.raises(Exception, match="is not available for this API key") as e:
+ with pytest.raises(Exception, match='is not available for this API key') as e:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
@@ -1504,7 +1505,9 @@ async def test_delete_cache_access_object():
],
)
@pytest.mark.asyncio
-async def test_get_resources_from_access_groups(resource_field, access_group_data, expected):
+async def test_get_resources_from_access_groups(
+ resource_field, access_group_data, expected
+):
"""Test _get_resources_from_access_groups returns correct resource list from access groups."""
from unittest.mock import AsyncMock, MagicMock, patch
@@ -2094,9 +2097,7 @@ def test_is_model_cost_zero_judges_an_alias_chain_by_the_deployment_its_entry_ro
model_group_alias={"chain-entry": "local-free", "local-free": "paid-gpt"},
)
expected: Final = {"chain-entry": True, "local-free": False, "paid-gpt": False}
- order: Final = (
- ("chain-entry", "local-free", "paid-gpt") if entry_first else ("local-free", "paid-gpt", "chain-entry")
- )
+ order: Final = ("chain-entry", "local-free", "paid-gpt") if entry_first else ("local-free", "paid-gpt", "chain-entry")
verdicts: Final = {name: is_model_cost_zero(model=name, llm_router=router) for name in order}
@@ -2339,7 +2340,10 @@ async def test_bulk_eviction_clears_this_worker_before_waiting_on_a_wedged_redis
monkeypatch.setattr(pubsub_module, "_BULK_EVICTION_WAIT_SECONDS", 0.5)
client = _WedgedPublishRedisClient()
cache: Final = UserApiKeyCache(key_object_in_memory_cache=InMemoryCache(max_size_in_memory=2_000))
- hashed_tokens: Final = tuple(hashlib.sha256(f"token-{index}".encode()).hexdigest() for index in range(600))
+ hashed_tokens: Final = tuple(
+ hashlib.sha256(f"token-{index}".encode()).hexdigest()
+ for index in range(600)
+ )
for hashed_token in hashed_tokens:
cache.set_local(hashed_token, {"token": hashed_token}, None)
seeded: Final = tuple(token for token in hashed_tokens if cache.get_local(token) is not None)
@@ -2468,6 +2472,28 @@ async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged(
assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens)
+@pytest.mark.parametrize("route", ("/v1/chat/completions", "/queue/chat/completions"))
+def test_chat_routes_require_the_user_param_when_enforced(route: str) -> None:
+ from starlette.requests import Request
+
+ from litellm.proxy.auth.auth_checks import _enforce_user_param_check
+
+ request: Final = Request({"type": "http", "method": "POST", "path": route})
+ with pytest.raises(Exception, match="'user' param not passed"):
+ _enforce_user_param_check({"enforce_user_param": True}, request, {}, route)
+ _enforce_user_param_check({"enforce_user_param": True}, request, {"user": "u"}, route)
+
+
+@pytest.mark.parametrize("route", ("/v1/chat/completions", "/queue/chat/completions"))
+def test_spent_proxy_budget_refuses_chat_routes(monkeypatch: pytest.MonkeyPatch, route: str) -> None:
+ from litellm.proxy.auth.auth_checks import _global_proxy_budget_check
+
+ monkeypatch.setattr(litellm, "max_budget", 0.5)
+ _global_proxy_budget_check(global_proxy_spend=0.4, skip_budget_checks=False, route=route)
+ with pytest.raises(litellm.BudgetExceededError):
+ _global_proxy_budget_check(global_proxy_spend=1.0, skip_budget_checks=False, route=route)
+
+
@pytest.mark.parametrize(
"spend, soft_budget, expect_alert",
[
@@ -2511,23 +2537,37 @@ async def test_user_soft_budget_check(spend: float, soft_budget: float | None, e
assert call_info.event_group == Litellm_EntityType.USER
-@pytest.mark.parametrize("route", ("/v1/chat/completions", "/queue/chat/completions"))
-def test_chat_routes_require_the_user_param_when_enforced(route: str) -> None:
- from starlette.requests import Request
+@pytest.mark.parametrize("max_budget", [None, 100.0])
+@pytest.mark.asyncio
+async def test_common_checks_alerts_user_over_soft_budget_without_blocking(max_budget: float | None) -> None:
+ from litellm.models.user import LiteLLM_UserTable
+ from litellm.proxy.auth.auth_checks import common_checks
- from litellm.proxy.auth.auth_checks import _enforce_user_param_check
+ alerts: Final[list[tuple[str, CallInfo]]] = []
- request: Final = Request({"type": "http", "method": "POST", "path": route})
- with pytest.raises(Exception, match="'user' param not passed"):
- _enforce_user_param_check({"enforce_user_param": True}, request, {}, route)
- _enforce_user_param_check({"enforce_user_param": True}, request, {"user": "u"}, route)
+ class RecordingProxyLogging:
+ async def budget_alerts(self, type: str, user_info: CallInfo) -> None:
+ alerts.append((type, user_info))
+ user_id: Final = f"soft-user-{uuid.uuid4().hex}"
+ result: Final = await common_checks(
+ request_body={"model": "gpt-4o-mini"},
+ team_object=None,
+ user_object=LiteLLM_UserTable(user_id=user_id, soft_budget=5.0, max_budget=max_budget, spend=7.0),
+ end_user_object=None,
+ global_proxy_spend=None,
+ general_settings={},
+ route="/chat/completions",
+ llm_router=None,
+ proxy_logging_obj=cast(ProxyLogging, RecordingProxyLogging()), # cast-ok: test double for budget_alerts
+ valid_token=UserAPIKeyAuth(token=f"token-{user_id}", user_id=user_id),
+ request=MagicMock(),
+ )
+ await asyncio.sleep(0)
-@pytest.mark.parametrize("route", ("/v1/chat/completions", "/queue/chat/completions"))
-def test_spent_proxy_budget_refuses_chat_routes(monkeypatch: pytest.MonkeyPatch, route: str) -> None:
- from litellm.proxy.auth.auth_checks import _global_proxy_budget_check
-
- monkeypatch.setattr(litellm, "max_budget", 0.5)
- _global_proxy_budget_check(global_proxy_spend=0.4, skip_budget_checks=False, route=route)
- with pytest.raises(litellm.BudgetExceededError):
- _global_proxy_budget_check(global_proxy_spend=1.0, skip_budget_checks=False, route=route)
+ assert result is True
+ soft_alerts: Final = [info for alert_type, info in alerts if alert_type == "soft_budget"]
+ assert len(soft_alerts) == 1, alerts
+ assert soft_alerts[0].user_id == user_id
+ assert soft_alerts[0].spend == 7.0
+ assert soft_alerts[0].soft_budget == 5.0
diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py
index d478a9ea828..8c0236213c8 100644
--- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py
+++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py
@@ -6060,3 +6060,93 @@ async def test_fips_user_update_hashes_off_the_event_loop(pbkdf2_on_event_loop:
assert pbkdf2_on_event_loop == [False]
assert data["password"].startswith("pbkdf2:")
+
+
+def _soft_budget_prisma(mocker: MockerFixture, saved_users: tuple[LiteLLM_UserTable, ...]) -> MagicMock:
+ prisma_client: Final = mocker.MagicMock()
+ prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=saved_users[0] if saved_users else None)
+ prisma_client.db.litellm_usertable.find_many = mocker.AsyncMock(return_value=list(saved_users))
+ prisma_client.get_data = mocker.AsyncMock(return_value=list(saved_users))
+ prisma_client.update_data = mocker.AsyncMock()
+ prisma_client.insert_data = mocker.AsyncMock()
+ prisma_client.tx = lambda: _transaction_over(prisma_client.db)
+ mocker.patch(
+ "litellm.proxy.proxy_server.prisma_client", prisma_client
+ ) # test-quality-ok: substitute the database dependency
+ return prisma_client
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("saved_users", "update"),
+ [
+ pytest.param(
+ (LiteLLM_UserTable(user_id="u-soft", user_role="proxy_admin_viewer", soft_budget=8),),
+ UpdateUserRequest(user_id="u-soft", user_role="internal_user"),
+ id="role-change-injects-lower-max",
+ ),
+ pytest.param(
+ (),
+ UpdateUserRequest(user_email="new@example.test", soft_budget=100, max_budget=50),
+ id="email-upsert-of-new-user",
+ ),
+ pytest.param(
+ (LiteLLM_UserTable(user_id="u-soft", soft_budget=8),),
+ UpdateUserRequest(user_id="u-soft", max_budget=8),
+ id="max-lowered-to-stored-soft",
+ ),
+ ],
+)
+async def test_user_update_rejects_soft_budget_at_or_above_effective_max_budget(
+ mocker: MockerFixture,
+ monkeypatch: pytest.MonkeyPatch,
+ saved_users: tuple[LiteLLM_UserTable, ...],
+ update: UpdateUserRequest,
+) -> None:
+ import litellm
+ from litellm.proxy.management_endpoints.internal_user_endpoints import _update_single_user_helper
+
+ monkeypatch.setattr(litellm, "max_internal_user_budget", 5.0)
+ prisma_client: Final = _soft_budget_prisma(mocker, saved_users)
+
+ with pytest.raises(HTTPException) as caught:
+ await _update_single_user_helper(
+ user_request=update,
+ user_api_key_dict=UserAPIKeyAuth(user_id="admin-soft", user_role=LitellmUserRoles.PROXY_ADMIN),
+ )
+
+ assert caught.value.status_code == 400
+ assert "soft_budget" in str(caught.value.detail)
+ prisma_client.update_data.assert_not_awaited()
+ prisma_client.insert_data.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "user_updates",
+ [{"soft_budget": 5.0}, {"max_budget": 4.0}],
+ ids=["soft-above-one-users-max", "max-below-one-users-soft"],
+)
+async def test_bulk_user_update_checks_soft_budget_against_each_user(
+ mocker: MockerFixture, user_updates: dict[str, float]
+) -> None:
+ from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update
+ from litellm.types.proxy.management_endpoints.internal_user_endpoints import BulkUpdateUserRequest
+
+ saved_users: Final = (
+ LiteLLM_UserTable(user_id="u-open"),
+ LiteLLM_UserTable(user_id="u-tight", max_budget=3.0, soft_budget=None),
+ LiteLLM_UserTable(user_id="u-soft", soft_budget=6.0),
+ )
+ prisma_client: Final = _soft_budget_prisma(mocker, saved_users)
+ prisma_client.db.litellm_usertable.update_many = mocker.AsyncMock(return_value=len(saved_users))
+
+ with pytest.raises(HTTPException) as caught:
+ await bulk_user_update(
+ data=BulkUpdateUserRequest(all_users=True, user_updates=user_updates),
+ user_api_key_dict=UserAPIKeyAuth(user_id="admin-soft", user_role=LitellmUserRoles.PROXY_ADMIN),
+ litellm_changed_by=None,
+ )
+
+ assert caught.value.status_code == 400
+ prisma_client.db.litellm_usertable.update_many.assert_not_awaited()