mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(proxy): log budget reservation notice once at config load (#40167)
* fix(proxy): log disable_budget_reservation notice once at config load The disabled-budget-reservation reminder fired as a WARNING inside request authentication, so every authenticated request on a proxy that deliberately set the flag produced one warning line. The notice now runs once per worker when general_settings loads, at INFO, and the request path only skips the reservation. Reservation skipping and read-time budget checks are unchanged * fix(proxy): keep budget notice sentinel with constants * fix(proxy): expose shared budget notice state
This commit is contained in:
parent
1761fe236f
commit
9bc9104102
9 changed files with 121 additions and 11 deletions
|
|
@ -55,6 +55,7 @@ S3_PREFIX_DIGEST_CHARS: Final = 16
|
|||
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
|
||||
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
|
||||
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))
|
||||
budget_reservation_disabled_info_emitted = False
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1))
|
||||
DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
|
||||
SQS_SEND_MESSAGE_ACTION: Final = "SendMessage"
|
||||
|
|
|
|||
|
|
@ -2833,7 +2833,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
"Enable only if your deployment is experiencing phantom "
|
||||
"BudgetExceededError responses caused by leaked reservations "
|
||||
"(see GitHub issue #27639). "
|
||||
"A proxy-level WARNING is logged on every request while this flag "
|
||||
"An INFO notice is logged once per worker at config load while this flag "
|
||||
"is active as a reminder that hard enforcement is relaxed."
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from fastapi import HTTPException, Request, status
|
|||
from pydantic import PositiveInt, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import Router, provider_list
|
||||
from litellm import Router, constants, provider_list
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import (
|
||||
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
|
||||
|
|
@ -1390,6 +1390,24 @@ def warn_once_if_custom_auth_skips_common_checks(
|
|||
_custom_auth_common_checks_warning_emitted = True
|
||||
|
||||
|
||||
def log_once_if_budget_reservation_disabled(
|
||||
*,
|
||||
disabled: bool,
|
||||
logger: Logger = verbose_proxy_logger,
|
||||
) -> None:
|
||||
if constants.budget_reservation_disabled_info_emitted or not disabled:
|
||||
return
|
||||
logger.info(
|
||||
"disable_budget_reservation is enabled: skipping optimistic budget "
|
||||
"reservation. Budget enforcement is read-time only. Concurrent "
|
||||
"requests can each pass the spend check before their cost is recorded, "
|
||||
"so a configured budget may be briefly exceeded under high concurrency. "
|
||||
"Set disable_budget_reservation to False or remove it to restore "
|
||||
"hard per-request budget enforcement."
|
||||
)
|
||||
constants.budget_reservation_disabled_info_emitted = True # rebind-ok: process-wide one-shot sentinel
|
||||
|
||||
|
||||
def is_pass_through_provider_route(route: str) -> bool:
|
||||
PROVIDER_SPECIFIC_PASS_THROUGH_ROUTES: Final = [
|
||||
"vertex-ai",
|
||||
|
|
|
|||
|
|
@ -2706,14 +2706,6 @@ async def _reserve_budget_after_common_checks(
|
|||
if skip_budget_checks:
|
||||
return
|
||||
if general_settings.get("disable_budget_reservation") is True:
|
||||
verbose_proxy_logger.warning(
|
||||
"disable_budget_reservation is enabled: skipping optimistic budget "
|
||||
"reservation. Budget enforcement is read-time only — concurrent "
|
||||
"requests can each pass the spend check before their cost is recorded, "
|
||||
"so a configured budget may be briefly exceeded under high concurrency. "
|
||||
"Set disable_budget_reservation to False or remove it to restore "
|
||||
"hard per-request budget enforcement."
|
||||
)
|
||||
return
|
||||
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
|
|
|
|||
|
|
@ -306,6 +306,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
from litellm.proxy.auth.auth_utils import (
|
||||
check_response_size_is_safe,
|
||||
is_request_body_safe,
|
||||
log_once_if_budget_reservation_disabled,
|
||||
warn_once_if_custom_auth_skips_common_checks,
|
||||
)
|
||||
from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
|
||||
|
|
@ -5653,6 +5654,10 @@ class ProxyConfig:
|
|||
run_common_checks=bool(general_settings.get("custom_auth_run_common_checks", False)),
|
||||
)
|
||||
|
||||
log_once_if_budget_reservation_disabled(
|
||||
disabled=general_settings.get("disable_budget_reservation") is True,
|
||||
)
|
||||
|
||||
custom_key_generate: Final = general_settings.get("custom_key_generate", None)
|
||||
if custom_key_generate is not None:
|
||||
user_custom_key_generate = get_instance_fn(value=custom_key_generate, config_file_path=config_file_path)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext
|
|||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from typing import Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -15,6 +16,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
abbreviate_api_key,
|
||||
check_complete_credentials,
|
||||
custom_auth_common_checks_warning,
|
||||
log_once_if_budget_reservation_disabled,
|
||||
warn_once_if_custom_auth_skips_common_checks,
|
||||
get_end_user_id_from_request_body,
|
||||
get_key_mcp_rpm_limit,
|
||||
|
|
@ -101,6 +103,41 @@ class TestWarnOnceIfCustomAuthSkipsCommonChecks:
|
|||
assert logger.warning.call_count == 0
|
||||
|
||||
|
||||
class TestLogOnceIfBudgetReservationDisabled:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_sentinel(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"litellm.constants.budget_reservation_disabled_info_emitted",
|
||||
False,
|
||||
)
|
||||
|
||||
def test_logs_info_only_once_when_enabled(self, caplog):
|
||||
with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"):
|
||||
log_once_if_budget_reservation_disabled(disabled=False)
|
||||
assert not any(
|
||||
"disable_budget_reservation is enabled" in record.message
|
||||
for record in caplog.records
|
||||
)
|
||||
for _ in range(3):
|
||||
log_once_if_budget_reservation_disabled(disabled=True)
|
||||
|
||||
records = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if "disable_budget_reservation is enabled" in record.message
|
||||
]
|
||||
assert len(records) == 1
|
||||
assert records[0].levelno == logging.INFO
|
||||
|
||||
def test_logs_to_injected_logger_only_once(self):
|
||||
logger = MagicMock()
|
||||
log_once_if_budget_reservation_disabled(disabled=False, logger=logger)
|
||||
for _ in range(3):
|
||||
log_once_if_budget_reservation_disabled(disabled=True, logger=logger)
|
||||
assert logger.info.call_count == 1
|
||||
assert "disable_budget_reservation is enabled" in logger.info.call_args[0][0]
|
||||
|
||||
|
||||
class TestGetKeyModelRpmLimit:
|
||||
"""Tests for get_key_model_rpm_limit function."""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -146,6 +147,35 @@ async def test_disable_budget_reservation_skips_reservation():
|
|||
assert user_api_key_auth_obj.budget_reservation is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disable_budget_reservation_does_not_log_per_request(caplog):
|
||||
user_api_key_auth_obj = UserAPIKeyAuth(token="test_token")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"):
|
||||
for _ in range(3):
|
||||
await _reserve_budget_after_common_checks(
|
||||
user_api_key_auth_obj=user_api_key_auth_obj,
|
||||
request_data={"model": "gpt-4o"},
|
||||
route="/v1/chat/completions",
|
||||
llm_router=None,
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=MagicMock(),
|
||||
proxy_logging_obj=MagicMock(),
|
||||
skip_budget_checks=False,
|
||||
general_settings={"disable_budget_reservation": True},
|
||||
)
|
||||
|
||||
records = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if "disable_budget_reservation is enabled" in record.message
|
||||
]
|
||||
assert records == []
|
||||
assert user_api_key_auth_obj.budget_reservation is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_reservation_runs_when_not_disabled():
|
||||
"""Control for #27639: with the flag absent, the reservation still runs and is stored."""
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ Pins covered:
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -1633,6 +1634,32 @@ async def test_ProxyConfig_load_config_minimal_yaml(tmp_path, monkeypatch):
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("setting", ["true", "false", "null", "'true'", None])
|
||||
async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monkeypatch, caplog, setting):
|
||||
config_file = tmp_path / "budget.yaml"
|
||||
flag = f" disable_budget_reservation: {setting}\n" if setting is not None else ""
|
||||
config_file.write_text(
|
||||
"model_list: []\nlitellm_settings: {}\ngeneral_settings:\n"
|
||||
" master_key: null\n" + flag
|
||||
)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
|
||||
monkeypatch.setattr("litellm.constants.budget_reservation_disabled_info_emitted", False)
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
config = ProxyConfig()
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"):
|
||||
for _ in range(3):
|
||||
await config.load_config(router=None, config_file_path=str(config_file))
|
||||
|
||||
records = [
|
||||
record for record in caplog.records
|
||||
if "disable_budget_reservation is enabled" in record.message
|
||||
]
|
||||
assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch):
|
||||
"""Regression: router_settings.plugins dotted-path strings must be resolved to
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25798,7 +25798,7 @@ export interface components {
|
|||
disable_auto_add_proxy_admin_to_teams?: boolean | null;
|
||||
/**
|
||||
* Disable Budget Reservation
|
||||
* @description If True, disables the optimistic per-request budget reservation introduced in v1.84.0. WARNING: This weakens hard budget enforcement. Without the reservation, a burst of concurrent requests from a single key can each pass the read-time spend check before any of them is charged, allowing a configured budget to be exceeded under high concurrency. Budgets are still evaluated on every request at read time, so an already-exhausted budget is still rejected. Enable only if your deployment is experiencing phantom BudgetExceededError responses caused by leaked reservations (see GitHub issue #27639). A proxy-level WARNING is logged on every request while this flag is active as a reminder that hard enforcement is relaxed.
|
||||
* @description If True, disables the optimistic per-request budget reservation introduced in v1.84.0. WARNING: This weakens hard budget enforcement. Without the reservation, a burst of concurrent requests from a single key can each pass the read-time spend check before any of them is charged, allowing a configured budget to be exceeded under high concurrency. Budgets are still evaluated on every request at read time, so an already-exhausted budget is still rejected. Enable only if your deployment is experiencing phantom BudgetExceededError responses caused by leaked reservations (see GitHub issue #27639). An INFO notice is logged once per worker at config load while this flag is active as a reminder that hard enforcement is relaxed.
|
||||
*/
|
||||
disable_budget_reservation?: boolean | null;
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue