From c71376d21ab48c4488e93f1a0e1548a8df651abb Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 8 Aug 2026 01:52:36 +0000 Subject: [PATCH] feat(proxy): let the license carry the SGR window, default to a calendar year Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 +- litellm/proxy/_types.py | 6 +++- litellm/proxy/db/gateway_request_limits.py | 25 +++++++++++----- .../proxy/db/test_gateway_request_limits.py | 29 ++++++++++++++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 09e592808ab..488706a412a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1490,7 +1490,7 @@ SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRE SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0)) SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)) DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute -SGR_LIMIT_CHECK_INTERVAL: Final = int(os.getenv("SGR_LIMIT_CHECK_INTERVAL", "300")) +SGR_LIMIT_CHECK_INTERVAL: Final = 300 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)) PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600)) MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50))) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fc4e6ee7646..9489daf18b1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2407,7 +2407,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ) sgr_limit_window: Literal["month", "year"] | None = Field( None, - description="Period `sgr_limit` is counted over, calendar aligned in UTC. Defaults to `month`", + description=( + "Period `sgr_limit` is counted over, calendar aligned in UTC. Takes precedence over an enterprise " + "license's `sgr_window`, and defaults to `year`" + ), ) ui_access_mode: Literal["admin_only", "all"] | None = Field("all", description="Control access to the Proxy UI") allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") @@ -4711,6 +4714,7 @@ class EnterpriseLicenseData(TypedDict, total=False): max_users: int max_teams: int max_sgr: int + sgr_window: Literal["month", "year"] class ResponseLiteLLM_ManagedVectorStore(TypedDict, total=False): diff --git a/litellm/proxy/db/gateway_request_limits.py b/litellm/proxy/db/gateway_request_limits.py index 6593c8cdf93..c51e892539a 100644 --- a/litellm/proxy/db/gateway_request_limits.py +++ b/litellm/proxy/db/gateway_request_limits.py @@ -2,10 +2,11 @@ Soft and hard limits on SGR (successful gateway requests). An allowance can come from two places. An enterprise license may carry -``max_sgr``, which makes the contracted volume visible to the deployment -without LiteLLM having to receive any telemetry back. ``general_settings`` can -also set it directly, which is what lets a customer self-serve a threshold -lower than their contract, and which wins when both are present. +``max_sgr`` and an optional ``sgr_window``, which makes the contracted volume +visible to the deployment without LiteLLM having to receive any telemetry back. +``general_settings`` can also set it directly, which is what lets a customer +self-serve a threshold lower than their contract, and which wins when both are +present. Absent both, the window is a calendar year. Crossing a threshold alerts, it does not reject: this reads the same ``LiteLLM_DailyGatewayRequests`` rollup the admin UI reads, on a scheduler, so @@ -32,6 +33,7 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging DEFAULT_SGR_SOFT_LIMIT_PERCENT: Final = 0.8 +DEFAULT_SGR_LIMIT_WINDOW: Final = SGRLimitWindow.YEAR class SGRLimitSettings(BaseModel): @@ -48,9 +50,9 @@ class SGRLimitSettings(BaseModel): le=1, description="Fraction of the hard limit at which the soft alert fires", ) - sgr_limit_window: SGRLimitWindow = Field( - default=SGRLimitWindow.MONTH, - description="Period the limit is counted over, calendar aligned in UTC", + sgr_limit_window: SGRLimitWindow | None = Field( + default=None, + description="Period the limit is counted over, calendar aligned in UTC. Overrides a license's sgr_window", ) @@ -64,6 +66,13 @@ def _license_sgr_limit(license_data: "EnterpriseLicenseData | None") -> int | No return max_sgr if isinstance(max_sgr, int) and max_sgr > 0 else None +def _license_sgr_window(license_data: "EnterpriseLicenseData | None") -> SGRLimitWindow | None: + if license_data is None: + return None + window: Final = license_data.get("sgr_window") + return SGRLimitWindow(window) if window in tuple(member.value for member in SGRLimitWindow) else None + + def resolve_sgr_limit( *, general_settings: Mapping[str, object], @@ -93,7 +102,7 @@ def resolve_sgr_limit( return SGRLimitConfig( limit=limit, soft_limit=max(1, int(limit * settings.sgr_soft_limit_percent)), - window=settings.sgr_limit_window, + window=settings.sgr_limit_window or _license_sgr_window(license_data) or DEFAULT_SGR_LIMIT_WINDOW, ) diff --git a/tests/test_litellm/proxy/db/test_gateway_request_limits.py b/tests/test_litellm/proxy/db/test_gateway_request_limits.py index aa4618f78ad..b7fd809e74e 100644 --- a/tests/test_litellm/proxy/db/test_gateway_request_limits.py +++ b/tests/test_litellm/proxy/db/test_gateway_request_limits.py @@ -30,7 +30,7 @@ def test_no_allowance_without_a_license_or_config(): def test_license_max_sgr_is_the_allowance(): config = resolve_sgr_limit(general_settings={}, license_data={"max_sgr": 1_000_000}) - assert config == SGRLimitConfig(limit=1_000_000, soft_limit=800_000, window=SGRLimitWindow.MONTH) + assert config == SGRLimitConfig(limit=1_000_000, soft_limit=800_000, window=SGRLimitWindow.YEAR) def test_config_overrides_the_license_so_a_customer_can_alert_earlier(): @@ -48,8 +48,29 @@ def test_soft_limit_percent_moves_the_soft_threshold(): assert config.soft_limit == 500 -def test_window_can_be_the_calendar_year(): - config = resolve_sgr_limit(general_settings={"sgr_limit": 10, "sgr_limit_window": "year"}, license_data=None) +def test_window_can_be_narrowed_to_the_calendar_month(): + config = resolve_sgr_limit(general_settings={"sgr_limit": 10, "sgr_limit_window": "month"}, license_data=None) + assert config is not None + assert config.window is SGRLimitWindow.MONTH + + +def test_the_license_can_carry_the_window(): + config = resolve_sgr_limit(general_settings={}, license_data={"max_sgr": 100, "sgr_window": "month"}) + assert config is not None + assert config.window is SGRLimitWindow.MONTH + + +def test_config_window_wins_over_the_license_window(): + config = resolve_sgr_limit( + general_settings={"sgr_limit_window": "year"}, + license_data={"max_sgr": 100, "sgr_window": "month"}, + ) + assert config is not None + assert config.window is SGRLimitWindow.YEAR + + +def test_a_nonsensical_license_window_falls_back_to_the_calendar_year(): + config = resolve_sgr_limit(general_settings={}, license_data={"max_sgr": 100, "sgr_window": "fortnight"}) assert config is not None assert config.window is SGRLimitWindow.YEAR @@ -83,7 +104,7 @@ def test_settings_left_unset_fall_back_to_the_license(): general_settings={"sgr_limit": None, "sgr_soft_limit_percent": None, "sgr_limit_window": None}, license_data={"max_sgr": 100}, ) - assert config == SGRLimitConfig(limit=100, soft_limit=80, window=SGRLimitWindow.MONTH) + assert config == SGRLimitConfig(limit=100, soft_limit=80, window=SGRLimitWindow.YEAR) def test_a_tiny_allowance_still_has_a_soft_threshold_of_at_least_one(): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a49ecddc93a..b6b8493707e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23635,7 +23635,7 @@ export interface components { sgr_limit?: number | null; /** * Sgr Limit Window - * @description Period `sgr_limit` is counted over, calendar aligned in UTC. Defaults to `month` + * @description Period `sgr_limit` is counted over, calendar aligned in UTC. Takes precedence over an enterprise license's `sgr_window`, and defaults to `year` */ sgr_limit_window?: ("month" | "year") | null; /**