From a8243079d336fa305bfc9b16d99667ed6714f752 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Tue, 7 Oct 2025 18:51:27 -0700 Subject: [PATCH] doc fix --- docs/my-website/docs/proxy/dynamic_rate_limit.md | 3 +++ litellm/proxy/proxy_config.yaml | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md index b3aed6a359e..06d49dfaf0f 100644 --- a/docs/my-website/docs/proxy/dynamic_rate_limit.md +++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md @@ -141,6 +141,7 @@ litellm_settings: "dev": 0.1 # 10% reserved for development (1 RPM) priority_reservation_settings: default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata + saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit general_settings: master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env @@ -156,6 +157,8 @@ general_settings: `priority_reservation_settings`: Object (Optional) - **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5) +- **saturation_threshold (float)**: Saturation level (0.0 to 1.0) at which strict priority enforcement begins for a model. Saturation is calculated as `max(current_rpm/max_rpm, current_tpm/max_tpm)`. Below this threshold, generous mode allows priority borrowing from unused capacity. Above this threshold, strict mode enforces normalized priority limits. + - Example: When model usage is low, keys can use more than their allocated share. When model usage is high, keys are strictly limited to their allocated share. **Start Proxy** diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0a91a179ff1..338fa98118c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,4 +12,5 @@ litellm_settings: "prod": 0.9 # 90% reserved for production (9 RPM) "dev": 0.1 # 10% reserved for development (1 RPM) priority_reservation_settings: - default_priority: 0.2 # Weight (0%) assigned to keys without explicit priority metadata \ No newline at end of file + default_priority: 0.2 # Weight (0%) assigned to keys without explicit priority metadata + saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit \ No newline at end of file