perf(proxy): reuse cached model group and deployment info in budget reservation

Profiling the sidecar-enabled gateway at 700 rps showed ~2.4% of all samples
in get_model_group_info called per request from budget reservation, plus
get_deployment_model_info for tiered pricing tables. Both are read-only lookups
over the model list, so serve them from the Router's lru caches and clear the
deployment cache alongside the group cache when the model list changes.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/spend_tracking/budget_reservation.py

@@ -1276,7 +1276,7 @@ def _get_model_cost_info(
     llm_router: Router | None,
 ) -> Mapping[str, object] | None:
     if llm_router is not None:
-        model_group_info: Final = llm_router.get_model_group_info(model_group=model)
+        model_group_info: Final = llm_router.cached_model_group_info(model)
         if model_group_info is not None:
             return model_group_info.model_dump()
     return dict(litellm.get_model_info(model=model))
@@ -1318,7 +1318,7 @@ def _deployment_tiered_pricing_table(
     backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model")
     if not isinstance(model_id, str) or not isinstance(backend_model, str):
         return None
-    deployment_model_info: Final = llm_router.get_deployment_model_info(model_id=model_id, model_name=backend_model)
+    deployment_model_info: Final = llm_router.cached_deployment_model_info(model_id, backend_model)
     if deployment_model_info is None:
         return None
     tiered_pricing: Final = deployment_model_info.get("tiered_pricing")

litellm/router.py

@@ -11017,6 +11017,14 @@ class Router:
         """
         return self.get_model_group_info(model_group)
 
+    def cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None:
+        return self._cached_get_model_group_info(model_group)
+
+    @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
+    def cached_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None:
+        """Read-only ``get_deployment_model_info``: the returned mapping is shared across callers."""
+        return self.get_deployment_model_info(model_id=model_id, model_name=model_name)
+
     async def get_remaining_model_group_usage(self, model_group: str) -> dict[str, int]:
         model_group_info: Final = self._cached_get_model_group_info(model_group)
 
@@ -11794,6 +11802,7 @@ class Router:
         result and bypass budget enforcement.
         """
         self._cached_get_model_group_info.cache_clear()
+        self.cached_deployment_model_info.cache_clear()
         self._zero_cost_cache.clear()
         self._routing_group_rows = None
 

tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py

@@ -1,4 +1,5 @@
 import json
+import math
 from typing import Final
 
 import pytest
@@ -14,8 +15,10 @@ from litellm.proxy.spend_tracking.budget_reservation import (
     reserve_budget_for_request,
 )
 from litellm.proxy.utils import ProxyLogging
+from litellm.router import Router
 from litellm.rust_bridge import bindings, configuration
 from litellm.rust_bridge import token_counter as rust_token_counter
+from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
 
 TOKEN_COUNTING_ROUTES: Final = (
     "/responses/input_tokens",
@@ -282,3 +285,47 @@ async def test_non_anthropic_tokenizer_models_stay_in_python(rust_counter: None)
 
     assert _RecordingCounter.bodies == []
     assert counts["gpt-4o"] != RUST_INPUT_TOKENS
+
+
+def _tiered_deployment(input_cost_per_token: float) -> Deployment:
+    return Deployment(
+        model_name="tiered-group",
+        litellm_params=LiteLLM_Params(model="dashscope/qwen3-max", api_key="sk-fake"),
+        model_info=ModelInfo(
+            id="tiered-deployment",
+            max_output_tokens=1000,
+            tiered_pricing=[
+                {
+                    "input_cost_per_token": input_cost_per_token,
+                    "output_cost_per_token": input_cost_per_token,
+                    "range": [0, 128000],
+                }
+            ],
+        ),
+    )
+
+
+TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}
+
+
+def test_repeated_estimates_reuse_cached_model_cost_info() -> None:
+    router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()])
+    first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)
+    hits_before: Final = Router.cached_deployment_model_info.cache_info().hits
+
+    second: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)
+
+    assert second == first
+    assert Router.cached_deployment_model_info.cache_info().hits == hits_before + 1
+
+
+def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
+    router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()])
+    before: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)
+    assert before is not None
+
+    router.upsert_deployment(_tiered_deployment(1e-03))
+
+    after: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)
+    assert after is not None
+    assert math.isclose(after, before * 1000)