From 3fc95c5e1a0b5fad27c06dedceacbc554b3c6035 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 10 Sep 2026 06:08:28 +0000 Subject: [PATCH] perf(proxy): reuse cached model group and deployment info in budget reservation Profiling the sidecar-enabled gateway at 700 rps showed ~2.4% of all samples in get_model_group_info called per request from budget reservation, plus get_deployment_model_info for tiered pricing tables. Both are read-only lookups over the model list, so serve them from the Router's lru caches and clear the deployment cache alongside the group cache when the model list changes. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend_tracking/budget_reservation.py | 4 +- litellm/router.py | 9 ++++ .../spend_tracking/test_budget_reservation.py | 47 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 61d82fb6158..ef21551ca93 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1276,7 +1276,7 @@ def _get_model_cost_info( llm_router: Router | None, ) -> Mapping[str, object] | None: if llm_router is not None: - model_group_info: Final = llm_router.get_model_group_info(model_group=model) + model_group_info: Final = llm_router.cached_model_group_info(model) if model_group_info is not None: return model_group_info.model_dump() return dict(litellm.get_model_info(model=model)) @@ -1318,7 +1318,7 @@ def _deployment_tiered_pricing_table( backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") if not isinstance(model_id, str) or not isinstance(backend_model, str): return None - deployment_model_info: Final = llm_router.get_deployment_model_info(model_id=model_id, model_name=backend_model) + deployment_model_info: Final = llm_router.cached_deployment_model_info(model_id, backend_model) if deployment_model_info is None: return None tiered_pricing: Final = deployment_model_info.get("tiered_pricing") diff --git a/litellm/router.py b/litellm/router.py index f9d4bf1428e..e394e890bbf 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11017,6 +11017,14 @@ class Router: """ return self.get_model_group_info(model_group) + def cached_model_group_info(self, model_group: str) -> ModelGroupInfo | None: + return self._cached_get_model_group_info(model_group) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) + def cached_deployment_model_info(self, model_id: str, model_name: str) -> ModelInfo | None: + """Read-only ``get_deployment_model_info``: the returned mapping is shared across callers.""" + return self.get_deployment_model_info(model_id=model_id, model_name=model_name) + async def get_remaining_model_group_usage(self, model_group: str) -> dict[str, int]: model_group_info: Final = self._cached_get_model_group_info(model_group) @@ -11794,6 +11802,7 @@ class Router: result and bypass budget enforcement. """ self._cached_get_model_group_info.cache_clear() + self.cached_deployment_model_info.cache_clear() self._zero_cost_cache.clear() self._routing_group_rows = None diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 23eb297b47f..c0ccb39eebd 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -1,4 +1,5 @@ import json +import math from typing import Final import pytest @@ -14,8 +15,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( reserve_budget_for_request, ) from litellm.proxy.utils import ProxyLogging +from litellm.router import Router from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge import token_counter as rust_token_counter +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( "/responses/input_tokens", @@ -282,3 +285,47 @@ async def test_non_anthropic_tokenizer_models_stay_in_python(rust_counter: None) assert _RecordingCounter.bodies == [] assert counts["gpt-4o"] != RUST_INPUT_TOKENS + + +def _tiered_deployment(input_cost_per_token: float) -> Deployment: + return Deployment( + model_name="tiered-group", + litellm_params=LiteLLM_Params(model="dashscope/qwen3-max", api_key="sk-fake"), + model_info=ModelInfo( + id="tiered-deployment", + max_output_tokens=1000, + tiered_pricing=[ + { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": input_cost_per_token, + "range": [0, 128000], + } + ], + ), + ) + + +TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10} + + +def test_repeated_estimates_reuse_cached_model_cost_info() -> None: + router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()]) + first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + hits_before: Final = Router.cached_deployment_model_info.cache_info().hits + + second: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + + assert second == first + assert Router.cached_deployment_model_info.cache_info().hits == hits_before + 1 + + +def test_deployment_pricing_update_invalidates_cached_estimate() -> None: + router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()]) + before: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + assert before is not None + + router.upsert_deployment(_tiered_deployment(1e-03)) + + after: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) + assert after is not None + assert math.isclose(after, before * 1000)