From d67d12fc54935b6c5eb238ad77db9ddad30f5f25 Mon Sep 17 00:00:00 2001
From: ryan-crabbe <128659760+ryan-crabbe@users.noreply.github.com>
Date: Fri, 23 Jan 2026 17:26:45 -0800
Subject: [PATCH] perf: Add LRU caching to get_model_info for faster cost
 lookups (#19606)

- Add @lru_cache decorator to get_model_info() and _cached_get_model_info_helper()
- Update _invalidate_model_cost_lowercase_map() to clear these caches when model_cost changes
- Update test to call cache invalidation after modifying litellm.model_cost

Reduces get_model_cost_information from 46% to <1% of request handling time.
---
 litellm/utils.py                           | 7 +++++++
 tests/test_litellm/test_cost_calculator.py | 6 ++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index 1865d364586..584ab8805a0 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -5135,10 +5135,15 @@ def _invalidate_model_cost_lowercase_map() -> None:
     """Invalidate the case-insensitive lookup map for model_cost.
 
     Call this whenever litellm.model_cost is modified to ensure the map is rebuilt.
+    Also clears related LRU caches that depend on model_cost data.
     """
     global _model_cost_lowercase_map
     _model_cost_lowercase_map = None
 
+    # Clear LRU caches that depend on model_cost data
+    get_model_info.cache_clear()
+    _cached_get_model_info_helper.cache_clear()
+
 
 def _rebuild_model_cost_lowercase_map() -> Dict[str, str]:
     """Rebuild the case-insensitive lookup map from the current model_cost.
@@ -5352,6 +5357,7 @@ def _get_max_position_embeddings(model_name: str) -> Optional[int]:
         return None
 
 
+@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
 def _cached_get_model_info_helper(
     model: str, custom_llm_provider: Optional[str]
 ) -> ModelInfoBase:
@@ -5699,6 +5705,7 @@ def _get_model_info_helper(  # noqa: PLR0915
         )
 
 
+@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
 def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> ModelInfo:
     """
     Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token  for a given model.
diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py
index 5bf7a70055e..9d968d482c6 100644
--- a/tests/test_litellm/test_cost_calculator.py
+++ b/tests/test_litellm/test_cost_calculator.py
@@ -70,8 +70,6 @@ def test_cost_calculator_with_response_cost_in_additional_headers():
 
 
 def test_cost_calculator_with_usage(monkeypatch):
-    from litellm import get_model_info
-
     os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
     litellm.model_cost = litellm.get_model_cost_map(url="")
 
@@ -123,6 +121,10 @@ def test_cost_calculator_with_usage(monkeypatch):
         },
     )
 
+    # Invalidate caches after modifying litellm.model_cost
+    from litellm.utils import _invalidate_model_cost_lowercase_map
+    _invalidate_model_cost_lowercase_map()
+
     result = response_cost_calculator(
         response_object=mr,
         model="",