diff --git a/litellm/router.py b/litellm/router.py index 633f060f208..2b07eb1aeae 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -981,6 +981,9 @@ class Router: self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( self.get_deployment_model_info ) + self._cached_get_model_group_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._cached_get_model_group_info + ) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -11276,7 +11279,6 @@ class Router: return total_itpm, total_otpm - @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _cached_get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: """ Cached version of get_model_group_info, uses @lru_cache wrapper diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1e6636ec3d6..625acb3b45f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1819,6 +1819,36 @@ def test_cached_get_model_group_info(): assert result5 is result6 +def test_cached_get_model_group_info_does_not_leak_router_instances(): + """ + _cached_get_model_group_info keys its cache on self as well as + model_group. If it were decorated with @lru_cache directly on the + class, the cache would key on self too, keeping every Router that + ever called it (set_response_headers does, on every request) + reachable for the life of the process, even after discard() and every + other reference are dropped. + """ + import gc + import weakref + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake"}, + }, + ] + ) + router._cached_get_model_group_info("gpt-4") + router.discard() + router_ref = weakref.ref(router) + + del router + gc.collect() + + assert router_ref() is None + + def test_model_group_info_cost_from_db_model_info(): """ When get_deployment_model_info fails (model_info is None fallback),