From 7f1296f4faa8485fe066366cdfc7f72ff85c7c5f Mon Sep 17 00:00:00 2001 From: Abhinav Tarigoppula Date: Thu, 17 Sep 2026 14:10:18 +0530 Subject: [PATCH] fix: Router._cached_get_model_group_info leaks every Router instance _cached_get_model_group_info is decorated with @lru_cache directly on the Router class. That keys the cache on self as well as model_group, so every Router that ever calls it stays reachable through the cache for the life of the process, even after discard() (the router's own documented cleanup method) and every other reference are dropped. set_response_headers calls it on every request, so this hits any process that constructs more than one Router over its lifetime, for example once per tenant or on every proxy config reload. The sibling method cached_deployment_model_info already wraps the bound method in __init__ instead of decorating it at class scope, which keeps the cache scoped to the instance. This does the same thing for _cached_get_model_group_info. As a side effect it also fixes cache_clear() on this cache: it currently clears every Router's entries process-wide instead of just the one being invalidated, since it is still the same shared class-level cache object. Added a regression test that constructs a Router, calls the cached method, calls discard(), drops the reference, and asserts the Router is collected. It fails against the current code and passes with this fix. --- litellm/router.py | 4 +++- tests/test_litellm/test_router.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 633f060f208..2b07eb1aeae 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -981,6 +981,9 @@ class Router: self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( self.get_deployment_model_info ) + self._cached_get_model_group_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( + self._cached_get_model_group_info + ) self._discovered_model_info_cache: InMemoryCache = InMemoryCache( max_size_in_memory=max(len(model_list or ()), 1), default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, @@ -11276,7 +11279,6 @@ class Router: return total_itpm, total_otpm - @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _cached_get_model_group_info(self, model_group: str) -> ModelGroupInfo | None: """ Cached version of get_model_group_info, uses @lru_cache wrapper diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 1e6636ec3d6..625acb3b45f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1819,6 +1819,36 @@ def test_cached_get_model_group_info(): assert result5 is result6 +def test_cached_get_model_group_info_does_not_leak_router_instances(): + """ + _cached_get_model_group_info keys its cache on self as well as + model_group. If it were decorated with @lru_cache directly on the + class, the cache would key on self too, keeping every Router that + ever called it (set_response_headers does, on every request) + reachable for the life of the process, even after discard() and every + other reference are dropped. + """ + import gc + import weakref + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake"}, + }, + ] + ) + router._cached_get_model_group_info("gpt-4") + router.discard() + router_ref = weakref.ref(router) + + del router + gc.collect() + + assert router_ref() is None + + def test_model_group_info_cost_from_db_model_info(): """ When get_deployment_model_info fails (model_info is None fallback),