From 0458accaa0b968ca37b98322012512572fadd0c7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:47:37 -0700 Subject: [PATCH] perf(auth): drop guaranteed-miss internal-cache Redis read from team object lookup (#38073) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 14 ---- .../access_group_endpoints.py | 2 - .../proxy/auth/test_auth_checks.py | 82 +++++++++++++++++-- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 4af942a357e..5d52af3b364 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -71,7 +71,6 @@ from litellm.proxy.auth.budget_throttle import ( ) from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation -from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, _safe_get_request_query_params, @@ -2741,20 +2740,9 @@ async def _get_team_object_from_user_api_key_cache( async def _get_team_object_from_cache( key: str, - proxy_logging_obj: ProxyLogging | None, user_api_key_cache: UserApiKeyCache, parent_otel_span: Span | None, ) -> LiteLLM_TeamTableCachedObj | None: - ## INTERNAL USAGE CACHE (plain DualCache) — checked before UserApiKeyCache stores ## - if proxy_logging_obj is not None and proxy_logging_obj.internal_usage_cache.dual_cache: - cached_raw: Final = await proxy_logging_obj.internal_usage_cache.dual_cache.async_get_cache( - key=key, parent_otel_span=parent_otel_span - ) - if cached_raw is not None: - from_internal: Final = CacheCodec.deserialize(cached_raw, LiteLLM_TeamTableCachedObj) - if from_internal is not None: - return from_internal - decoded: Final = await user_api_key_cache.async_get_cache( key=key, parent_otel_span=parent_otel_span, @@ -2790,7 +2778,6 @@ async def get_team_object( if not check_db_only: cached_team_obj: Final = await _get_team_object_from_cache( key=key, - proxy_logging_obj=proxy_logging_obj, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, ) @@ -2953,7 +2940,6 @@ async def get_team_object_by_alias( cached_team_obj: Final = await _get_team_object_from_cache( key=cache_key, - proxy_logging_obj=proxy_logging_obj, user_api_key_cache=user_api_key_cache, parent_otel_span=parent_otel_span, ) diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 2271501d480..c803d87e436 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -210,7 +210,6 @@ async def _patch_team_caches_add_access_group( for team_id in team_ids: cached_team = await _get_team_object_from_cache( key=f"team_id:{team_id}", - proxy_logging_obj=proxy_logging_obj, user_api_key_cache=user_api_key_cache, parent_otel_span=None, ) @@ -240,7 +239,6 @@ async def _patch_team_caches_remove_access_group( for team_id in team_ids: cached_team = await _get_team_object_from_cache( key=f"team_id:{team_id}", - proxy_logging_obj=proxy_logging_obj, user_api_key_cache=user_api_key_cache, parent_otel_span=None, ) diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index abe73f7d05c..06fd3730f7d 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -4869,10 +4869,9 @@ async def test_cache_team_object_writes_team_id_and_invalidates_team_alias(): 3. When team_alias is None, NO alias-key operation happens (no delete of an empty-keyed entry, no spurious write). 4. DELETES the team_id-keyed entry from the internal usage cache - BEFORE the fresh write (LIT-4391). `_get_team_object_from_cache` - consults the internal usage cache first, so a leftover copy there - (backfilled from a Redis shared with `user_api_key_cache`) would - keep serving the pre-update team allowlist. + BEFORE the fresh write (LIT-4391). `_get_team_object_from_cache` no + longer reads the internal usage cache (LIT-5944), but the delete + protects mixed-version rolling deploys where older workers still do. """ from unittest.mock import AsyncMock, MagicMock @@ -4981,8 +4980,10 @@ async def test_team_update_not_shadowed_by_internal_usage_cache_lit_4391(): Regression test for LIT-4391: keys with models=["all-team-models"] kept getting 403 team_model_access_denied for models added via /team/update. - `_get_team_object_from_cache` consults the internal usage cache BEFORE - `user_api_key_cache`. When both share one Redis (enable_redis_auth_cache), + `_get_team_object_from_cache` used to consult the internal usage cache + BEFORE `user_api_key_cache` (removed in LIT-5944; this test now also + guards against reintroducing that read). + When both share one Redis (enable_redis_auth_cache), any team read backfills the internal cache's in-memory tier with the team object. `_cache_team_object` (the /team/update refresh) only wrote `user_api_key_cache`, so that backfilled copy kept shadowing the update @@ -5054,6 +5055,75 @@ async def test_team_update_not_shadowed_by_internal_usage_cache_lit_4391(): ) +class _CountingFakeRedis(_SharedFakeRedis): + """Counts per-key Redis round-trips so tests can pin the number of + network operations a code path issues.""" + + def __init__(self): + super().__init__() + self.get_calls: int = 0 + + async def async_get_cache(self, key, **kwargs): + self.get_calls += 1 + return await super().async_get_cache(key, **kwargs) + + +@pytest.mark.asyncio +async def test_warm_team_object_reads_issue_no_redis_ops_lit_5944(): + """ + Regression test for LIT-5944: project/team-scoped virtual-key requests + paid ~4 awaited Redis GETs per request just to re-read the team object. + + `_get_team_object_from_cache` used to consult + `proxy_logging_obj.internal_usage_cache.dual_cache` (in-memory TTL 1s, + Redis-backed) BEFORE `user_api_key_cache`. Nothing writes team objects + into that internal cache — `_cache_team_object` only DELETES the key + there — so when `user_api_key_cache` has no Redis tier the shared Redis + key stays absent forever and every team lookup in the auth hot path + (4 call sites per chat-completion request) became a guaranteed-miss + Redis round-trip, saturating the event loop at high TPS. + + Pins: once `_cache_team_object` has cached a team, repeated + `get_team_object` reads are served from `user_api_key_cache`'s in-memory + tier and issue ZERO Redis operations. + """ + from litellm.caching.dual_cache import DualCache + from litellm.proxy._types import LiteLLM_TeamTableCachedObj + from litellm.proxy.auth.auth_checks import _cache_team_object, get_team_object + + team_id = "team-lit-5944" + counting_redis = _CountingFakeRedis() + user_api_key_cache = UserApiKeyCache() + proxy_logging_obj = MagicMock() + proxy_logging_obj.internal_usage_cache.dual_cache = DualCache( + redis_cache=counting_redis, + default_in_memory_ttl=1, + ) + prisma_client = MagicMock() + + await _cache_team_object( + team_id=team_id, + team_table=LiteLLM_TeamTableCachedObj(team_id=team_id, models=["model-a"]), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + for _ in range(4): + team_obj = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + assert team_obj is not None and team_obj.models == ["model-a"] + + assert counting_redis.get_calls == 0, ( + "Warm team-object reads must be served from user_api_key_cache's " + "in-memory tier without any Redis round-trips. " + f"Got {counting_redis.get_calls} Redis GETs for 4 get_team_object calls." + ) + + @pytest.mark.asyncio async def test_cache_team_object_tolerates_cache_invalidation_failures(): """