scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(auth): cache prefetched org entries with the management TTL - the 5s default fuse expired before slow requests reached get_org_object, sending them to the DB (proxy-behavior auth prefetch test failed under load with dead_db TypeError)

Browse source
This commit is contained in:
onatozmenn 2026-09-14 01:21:38 +03:00
parent 14d81858db
commit d1cea94f49
No known key found for this signature in database
2 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/proxy/auth/auth_object_prefetch.py
View file

@ -14,7 +14,6 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.redis_cache import RedisCache
from litellm.constants import DEFAULT_IN_MEMORY_TTL
from litellm.models.organization import LiteLLM_OrganizationTable
from litellm.models.team import LiteLLM_TeamTableCachedObj
from litellm.models.team_membership import LiteLLM_TeamMembership
@ -190,14 +189,17 @@ def _iter_entries(refs: AuthObjectRefs, management_ttl: float) -> Iterator[_Cach
None,
)
if refs.organization_id is not None:
# Organization entries use the management TTL like every other prefetched object: a
# 5s fuse expires before slow requests reach the getters, sending them back to
# the DB the prefetch was meant to spare.
yield _CacheEntry(
f"org_id:{refs.organization_id}", "organization_row", LiteLLM_OrganizationTable, DEFAULT_IN_MEMORY_TTL
f"org_id:{refs.organization_id}", "organization_row", LiteLLM_OrganizationTable, management_ttl
)
yield _CacheEntry(
f"org_id:{refs.organization_id}:with_budget",
"organization_row",
LiteLLM_OrganizationTable,
DEFAULT_IN_MEMORY_TTL,
management_ttl,
)
if refs.project_id is not None:
yield _CacheEntry(f"project_id:{refs.project_id}", "project_row", LiteLLM_ProjectTableCachedObj, management_ttl)

4
tests/test_litellm/proxy/auth/test_auth_object_prefetch.py
View file

@ -181,8 +181,8 @@ async def test_cold_regime_is_one_mget_one_query_and_the_getters_never_touch_io_
assert sets == sorted(
[
f"SET {TEAM_ID}_{USER_ID} ttl=5",
f"SET org_id:{ORG_ID} ttl=5",
f"SET org_id:{ORG_ID}:with_budget ttl=5",
f"SET org_id:{ORG_ID} ttl=60",
f"SET org_id:{ORG_ID}:with_budget ttl=60",
f"SET {USER_ID} ttl=60",
f"SET team_id:{TEAM_ID} ttl=60",
f"SET team_membership:{USER_ID}:{TEAM_ID} ttl=None",