diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index a55a1a577dc..ddcd02abe5f 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -70,7 +70,7 @@ class DualCache(BaseCache): self.redis_batch_cache_expiry = ( default_redis_batch_cache_expiry or litellm.default_redis_batch_cache_expiry - or 5 + or 10 ) self.default_in_memory_ttl = ( default_in_memory_ttl or litellm.default_in_memory_ttl diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a3bbb244e27..a1d4b781ab7 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -281,21 +281,6 @@ class OpenTelemetry(CustomLogger): # End Parent OTEL Sspan parent_otel_span.end(end_time=self._to_ns(datetime.now())) - async def async_post_call_success_hook( - self, - data: dict, - user_api_key_dict: UserAPIKeyAuth, - response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse], - ): - from opentelemetry import trace - from opentelemetry.trace import Status, StatusCode - - parent_otel_span = user_api_key_dict.parent_otel_span - if parent_otel_span is not None: - parent_otel_span.set_status(Status(StatusCode.OK)) - # End Parent OTEL Sspan - parent_otel_span.end(end_time=self._to_ns(datetime.now())) - def _handle_sucess(self, kwargs, response_obj, start_time, end_time): from opentelemetry import trace from opentelemetry.trace import Status, StatusCode diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 5bef202f6a1..43285237951 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -18,6 +18,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache +from litellm.caching.dual_cache import LimitedSizeOrderedDict from litellm.proxy._types import ( LiteLLM_EndUserTable, LiteLLM_JWTAuth, @@ -42,6 +43,10 @@ if TYPE_CHECKING: else: Span = Any + +last_db_access_time = LimitedSizeOrderedDict(max_size=100) +db_cache_expiry = 5 # refresh every 5s + all_routes = LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.management_routes.value @@ -383,6 +388,18 @@ def model_in_access_group(model: str, team_models: Optional[List[str]]) -> bool: return False +def _should_check_db( + key: str, last_db_access_time: LimitedSizeOrderedDict, db_cache_expiry: int +) -> bool: + current_time = time.time() + if ( + key not in last_db_access_time + or current_time - last_db_access_time[key] >= db_cache_expiry + ): + return True + return False + + @log_to_opentelemetry async def get_user_object( user_id: str, @@ -412,10 +429,18 @@ async def get_user_object( if prisma_client is None: raise Exception("No db connected") try: - - response = await prisma_client.db.litellm_usertable.find_unique( - where={"user_id": user_id}, include={"organization_memberships": True} + db_access_time_key = "user_id:{}".format(user_id) + should_check_db = _should_check_db( + key=db_access_time_key, + last_db_access_time=last_db_access_time, + db_cache_expiry=db_cache_expiry, ) + if should_check_db: + response = await prisma_client.db.litellm_usertable.find_unique( + where={"user_id": user_id}, include={"organization_memberships": True} + ) + else: + response = None if response is None: if user_id_upsert: @@ -444,6 +469,9 @@ async def get_user_object( # save the user object to cache await user_api_key_cache.async_set_cache(key=user_id, value=response_dict) + # save to db access time + last_db_access_time[db_access_time_key] = time.time() + return _response except Exception as e: # if user not in db raise ValueError( @@ -515,6 +543,12 @@ async def _delete_cache_key_object( @log_to_opentelemetry +async def _get_team_db_check(team_id: str, prisma_client: PrismaClient): + return await prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + + async def get_team_object( team_id: str, prisma_client: Optional[PrismaClient], @@ -544,7 +578,7 @@ async def get_team_object( ): cached_team_obj = ( await proxy_logging_obj.internal_usage_cache.dual_cache.async_get_cache( - key=key + key=key, parent_otel_span=parent_otel_span ) ) @@ -564,9 +598,18 @@ async def get_team_object( # else, check db try: - response = await prisma_client.db.litellm_teamtable.find_unique( - where={"team_id": team_id} + db_access_time_key = "team_id:{}".format(team_id) + should_check_db = _should_check_db( + key=db_access_time_key, + last_db_access_time=last_db_access_time, + db_cache_expiry=db_cache_expiry, ) + if should_check_db: + response = await _get_team_db_check( + team_id=team_id, prisma_client=prisma_client + ) + else: + response = None if response is None: raise Exception @@ -580,6 +623,9 @@ async def get_team_object( proxy_logging_obj=proxy_logging_obj, ) + # save to db access time + last_db_access_time[db_access_time_key] = time.time() + return _response except Exception: raise Exception(