From cc63f1feb1ffb764c7ab5f397b8053cf59596435 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:19:19 +0800 Subject: [PATCH] fix(alerts): return retryable errors when analytics admission times out --- litellm/repositories/analytics_budget.py | 5 ++++ .../repositories/test_analytics_budget.py | 29 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/litellm/repositories/analytics_budget.py b/litellm/repositories/analytics_budget.py index 11ac67d970a..06428b9a8a7 100644 --- a/litellm/repositories/analytics_budget.py +++ b/litellm/repositories/analytics_budget.py @@ -6,6 +6,7 @@ from datetime import timedelta from threading import BoundedSemaphore from typing import Final, Protocol +from prisma.engine.errors import EngineRequestError from prisma.errors import RawQueryError, TransactionExpiredError, TransactionNotStartedError from pydantic import BaseModel @@ -109,6 +110,10 @@ class BoundedAnalyticsDatabase: return await asyncio.wait_for( self._query(query, args), timeout=6.25 if remaining is None else min(6.25, remaining) ) + except EngineRequestError as error: + if error.response.status not in (503, 504): + raise + raise AnalyticsCapacityError("Analytics exceeded its connection budget. Try again shortly") from error except RawQueryError as error: if _PostgresError.model_validate(error.meta or {}).code not in ("57014", "55P03"): raise diff --git a/tests/unit/repositories/test_analytics_budget.py b/tests/unit/repositories/test_analytics_budget.py index 90f8ab83f3b..1c76bd6dcf6 100644 --- a/tests/unit/repositories/test_analytics_budget.py +++ b/tests/unit/repositories/test_analytics_budget.py @@ -6,7 +6,10 @@ from contextlib import asynccontextmanager, nullcontext from datetime import timedelta from typing import Final, Literal +import httpx import pytest +from prisma.engine.errors import EngineRequestError +from prisma.http import Response from litellm.repositories.analytics_budget import ( AnalyticsCapacityError, @@ -36,18 +39,23 @@ class _ClockedLoop(asyncio.SelectorEventLoop): class _Driver: - def __init__(self, stage: Literal["begin", "query", "commit", "rollback", "healthy"]) -> None: + def __init__( + self, stage: Literal["begin", "query", "commit", "rollback", "healthy"], *, error: BaseException | None = None + ) -> None: self.stage = stage + self.error = error self.cancelled = asyncio.Event() async def _stall(self) -> None: + if self.error is not None: + raise self.error try: await asyncio.Event().wait() finally: self.cancelled.set() @asynccontextmanager - async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction, None]: if self.stage == "begin": await self._stall() try: @@ -161,7 +169,7 @@ class _TrackedDriver: self.engine = _TrackedPrismaEngine(engine, self.tracker) @asynccontextmanager - async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction, None]: identifier: Final = await self.engine.start_transaction(content="{}") try: yield self @@ -225,3 +233,18 @@ async def test_cancelled_analytics_keeps_admission_until_detached_engine_work_an ({"result": 1},), ({"result": 1},), ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", (503, 504, 422)) +async def test_engine_admission_timeout_is_retryable_but_invalid_requests_are_not(status: int) -> None: + response: Final = Response(httpx.Response(status)) + error: Final = EngineRequestError(response, '{"error_code":"P2028"}') + expected: Final = AnalyticsCapacityError if status in (503, 504) else EngineRequestError + with pytest.raises(expected): + await BoundedAnalyticsDatabase(_Driver("begin", error=error)).query_raw("SELECT 1 AS result") + healthy: Final = BoundedAnalyticsDatabase(_Driver("healthy")) + assert await asyncio.gather(healthy.query_raw("SELECT 1 AS result"), healthy.query_raw("SELECT 1 AS result")) == [ + ({"result": 1},), + ({"result": 1},), + ]