From 2accbf360838cd57035d634eb8a157e18e37c3f3 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:48:58 +0000 Subject: [PATCH 1/9] fix(spend): drain spend log queues before prisma disconnect on shutdown --- litellm/proxy/proxy_server.py | 10 +++- litellm/proxy/utils.py | 4 ++ .../proxy/proxy_server/test_lifecycle.py | 46 +++++++++++++++++++ .../prisma_and_spend/test_spend_functions.py | 39 ++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d9f8e04ebda..fb71223f88a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -987,7 +987,15 @@ async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = N if worker_heartbeat is not None and prisma_client: await worker_heartbeat.deregister() if prisma_client: - # Drain the SGR fold first: it lives in memory, so an un-drained interval + # Request-time spend queues live in this process. A worker recycle + # (`--max_requests_before_restart`) or deploy otherwise drops whatever + # the periodic flush has not written. Drain while Prisma is still + # connected; a write after disconnect is ClientNotConnectedError and + # the rows never reach LiteLLM_SpendLogs. The same emptiness owner + # (`_total_queued_spend_transactions`) also covers + # tool_usage_transactions and autorouter_turn_transactions. + await _flush_spend_logs_queue_on_shutdown() + # Drain the SGR fold next: it lives in memory, so an un-drained interval # is lost, and a write attempted after disconnect raises # ClientNotConnectedError rather than persisting anything. Ordering this # inside the same guard is what keeps the two from drifting apart. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c095586b6c9..92510a0c1f8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7127,6 +7127,7 @@ async def update_spend_logs_job( MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20 +MAX_SPEND_LOG_DRAIN_SECONDS: Final = 15.0 async def drain_spend_logs_queue( @@ -7141,9 +7142,12 @@ async def drain_spend_logs_queue( await monitor_task prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle + deadline: Final = time.monotonic() + MAX_SPEND_LOG_DRAIN_SECONDS for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS): if await _total_queued_spend_transactions(prisma_client) == 0: return + if time.monotonic() >= deadline: + break await update_spend_logs_job( prisma_client=prisma_client, db_writer_client=db_writer_client, diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index deb7289d2d1..6ed3ba51a37 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -102,6 +102,7 @@ async def test_proxy_shutdown_event_disconnects_prisma_and_resets(monkeypatch): fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock() monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) monkeypatch.setattr(ps, "master_key", "sk-x", raising=False) fake_jwt = MagicMock() @@ -146,6 +147,7 @@ async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monke fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) async def _record_flush(client, accumulator): calls.append("flush") @@ -168,6 +170,49 @@ async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monke assert calls == ["flush", "disconnect"] +@pytest.mark.asyncio +async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch): + """ + SpendLogs, tool-usage, and auto-router turn queues live in memory. + + ``proxy_shutdown_event`` is the function that disconnects Prisma, so it + must drain those queues first. The lifespan helper already flushes once, + but anything queued after that — or a caller that hits this function + directly — is otherwise discarded on worker recycle with no log line. + Ordering is the behavior, so assert drain then disconnect. + """ + calls: list = [] # mutable-ok: records call order, which is the assertion + + fake_prisma = MagicMock() + fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + + async def _record_drain(): + calls.append("drain_spend") + + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", _record_drain, raising=False) + + async def _record_flush(client, accumulator): + calls.append("flush_gateway") + assert client is fake_prisma + + monkeypatch.setattr(ps, "flush_gateway_requests", _record_flush, raising=False) + + fake_jwt = MagicMock() + fake_jwt.close = AsyncMock() + monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + import litellm + + monkeypatch.setattr(litellm, "cache", None, raising=False) + monkeypatch.setattr(litellm, "success_callback", [], raising=False) + + await proxy_shutdown_event() + + assert calls == ["drain_spend", "flush_gateway", "disconnect"] + + @pytest.mark.asyncio async def test_proxy_shutdown_skips_gateway_flush_without_a_database(monkeypatch): """No prisma client means nothing to drain to, and no attempt is made.""" @@ -195,6 +240,7 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch): fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock(side_effect=RuntimeError("db gone")) monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) fake_jwt = MagicMock() fake_jwt.close = AsyncMock() diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py index c8b87bd671e..2adf0fe6dc4 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -19,6 +19,7 @@ import pytest from litellm.proxy.utils import ( MAX_SPEND_LOG_DRAIN_ITERATIONS, + MAX_SPEND_LOG_DRAIN_SECONDS, _monitor_spend_logs_queue, _raise_failed_update_spend_exception, drain_spend_logs_queue, @@ -458,6 +459,44 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes( ) +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_gives_up_when_time_budget_expires( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + import litellm.proxy.utils as utils_mod + + monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False) + monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + monotonic = {"t": 0.0} + + def _now() -> float: + return monotonic["t"] + + async def _write_and_refill(*args: Any, **kwargs: Any) -> None: + monotonic["t"] += MAX_SPEND_LOG_DRAIN_SECONDS + mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) + + monkeypatch.setattr(utils_mod.time, "monotonic", _now) + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + # First pass starts under the deadline; the write then consumes the whole + # budget, so the next loop exits instead of spinning to max iterations. + assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1 + + @pytest.mark.asyncio async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty( mock_prisma_client: Any, From 15574078f50a301fe0837cf754803158055a1b24 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:45:29 +0800 Subject: [PATCH 2/9] fix: bound spend log shutdown drain --- litellm/proxy/utils.py | 19 +++++++++-- .../prisma_and_spend/test_spend_functions.py | 33 ++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 92510a0c1f8..7d463d88e2e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7028,6 +7028,7 @@ async def update_spend_logs_job( prisma_client: PrismaClient, db_writer_client: AsyncHTTPHandler | None, proxy_logging_obj: ProxyLogging, + timeout: float | None = None, ): """ Job to process spend_log_transactions queue. @@ -7047,13 +7048,17 @@ async def update_spend_logs_job( logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL) try: - await ProxyUpdateSpend.update_spend_logs( + update_spend_logs = ProxyUpdateSpend.update_spend_logs( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, db_writer_client=db_writer_client, logs_to_process=logs_to_process, ) + if timeout is None: + await update_spend_logs + else: + await asyncio.wait_for(update_spend_logs, timeout=timeout) except asyncio.CancelledError: await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) verbose_proxy_logger.warning( @@ -7061,6 +7066,14 @@ async def update_spend_logs_job( len(logs_to_process), ) raise + except asyncio.TimeoutError: + await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) + verbose_proxy_logger.warning( + "Spend tracking - spend log write timed out after %.2f seconds; requeued %d rows", + timeout or 0.0, + len(logs_to_process), + ) + return # Guardrail/policy usage tracking (same batch, outside spend-logs update) try: @@ -7146,12 +7159,14 @@ async def drain_spend_logs_queue( for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS): if await _total_queued_spend_transactions(prisma_client) == 0: return - if time.monotonic() >= deadline: + remaining_seconds: Final = deadline - time.monotonic() + if remaining_seconds <= 0: break await update_spend_logs_job( prisma_client=prisma_client, db_writer_client=db_writer_client, proxy_logging_obj=proxy_logging_obj, + timeout=remaining_seconds, ) remaining: Final = await _total_queued_spend_transactions(prisma_client) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py index 2adf0fe6dc4..42864371a4e 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -479,7 +479,10 @@ async def test_drain_spend_logs_queue_gives_up_when_time_budget_expires( def _now() -> float: return monotonic["t"] - async def _write_and_refill(*args: Any, **kwargs: Any) -> None: + async def _write_and_refill( + *, data: list[dict[str, object]], skip_duplicates: bool + ) -> None: + del data, skip_duplicates monotonic["t"] += MAX_SPEND_LOG_DRAIN_SECONDS mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) @@ -497,6 +500,34 @@ async def test_drain_spend_logs_queue_gives_up_when_time_budget_expires( assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1 +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_requeues_when_write_exceeds_time_budget( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.utils as utils_mod + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + row = make_spend_log_row(request_id="r1") + mock_prisma_client.spend_log_transactions = [row] + monkeypatch.setattr(utils_mod, "MAX_SPEND_LOG_DRAIN_SECONDS", 0.01) + + async def _hang(**_: object) -> None: + await asyncio.Event().wait() + + write_mock = AsyncMock(side_effect=_hang) + monkeypatch.setattr(utils_mod.ProxyUpdateSpend, "update_spend_logs", write_mock) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert write_mock.await_count == 1 + assert mock_prisma_client.spend_log_transactions == [row] + + @pytest.mark.asyncio async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty( mock_prisma_client: Any, From ff51a7039967cf34444a8561ebc5c703561259f9 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:57:09 +0000 Subject: [PATCH 3/9] test: tighten spend-drain helper types --- .../proxy/proxy_server/test_lifecycle.py | 7 ++++--- .../prisma_and_spend/test_spend_functions.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 6ed3ba51a37..08e5eab8571 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -181,18 +181,19 @@ async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch directly — is otherwise discarded on worker recycle with no log line. Ordering is the behavior, so assert drain then disconnect. """ - calls: list = [] # mutable-ok: records call order, which is the assertion + calls: list[str] = [] # mutable-ok: records call order, which is the assertion fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) - async def _record_drain(): + async def _record_drain() -> None: calls.append("drain_spend") monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", _record_drain, raising=False) - async def _record_flush(client, accumulator): + async def _record_flush(client: object, accumulator: object) -> None: + del accumulator calls.append("flush_gateway") assert client is fake_prisma diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py index 42864371a4e..69ee3ffed1f 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -351,8 +351,9 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining( written: list[str] = [] - async def _write(*args: Any, **kwargs: Any) -> None: - written.extend(row["request_id"] for row in kwargs["data"]) + async def _write(*, data: list[dict[str, object]], skip_duplicates: bool) -> None: + del skip_duplicates + written.extend(row["request_id"] for row in data) if len(written) == 1: mock_prisma_client.spend_log_transactions.append( make_spend_log_row(request_id="r2") @@ -392,12 +393,13 @@ async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows( written: list[str] = [] write_calls = {"n": 0} - async def _write(*args: Any, **kwargs: Any) -> None: + async def _write(*, data: list[dict[str, object]], skip_duplicates: bool) -> None: + del skip_duplicates write_calls["n"] += 1 if write_calls["n"] == 1: write_started.set() await asyncio.Event().wait() - written.extend(row["request_id"] for row in kwargs["data"]) + written.extend(row["request_id"] for row in data) mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) @@ -440,7 +442,10 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes( proxy_logging.failure_handler = AsyncMock() mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] - async def _write_and_refill(*args: Any, **kwargs: Any) -> None: + async def _write_and_refill( + *, data: list[dict[str, object]], skip_duplicates: bool + ) -> None: + del data, skip_duplicates mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( From faede56e64643fd84fbc0197b952e95e19c0d953 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:05:57 +0000 Subject: [PATCH 4/9] fix: time out spend drain writes without parking a coroutine --- litellm/proxy/utils.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 7d463d88e2e..ee77c3a2d54 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7048,17 +7048,14 @@ async def update_spend_logs_job( logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL) try: - update_spend_logs = ProxyUpdateSpend.update_spend_logs( - n_retry_times=n_retry_times, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - db_writer_client=db_writer_client, - logs_to_process=logs_to_process, - ) - if timeout is None: - await update_spend_logs - else: - await asyncio.wait_for(update_spend_logs, timeout=timeout) + async with asyncio.timeout(timeout): + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + db_writer_client=db_writer_client, + logs_to_process=logs_to_process, + ) except asyncio.CancelledError: await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) verbose_proxy_logger.warning( @@ -7066,7 +7063,7 @@ async def update_spend_logs_job( len(logs_to_process), ) raise - except asyncio.TimeoutError: + except TimeoutError: await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) verbose_proxy_logger.warning( "Spend tracking - spend log write timed out after %.2f seconds; requeued %d rows", From 95756206e7ae2594627457c320e8d28c04cba837 Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:32:31 +0800 Subject: [PATCH 5/9] fix: avoid Final binding inside drain loop --- litellm/proxy/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ee77c3a2d54..f0a43e768aa 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7156,7 +7156,7 @@ async def drain_spend_logs_queue( for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS): if await _total_queued_spend_transactions(prisma_client) == 0: return - remaining_seconds: Final = deadline - time.monotonic() + remaining_seconds = deadline - time.monotonic() if remaining_seconds <= 0: break await update_spend_logs_job( From 190f53559536e3f3d47a441b2225a1822686a77a Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Sun, 30 Aug 2026 14:15:52 +0800 Subject: [PATCH 6/9] fix: use Python 3.10-compatible spend log timeout --- litellm/proxy/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f0a43e768aa..d74a0a6b60e 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7048,14 +7048,16 @@ async def update_spend_logs_job( logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL) try: - async with asyncio.timeout(timeout): - await ProxyUpdateSpend.update_spend_logs( + await asyncio.wait_for( + ProxyUpdateSpend.update_spend_logs( n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, db_writer_client=db_writer_client, logs_to_process=logs_to_process, - ) + ), + timeout=timeout, + ) except asyncio.CancelledError: await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) verbose_proxy_logger.warning( @@ -7063,7 +7065,7 @@ async def update_spend_logs_job( len(logs_to_process), ) raise - except TimeoutError: + except asyncio.TimeoutError: await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True) verbose_proxy_logger.warning( "Spend tracking - spend log write timed out after %.2f seconds; requeued %d rows", From 14608394de6906e282cc653aa2510935babe163f Mon Sep 17 00:00:00 2001 From: lei_lei Date: Sun, 13 Sep 2026 07:46:06 +0000 Subject: [PATCH 7/9] chore(ui): regenerate schema.d.ts for OpenAPI sync --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..839aa52fa84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) From 57f2a0532e1131be131580d83826e9248780a1bb Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Wed, 16 Sep 2026 15:09:44 +0000 Subject: [PATCH 8/9] fix: await logging callbacks before spend drain on shutdown Detached streaming success callbacks enqueue spend through GLOBAL_LOGGING_WORKER; flush them before draining queues so late rows are not lost when Prisma disconnects. --- litellm/proxy/proxy_server.py | 29 +++ .../proxy/proxy_server/test_lifecycle.py | 171 ++++++++++++++---- 2 files changed, 160 insertions(+), 40 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fb71223f88a..ba6f18f9b70 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -965,6 +965,28 @@ async def flush_spend_counters_on_shutdown() -> None: verbose_proxy_logger.exception("Error flushing spend counters on shutdown: %s", e) +async def _await_logging_callbacks_on_shutdown() -> None: + """Let detached stream success callbacks finish before spend drain. + + Streaming paths enqueue spend via ``GLOBAL_LOGGING_WORKER``. If we drain + spend queues (and disconnect Prisma) while those coroutines are still + queued, the callback can enqueue after the final drain and the row is + lost. Bound the wait so a stuck callback cannot hang shutdown forever. + """ + from litellm.constants import MAX_TIME_TO_CLEAR_QUEUE + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + try: + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=MAX_TIME_TO_CLEAR_QUEUE) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "Timed out waiting for logging callbacks on shutdown after %ss; continuing spend drain", + MAX_TIME_TO_CLEAR_QUEUE, + ) + except Exception as e: # noqa: BLE001 # shutdown must continue even if callback flush fails + verbose_proxy_logger.exception("Error awaiting logging callbacks on shutdown: %s", e) + + async def _flush_spend_logs_queue_on_shutdown() -> None: if prisma_client is None: return @@ -987,6 +1009,9 @@ async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = N if worker_heartbeat is not None and prisma_client: await worker_heartbeat.deregister() if prisma_client: + # Detached streaming callbacks can still enqueue spend after an earlier + # lifespan drain. Await them first so the drain below sees those rows. + await _await_logging_callbacks_on_shutdown() # Request-time spend queues live in this process. A worker recycle # (`--max_requests_before_restart`) or deploy otherwise drops whatever # the periodic flush has not written. Drain while Prisma is still @@ -1408,6 +1433,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: except Exception as e: verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) + # Streaming spend callbacks enqueue through GLOBAL_LOGGING_WORKER; await + # them before draining the producer/queues so late rows are not dropped. + await _await_logging_callbacks_on_shutdown() + await _drain_spend_event_producer_on_shutdown() await flush_spend_counters_on_shutdown() diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 08e5eab8571..062a64d5956 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -97,11 +97,52 @@ def test_cleanup_router_config_variables_fails_on_unknown_attr_raises(): # --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monkeypatch): + """ + The gateway request fold lives in memory, so shutdown drains it to the database. + + That drain has to happen while prisma is still connected: a write attempted + after ``disconnect()`` raises ClientNotConnectedError, the flush swallows it + and merges the counts back onto an accumulator the process is about to + discard, and the final interval is lost silently on every restart. Ordering is + the whole behavior here, so assert the order rather than that both ran. + """ + calls: list = [] # mutable-ok: records call order, which is the assertion + + fake_prisma = MagicMock() + fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False) + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) + + async def _record_flush(client, accumulator): + calls.append("flush") + assert client is fake_prisma + + monkeypatch.setattr(ps, "flush_gateway_requests", _record_flush, raising=False) + + fake_jwt = MagicMock() + fake_jwt.close = AsyncMock() + monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + import litellm + + monkeypatch.setattr(litellm, "cache", None, raising=False) + monkeypatch.setattr(litellm, "success_callback", [], raising=False) + + await proxy_shutdown_event() + + assert calls == ["flush", "disconnect"] + + @pytest.mark.asyncio async def test_proxy_shutdown_event_disconnects_prisma_and_resets(monkeypatch): fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock() monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False) monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) monkeypatch.setattr(ps, "master_key", "sk-x", raising=False) @@ -131,45 +172,6 @@ async def test_proxy_shutdown_event_disconnects_prisma_and_resets(monkeypatch): } -@pytest.mark.asyncio -async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monkeypatch): - """ - The gateway request fold lives in memory, so shutdown drains it to the database. - - That drain has to happen while prisma is still connected: a write attempted - after ``disconnect()`` raises ClientNotConnectedError, the flush swallows it - and merges the counts back onto an accumulator the process is about to - discard, and the final interval is lost silently on every restart. Ordering is - the whole behavior here, so assert the order rather than that both ran. - """ - calls: list = [] # mutable-ok: records call order, which is the assertion - - fake_prisma = MagicMock() - fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) - monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) - monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) - - async def _record_flush(client, accumulator): - calls.append("flush") - assert client is fake_prisma - - monkeypatch.setattr(ps, "flush_gateway_requests", _record_flush, raising=False) - - fake_jwt = MagicMock() - fake_jwt.close = AsyncMock() - monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False) - monkeypatch.setattr(ps, "db_writer_client", None, raising=False) - - import litellm - - monkeypatch.setattr(litellm, "cache", None, raising=False) - monkeypatch.setattr(litellm, "success_callback", [], raising=False) - - await proxy_shutdown_event() - - assert calls == ["flush", "disconnect"] - - @pytest.mark.asyncio async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch): """ @@ -187,6 +189,11 @@ async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + async def _record_await_callbacks() -> None: + calls.append("await_callbacks") + + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", _record_await_callbacks, raising=False) + async def _record_drain() -> None: calls.append("drain_spend") @@ -211,7 +218,89 @@ async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch await proxy_shutdown_event() - assert calls == ["drain_spend", "flush_gateway", "disconnect"] + assert calls == ["await_callbacks", "drain_spend", "flush_gateway", "disconnect"] + + +@pytest.mark.asyncio +async def test_proxy_shutdown_awaits_logging_callbacks_before_spend_drain(monkeypatch): + """ + Detached streaming success callbacks enqueue spend through the logging + worker. Awaiting them before the spend drain is what closes the race + Greptile flagged: callback finishes after drain, Prisma disconnects, row + never reaches LiteLLM_SpendLogs. + """ + calls: list[str] = [] # mutable-ok: records call order, which is the assertion + + fake_prisma = MagicMock() + fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect")) + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + + async def _record_await_callbacks() -> None: + calls.append("await_callbacks") + + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", _record_await_callbacks, raising=False) + + async def _record_drain() -> None: + calls.append("drain_spend") + + monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", _record_drain, raising=False) + async def _record_gateway(*_a: object, **_k: object) -> None: + calls.append("flush_gateway") + + monkeypatch.setattr(ps, "flush_gateway_requests", _record_gateway, raising=False) + + fake_jwt = MagicMock() + fake_jwt.close = AsyncMock() + monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + import litellm + + monkeypatch.setattr(litellm, "cache", None, raising=False) + monkeypatch.setattr(litellm, "success_callback", [], raising=False) + + await proxy_shutdown_event() + + assert calls.index("await_callbacks") < calls.index("drain_spend") < calls.index("disconnect") + + +@pytest.mark.asyncio +async def test_await_logging_callbacks_on_shutdown_flushes_worker(monkeypatch): + flush = AsyncMock() + + class _Worker: + async def flush(self) -> None: + await flush() + + monkeypatch.setattr( + "litellm.litellm_core_utils.logging_worker.GLOBAL_LOGGING_WORKER", + _Worker(), + raising=False, + ) + monkeypatch.setattr("litellm.constants.MAX_TIME_TO_CLEAR_QUEUE", 1.0, raising=False) + + await ps._await_logging_callbacks_on_shutdown() + + assert flush.await_count == 1 + + +@pytest.mark.asyncio +async def test_await_logging_callbacks_on_shutdown_continues_after_timeout(monkeypatch): + async def _hang() -> None: + await asyncio.sleep(60) + + class _Worker: + async def flush(self) -> None: + await _hang() + + monkeypatch.setattr( + "litellm.litellm_core_utils.logging_worker.GLOBAL_LOGGING_WORKER", + _Worker(), + raising=False, + ) + monkeypatch.setattr("litellm.constants.MAX_TIME_TO_CLEAR_QUEUE", 0.01, raising=False) + + await ps._await_logging_callbacks_on_shutdown() @pytest.mark.asyncio @@ -220,6 +309,7 @@ async def test_proxy_shutdown_skips_gateway_flush_without_a_database(monkeypatch flush = AsyncMock() monkeypatch.setattr(ps, "flush_gateway_requests", flush, raising=False) monkeypatch.setattr(ps, "prisma_client", None, raising=False) + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False) fake_jwt = MagicMock() fake_jwt.close = AsyncMock() @@ -241,6 +331,7 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch): fake_prisma = MagicMock() fake_prisma.disconnect = AsyncMock(side_effect=RuntimeError("db gone")) monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False) monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False) fake_jwt = MagicMock() From b8242d0c233c5d4853237f05c2b0d007ac206ce8 Mon Sep 17 00:00:00 2001 From: leilei3167 Date: Wed, 16 Sep 2026 15:24:27 +0000 Subject: [PATCH 9/9] test: assert shutdown callback flush times out cleanly --- .../test_litellm/proxy/proxy_server/test_lifecycle.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 062a64d5956..b79ddeccf48 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -285,7 +285,7 @@ async def test_await_logging_callbacks_on_shutdown_flushes_worker(monkeypatch): @pytest.mark.asyncio -async def test_await_logging_callbacks_on_shutdown_continues_after_timeout(monkeypatch): +async def test_await_logging_callbacks_on_shutdown_continues_after_timeout(monkeypatch, caplog): async def _hang() -> None: await asyncio.sleep(60) @@ -300,7 +300,13 @@ async def test_await_logging_callbacks_on_shutdown_continues_after_timeout(monke ) monkeypatch.setattr("litellm.constants.MAX_TIME_TO_CLEAR_QUEUE", 0.01, raising=False) - await ps._await_logging_callbacks_on_shutdown() + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + started = asyncio.get_running_loop().time() + await ps._await_logging_callbacks_on_shutdown() + elapsed = asyncio.get_running_loop().time() - started + + assert elapsed < 5 + assert "Timed out waiting for logging callbacks on shutdown" in caplog.text @pytest.mark.asyncio