This commit is contained in:
leilei3167 2026-09-19 14:58:17 -04:00 • committed by GitHub
commit 5d0f0060b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 297 additions and 23 deletions

View file

@ -1012,6 +1012,28 @@ async def flush_spend_counters_on_shutdown() -> None:
verbose_proxy_logger.exception("Error flushing spend counters on shutdown: %s", e)
async def _await_logging_callbacks_on_shutdown() -> None:
"""Let detached stream success callbacks finish before spend drain.
Streaming paths enqueue spend via ``GLOBAL_LOGGING_WORKER``. If we drain
spend queues (and disconnect Prisma) while those coroutines are still
queued, the callback can enqueue after the final drain and the row is
lost. Bound the wait so a stuck callback cannot hang shutdown forever.
"""
from litellm.constants import MAX_TIME_TO_CLEAR_QUEUE
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
try:
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=MAX_TIME_TO_CLEAR_QUEUE)
except asyncio.TimeoutError:
verbose_proxy_logger.warning(
"Timed out waiting for logging callbacks on shutdown after %ss; continuing spend drain",
MAX_TIME_TO_CLEAR_QUEUE,
)
except Exception as e: # noqa: BLE001 # shutdown must continue even if callback flush fails
verbose_proxy_logger.exception("Error awaiting logging callbacks on shutdown: %s", e)
async def _flush_spend_logs_queue_on_shutdown() -> None:
if prisma_client is None:
return
@ -1034,7 +1056,18 @@ async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = N
if worker_heartbeat is not None and prisma_client:
await worker_heartbeat.deregister()
if prisma_client:
# Drain the SGR fold first: it lives in memory, so an un-drained interval
# Detached streaming callbacks can still enqueue spend after an earlier
# lifespan drain. Await them first so the drain below sees those rows.
await _await_logging_callbacks_on_shutdown()
# Request-time spend queues live in this process. A worker recycle
# (`--max_requests_before_restart`) or deploy otherwise drops whatever
# the periodic flush has not written. Drain while Prisma is still
# connected; a write after disconnect is ClientNotConnectedError and
# the rows never reach LiteLLM_SpendLogs. The same emptiness owner
# (`_total_queued_spend_transactions`) also covers
# tool_usage_transactions and autorouter_turn_transactions.
await _flush_spend_logs_queue_on_shutdown()
# Drain the SGR fold next: it lives in memory, so an un-drained interval
# is lost, and a write attempted after disconnect raises
# ClientNotConnectedError rather than persisting anything. Ordering this
# inside the same guard is what keeps the two from drifting apart.
@ -1464,6 +1497,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception as e:
verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e)
# Streaming spend callbacks enqueue through GLOBAL_LOGGING_WORKER; await
# them before draining the producer/queues so late rows are not dropped.
await _await_logging_callbacks_on_shutdown()
await _drain_spend_event_producer_on_shutdown()
await flush_spend_counters_on_shutdown()

View file

@ -7208,6 +7208,7 @@ async def update_spend_logs_job(
prisma_client: PrismaClient,
db_writer_client: AsyncHTTPHandler | None,
proxy_logging_obj: ProxyLogging,
timeout: float | None = None,
):
"""
Job to process spend_log_transactions queue.
@ -7227,12 +7228,15 @@ async def update_spend_logs_job(
logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL)
try:
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
db_writer_client=db_writer_client,
logs_to_process=logs_to_process,
await asyncio.wait_for(
ProxyUpdateSpend.update_spend_logs(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
db_writer_client=db_writer_client,
logs_to_process=logs_to_process,
),
timeout=timeout,
)
except asyncio.CancelledError:
await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True)
@ -7241,6 +7245,14 @@ async def update_spend_logs_job(
len(logs_to_process),
)
raise
except asyncio.TimeoutError:
await enqueue_spend_logs(prisma_client, logs_to_process, at_head=True)
verbose_proxy_logger.warning(
"Spend tracking - spend log write timed out after %.2f seconds; requeued %d rows",
timeout or 0.0,
len(logs_to_process),
)
return
# Guardrail/policy usage tracking (same batch, outside spend-logs update)
try:
@ -7307,6 +7319,7 @@ async def update_spend_logs_job(
MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20
MAX_SPEND_LOG_DRAIN_SECONDS: Final = 15.0
async def drain_spend_logs_queue(
@ -7321,13 +7334,18 @@ async def drain_spend_logs_queue(
await monitor_task
prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle
deadline: Final = time.monotonic() + MAX_SPEND_LOG_DRAIN_SECONDS
for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS):
if await _total_queued_spend_transactions(prisma_client) == 0:
return
remaining_seconds = deadline - time.monotonic()
if remaining_seconds <= 0:
break
await update_spend_logs_job(
prisma_client=prisma_client,
db_writer_client=db_writer_client,
proxy_logging_obj=proxy_logging_obj,
timeout=remaining_seconds,
)
remaining: Final = await _total_queued_spend_transactions(prisma_client)

View file

@ -98,11 +98,53 @@ def test_cleanup_router_config_variables_fails_on_unknown_attr_raises():
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monkeypatch):
"""
The gateway request fold lives in memory, so shutdown drains it to the database.
That drain has to happen while prisma is still connected: a write attempted
after ``disconnect()`` raises ClientNotConnectedError, the flush swallows it
and merges the counts back onto an accumulator the process is about to
discard, and the final interval is lost silently on every restart. Ordering is
the whole behavior here, so assert the order rather than that both ran.
"""
calls: list = [] # mutable-ok: records call order, which is the assertion
fake_prisma = MagicMock()
fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect"))
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False)
monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False)
async def _record_flush(client, accumulator):
calls.append("flush")
assert client is fake_prisma
monkeypatch.setattr(ps, "flush_gateway_requests", _record_flush, raising=False)
fake_jwt = MagicMock()
fake_jwt.close = AsyncMock()
monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False)
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
import litellm
monkeypatch.setattr(litellm, "cache", None, raising=False)
monkeypatch.setattr(litellm, "success_callback", [], raising=False)
await proxy_shutdown_event()
assert calls == ["flush", "disconnect"]
@pytest.mark.asyncio
async def test_proxy_shutdown_event_disconnects_prisma_and_resets(monkeypatch):
fake_prisma = MagicMock()
fake_prisma.disconnect = AsyncMock()
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False)
monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False)
monkeypatch.setattr(ps, "master_key", "sk-x", raising=False)
fake_jwt = MagicMock()
@ -132,24 +174,35 @@ async def test_proxy_shutdown_event_disconnects_prisma_and_resets(monkeypatch):
@pytest.mark.asyncio
async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monkeypatch):
async def test_proxy_shutdown_drains_spend_logs_before_disconnecting(monkeypatch):
"""
The gateway request fold lives in memory, so shutdown drains it to the database.
SpendLogs, tool-usage, and auto-router turn queues live in memory.
That drain has to happen while prisma is still connected: a write attempted
after ``disconnect()`` raises ClientNotConnectedError, the flush swallows it
and merges the counts back onto an accumulator the process is about to
discard, and the final interval is lost silently on every restart. Ordering is
the whole behavior here, so assert the order rather than that both ran.
``proxy_shutdown_event`` is the function that disconnects Prisma, so it
must drain those queues first. The lifespan helper already flushes once,
but anything queued after that — or a caller that hits this function
directly — is otherwise discarded on worker recycle with no log line.
Ordering is the behavior, so assert drain then disconnect.
"""
calls: list = [] # mutable-ok: records call order, which is the assertion
calls: list[str] = [] # mutable-ok: records call order, which is the assertion
fake_prisma = MagicMock()
fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect"))
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
async def _record_flush(client, accumulator):
calls.append("flush")
async def _record_await_callbacks() -> None:
calls.append("await_callbacks")
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", _record_await_callbacks, raising=False)
async def _record_drain() -> None:
calls.append("drain_spend")
monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", _record_drain, raising=False)
async def _record_flush(client: object, accumulator: object) -> None:
del accumulator
calls.append("flush_gateway")
assert client is fake_prisma
monkeypatch.setattr(ps, "flush_gateway_requests", _record_flush, raising=False)
@ -166,7 +219,95 @@ async def test_proxy_shutdown_drains_gateway_requests_before_disconnecting(monke
await proxy_shutdown_event()
assert calls == ["flush", "disconnect"]
assert calls == ["await_callbacks", "drain_spend", "flush_gateway", "disconnect"]
@pytest.mark.asyncio
async def test_proxy_shutdown_awaits_logging_callbacks_before_spend_drain(monkeypatch):
"""
Detached streaming success callbacks enqueue spend through the logging
worker. Awaiting them before the spend drain is what closes the race
Greptile flagged: callback finishes after drain, Prisma disconnects, row
never reaches LiteLLM_SpendLogs.
"""
calls: list[str] = [] # mutable-ok: records call order, which is the assertion
fake_prisma = MagicMock()
fake_prisma.disconnect = AsyncMock(side_effect=lambda: calls.append("disconnect"))
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
async def _record_await_callbacks() -> None:
calls.append("await_callbacks")
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", _record_await_callbacks, raising=False)
async def _record_drain() -> None:
calls.append("drain_spend")
monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", _record_drain, raising=False)
async def _record_gateway(*_a: object, **_k: object) -> None:
calls.append("flush_gateway")
monkeypatch.setattr(ps, "flush_gateway_requests", _record_gateway, raising=False)
fake_jwt = MagicMock()
fake_jwt.close = AsyncMock()
monkeypatch.setattr(ps, "jwt_handler", fake_jwt, raising=False)
monkeypatch.setattr(ps, "db_writer_client", None, raising=False)
import litellm
monkeypatch.setattr(litellm, "cache", None, raising=False)
monkeypatch.setattr(litellm, "success_callback", [], raising=False)
await proxy_shutdown_event()
assert calls.index("await_callbacks") < calls.index("drain_spend") < calls.index("disconnect")
@pytest.mark.asyncio
async def test_await_logging_callbacks_on_shutdown_flushes_worker(monkeypatch):
flush = AsyncMock()
class _Worker:
async def flush(self) -> None:
await flush()
monkeypatch.setattr(
"litellm.litellm_core_utils.logging_worker.GLOBAL_LOGGING_WORKER",
_Worker(),
raising=False,
)
monkeypatch.setattr("litellm.constants.MAX_TIME_TO_CLEAR_QUEUE", 1.0, raising=False)
await ps._await_logging_callbacks_on_shutdown()
assert flush.await_count == 1
@pytest.mark.asyncio
async def test_await_logging_callbacks_on_shutdown_continues_after_timeout(monkeypatch, caplog):
async def _hang() -> None:
await asyncio.sleep(60)
class _Worker:
async def flush(self) -> None:
await _hang()
monkeypatch.setattr(
"litellm.litellm_core_utils.logging_worker.GLOBAL_LOGGING_WORKER",
_Worker(),
raising=False,
)
monkeypatch.setattr("litellm.constants.MAX_TIME_TO_CLEAR_QUEUE", 0.01, raising=False)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
started = asyncio.get_running_loop().time()
await ps._await_logging_callbacks_on_shutdown()
elapsed = asyncio.get_running_loop().time() - started
assert elapsed < 5
assert "Timed out waiting for logging callbacks on shutdown" in caplog.text
@pytest.mark.asyncio
@ -175,6 +316,7 @@ async def test_proxy_shutdown_skips_gateway_flush_without_a_database(monkeypatch
flush = AsyncMock()
monkeypatch.setattr(ps, "flush_gateway_requests", flush, raising=False)
monkeypatch.setattr(ps, "prisma_client", None, raising=False)
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False)
fake_jwt = MagicMock()
fake_jwt.close = AsyncMock()
@ -196,6 +338,8 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch):
fake_prisma = MagicMock()
fake_prisma.disconnect = AsyncMock(side_effect=RuntimeError("db gone"))
monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False)
monkeypatch.setattr(ps, "_await_logging_callbacks_on_shutdown", AsyncMock(), raising=False)
monkeypatch.setattr(ps, "_flush_spend_logs_queue_on_shutdown", AsyncMock(), raising=False)
fake_jwt = MagicMock()
fake_jwt.close = AsyncMock()

View file

@ -19,6 +19,7 @@ import pytest
from litellm.proxy.utils import (
MAX_SPEND_LOG_DRAIN_ITERATIONS,
MAX_SPEND_LOG_DRAIN_SECONDS,
_monitor_spend_logs_queue,
_raise_failed_update_spend_exception,
drain_spend_logs_queue,
@ -350,8 +351,9 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
written: list[str] = []
async def _write(*args: Any, **kwargs: Any) -> None:
written.extend(row["request_id"] for row in kwargs["data"])
async def _write(*, data: list[dict[str, object]], skip_duplicates: bool) -> None:
del skip_duplicates
written.extend(row["request_id"] for row in data)
if len(written) == 1:
mock_prisma_client.spend_log_transactions.append(
make_spend_log_row(request_id="r2")
@ -391,12 +393,13 @@ async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows(
written: list[str] = []
write_calls = {"n": 0}
async def _write(*args: Any, **kwargs: Any) -> None:
async def _write(*, data: list[dict[str, object]], skip_duplicates: bool) -> None:
del skip_duplicates
write_calls["n"] += 1
if write_calls["n"] == 1:
write_started.set()
await asyncio.Event().wait()
written.extend(row["request_id"] for row in kwargs["data"])
written.extend(row["request_id"] for row in data)
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
@ -439,7 +442,10 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes(
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
async def _write_and_refill(
*, data: list[dict[str, object]], skip_duplicates: bool
) -> None:
del data, skip_duplicates
mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
@ -458,6 +464,75 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes(
)
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_gives_up_when_time_budget_expires(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.db.spend_log_tool_index as tool_mod
import litellm.proxy.guardrails.usage_tracking as guard_mod
import litellm.proxy.utils as utils_mod
monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
monotonic = {"t": 0.0}
def _now() -> float:
return monotonic["t"]
async def _write_and_refill(
*, data: list[dict[str, object]], skip_duplicates: bool
) -> None:
del data, skip_duplicates
monotonic["t"] += MAX_SPEND_LOG_DRAIN_SECONDS
mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
monkeypatch.setattr(utils_mod.time, "monotonic", _now)
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
# First pass starts under the deadline; the write then consumes the whole
# budget, so the next loop exits instead of spinning to max iterations.
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1
@pytest.mark.asyncio
async def test_drain_spend_logs_queue_requeues_when_write_exceeds_time_budget(
mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm.proxy.utils as utils_mod
proxy_logging = MagicMock()
proxy_logging.failure_handler = AsyncMock()
row = make_spend_log_row(request_id="r1")
mock_prisma_client.spend_log_transactions = [row]
monkeypatch.setattr(utils_mod, "MAX_SPEND_LOG_DRAIN_SECONDS", 0.01)
async def _hang(**_: object) -> None:
await asyncio.Event().wait()
write_mock = AsyncMock(side_effect=_hang)
monkeypatch.setattr(utils_mod.ProxyUpdateSpend, "update_spend_logs", write_mock)
await drain_spend_logs_queue(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert write_mock.await_count == 1
assert mock_prisma_client.spend_log_transactions == [row]
@pytest.mark.asyncio
async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty(
mock_prisma_client: Any,