diff --git a/.env.example b/.env.example index dc1fd5a6ccb..23de0084e9e 100644 --- a/.env.example +++ b/.env.example @@ -30,3 +30,8 @@ INFINITY_API_KEY = "" LITELLM_MASTER_KEY = "" DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB = "True" + +# Worker heartbeat interval in seconds; 0 disables heartbeat writes and worker counts +# Set the same value on all workers sharing a database, then restart them +# Invalid values fall back to 60; other background jobs must be configured separately for idle databases +PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS = "60" diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py index 02015c699c9..6c026538d72 100644 --- a/litellm/proxy/db/proxy_worker_heartbeat.py +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -1,7 +1,7 @@ """ Live proxy worker census, one row per worker process. -Every uvicorn worker upserts its own row on a fixed heartbeat, so counting +Every uvicorn worker upserts its own row on a configurable heartbeat, so counting rows with a recent heartbeat answers "how many workers share this database?" without any coordination. The Admin UI's "no Redis" banner uses that count to hide itself for deployments that are provably a single worker, where per-worker @@ -13,13 +13,16 @@ still agree. from __future__ import annotations import socket -from typing import TYPE_CHECKING, Final +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final, Protocol from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.constants import APSCHEDULER_MISFIRE_GRACE_TIME +from litellm.litellm_core_utils.env_utils import get_env_int from litellm.proxy.db.db_span import db_span from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper @@ -58,22 +61,63 @@ class _LiveWorkerCountRow(TypedDict): _COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...]) +class HeartbeatScheduler(Protocol): + def add_job( + self, + func: Callable[[], Awaitable[None]], + trigger: str, + *, + seconds: int, + id: str, + replace_existing: bool, + misfire_grace_time: int, + ) -> object: ... + + +def _heartbeat_interval_seconds() -> int: + interval: Final = get_env_int("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS) + if interval >= 0: + return interval + verbose_proxy_logger.warning("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS must be non-negative; using 60 seconds") + return PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS + + class ProxyWorkerHeartbeat: def __init__(self, prisma_client: PrismaClient, worker_id: str | None = None) -> None: self.prisma_client: Final = prisma_client self.worker_id: Final[str] = worker_id or str(uuid.uuid4()) self.hostname: Final = socket.gethostname() + self.interval_seconds: Final = _heartbeat_interval_seconds() + + async def start(self, scheduler: HeartbeatScheduler) -> None: + if self.interval_seconds == 0: + return + await self.beat() + scheduler.add_job( + self.beat, + "interval", + seconds=self.interval_seconds, + id="proxy_worker_heartbeat_job", + replace_existing=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) async def beat(self) -> None: + if self.interval_seconds == 0: + return try: async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"): await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname) async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"): - await self.prisma_client.db.execute_raw(PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) + await self.prisma_client.db.execute_raw( + PRUNE_SQL, max(STALE_ROW_RETENTION_SECONDS, 3 * self.interval_seconds) + ) except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err) async def deregister(self) -> None: + if self.interval_seconds == 0: + return try: async with db_span("deregister_proxy_worker", "LiteLLM_ProxyWorkerHeartbeat"): await self.prisma_client.db.execute_raw(DEREGISTER_SQL, self.worker_id) @@ -87,11 +131,14 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None: cannot answer. Callers must treat None as "unknown", not as zero. Always counts on the primary: a lagging read replica must never undercount. """ + interval_seconds: Final = _heartbeat_interval_seconds() + if interval_seconds == 0: + return None try: db: Final = prisma_client.db primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"): - rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + rows: Final = await primary_db.query_raw(COUNT_SQL, 3 * interval_seconds) return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"] except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503 verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 67fec38e39e..4458ee5d83c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -556,7 +556,7 @@ from litellm.proxy.db.master_key_migration import ( migrate_if_requested, ) from litellm.proxy.db.proxy_worker_heartbeat import ( - PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS, + HeartbeatScheduler, ProxyWorkerHeartbeat, ) from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed @@ -10911,14 +10911,8 @@ class ProxyStartupEvent: ### PROXY WORKER HEARTBEAT ### worker_heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma_client) - await worker_heartbeat.beat() - scheduler.add_job( - worker_heartbeat.beat, - "interval", - seconds=PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS, - id="proxy_worker_heartbeat_job", - replace_existing=True, - misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + await worker_heartbeat.start( + cast(HeartbeatScheduler, scheduler) # cast-ok: APScheduler has no stubs for its add_job API ) ### RESET BUDGET ### diff --git a/tests/unit/proxy/db/test_proxy_worker_heartbeat.py b/tests/unit/proxy/db/test_proxy_worker_heartbeat.py index 967e4d3471f..89e922ecf25 100644 --- a/tests/unit/proxy/db/test_proxy_worker_heartbeat.py +++ b/tests/unit/proxy/db/test_proxy_worker_heartbeat.py @@ -1,7 +1,10 @@ from collections.abc import Awaitable, Callable +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.interval import IntervalTrigger from litellm.proxy.db.proxy_worker_heartbeat import ( BEAT_SQL, @@ -24,6 +27,72 @@ def _prisma(): return prisma +@pytest.mark.asyncio +async def test_disabled_heartbeat_does_not_access_the_database(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "0") + prisma: Final = _prisma() + heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="disabled-worker") + + await heartbeat.beat() + await heartbeat.deregister() + + assert await count_live_proxy_workers(prisma) is None + prisma.db.execute_raw.assert_not_awaited() + prisma.db.query_raw.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("configured_interval", "expected_interval"), + ((None, 60), ("15", 15), ("1800", 1800), ("-1", 60), ("invalid", 60), ("", 60), ("1.5", 60)), +) +async def test_heartbeat_uses_configured_liveness_and_retention( + monkeypatch: pytest.MonkeyPatch, configured_interval: str | None, expected_interval: int +) -> None: + if configured_interval is None: + monkeypatch.delenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", raising=False) + else: + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", configured_interval) + prisma: Final = _prisma() + prisma.db.query_raw.return_value = [{"live_workers": 2}] + heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="configured-worker") + + await heartbeat.beat() + + assert prisma.db.execute_raw.call_args_list[0].args == (BEAT_SQL, "configured-worker", heartbeat.hostname) + assert prisma.db.execute_raw.call_args_list[1].args == ( + PRUNE_SQL, + max(STALE_ROW_RETENTION_SECONDS, 3 * expected_interval), + ) + assert await count_live_proxy_workers(prisma) == 2 + prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, 3 * expected_interval) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("interval_seconds", (0, 15, 60, 1800)) +async def test_start_registers_only_enabled_heartbeats(monkeypatch: pytest.MonkeyPatch, interval_seconds: int) -> None: + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(interval_seconds)) + prisma: Final = _prisma() + heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="scheduled-worker") + scheduler: Final = AsyncIOScheduler() + + await heartbeat.start(scheduler) + + job: Final = scheduler.get_job("proxy_worker_heartbeat_job") + if interval_seconds == 0: + assert job is None + prisma.db.execute_raw.assert_not_awaited() + return + + assert job is not None + assert isinstance(job.trigger, IntervalTrigger) + assert job.trigger.interval.total_seconds() == interval_seconds + assert prisma.db.execute_raw.await_count == 2 + await job.func() + assert prisma.db.execute_raw.await_count == 4 + assert prisma.db.execute_raw.call_args_list[2].args == (BEAT_SQL, "scheduled-worker", heartbeat.hostname) + + @pytest.mark.asyncio async def test_beat_upserts_own_row_then_prunes_stale_rows(): prisma = _prisma()