mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
feat(proxy): make worker heartbeat interval configurable
This commit is contained in:
parent
ee47da334d
commit
3f555cfebd
4 changed files with 128 additions and 13 deletions
|
|
@ -30,3 +30,8 @@ INFINITY_API_KEY = ""
|
|||
LITELLM_MASTER_KEY = ""
|
||||
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
|
||||
STORE_MODEL_IN_DB = "True"
|
||||
|
||||
# Worker heartbeat interval in seconds; 0 disables heartbeat writes and worker counts
|
||||
# Set the same value on all workers sharing a database, then restart them
|
||||
# Invalid values fall back to 60; other background jobs must be configured separately for idle databases
|
||||
PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS = "60"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Live proxy worker census, one row per worker process.
|
||||
|
||||
Every uvicorn worker upserts its own row on a fixed heartbeat, so counting
|
||||
Every uvicorn worker upserts its own row on a configurable heartbeat, so counting
|
||||
rows with a recent heartbeat answers "how many workers share this database?"
|
||||
without any coordination. The Admin UI's "no Redis" banner uses that count to
|
||||
hide itself for deployments that are provably a single worker, where per-worker
|
||||
|
|
@ -13,13 +13,16 @@ still agree.
|
|||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import APSCHEDULER_MISFIRE_GRACE_TIME
|
||||
from litellm.litellm_core_utils.env_utils import get_env_int
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
|
||||
|
||||
|
|
@ -58,22 +61,63 @@ class _LiveWorkerCountRow(TypedDict):
|
|||
_COUNT_ROWS_ADAPTER: Final = TypeAdapter(tuple[_LiveWorkerCountRow, ...])
|
||||
|
||||
|
||||
class HeartbeatScheduler(Protocol):
|
||||
def add_job(
|
||||
self,
|
||||
func: Callable[[], Awaitable[None]],
|
||||
trigger: str,
|
||||
*,
|
||||
seconds: int,
|
||||
id: str,
|
||||
replace_existing: bool,
|
||||
misfire_grace_time: int,
|
||||
) -> object: ...
|
||||
|
||||
|
||||
def _heartbeat_interval_seconds() -> int:
|
||||
interval: Final = get_env_int("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS)
|
||||
if interval >= 0:
|
||||
return interval
|
||||
verbose_proxy_logger.warning("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS must be non-negative; using 60 seconds")
|
||||
return PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS
|
||||
|
||||
|
||||
class ProxyWorkerHeartbeat:
|
||||
def __init__(self, prisma_client: PrismaClient, worker_id: str | None = None) -> None:
|
||||
self.prisma_client: Final = prisma_client
|
||||
self.worker_id: Final[str] = worker_id or str(uuid.uuid4())
|
||||
self.hostname: Final = socket.gethostname()
|
||||
self.interval_seconds: Final = _heartbeat_interval_seconds()
|
||||
|
||||
async def start(self, scheduler: HeartbeatScheduler) -> None:
|
||||
if self.interval_seconds == 0:
|
||||
return
|
||||
await self.beat()
|
||||
scheduler.add_job(
|
||||
self.beat,
|
||||
"interval",
|
||||
seconds=self.interval_seconds,
|
||||
id="proxy_worker_heartbeat_job",
|
||||
replace_existing=True,
|
||||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
)
|
||||
|
||||
async def beat(self) -> None:
|
||||
if self.interval_seconds == 0:
|
||||
return
|
||||
try:
|
||||
async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname)
|
||||
async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
await self.prisma_client.db.execute_raw(PRUNE_SQL, STALE_ROW_RETENTION_SECONDS)
|
||||
await self.prisma_client.db.execute_raw(
|
||||
PRUNE_SQL, max(STALE_ROW_RETENTION_SECONDS, 3 * self.interval_seconds)
|
||||
)
|
||||
except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker
|
||||
verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err)
|
||||
|
||||
async def deregister(self) -> None:
|
||||
if self.interval_seconds == 0:
|
||||
return
|
||||
try:
|
||||
async with db_span("deregister_proxy_worker", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
await self.prisma_client.db.execute_raw(DEREGISTER_SQL, self.worker_id)
|
||||
|
|
@ -87,11 +131,14 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None:
|
|||
cannot answer. Callers must treat None as "unknown", not as zero. Always
|
||||
counts on the primary: a lagging read replica must never undercount.
|
||||
"""
|
||||
interval_seconds: Final = _heartbeat_interval_seconds()
|
||||
if interval_seconds == 0:
|
||||
return None
|
||||
try:
|
||||
db: Final = prisma_client.db
|
||||
primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db
|
||||
async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
|
||||
rows: Final = await primary_db.query_raw(COUNT_SQL, 3 * interval_seconds)
|
||||
return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"]
|
||||
except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503
|
||||
verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err)
|
||||
|
|
|
|||
|
|
@ -556,7 +556,7 @@ from litellm.proxy.db.master_key_migration import (
|
|||
migrate_if_requested,
|
||||
)
|
||||
from litellm.proxy.db.proxy_worker_heartbeat import (
|
||||
PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS,
|
||||
HeartbeatScheduler,
|
||||
ProxyWorkerHeartbeat,
|
||||
)
|
||||
from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed
|
||||
|
|
@ -10911,14 +10911,8 @@ class ProxyStartupEvent:
|
|||
|
||||
### PROXY WORKER HEARTBEAT ###
|
||||
worker_heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma_client)
|
||||
await worker_heartbeat.beat()
|
||||
scheduler.add_job(
|
||||
worker_heartbeat.beat,
|
||||
"interval",
|
||||
seconds=PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS,
|
||||
id="proxy_worker_heartbeat_job",
|
||||
replace_existing=True,
|
||||
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
|
||||
await worker_heartbeat.start(
|
||||
cast(HeartbeatScheduler, scheduler) # cast-ok: APScheduler has no stubs for its add_job API
|
||||
)
|
||||
|
||||
### RESET BUDGET ###
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from litellm.proxy.db.proxy_worker_heartbeat import (
|
||||
BEAT_SQL,
|
||||
|
|
@ -24,6 +27,72 @@ def _prisma():
|
|||
return prisma
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_heartbeat_does_not_access_the_database(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "0")
|
||||
prisma: Final = _prisma()
|
||||
heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="disabled-worker")
|
||||
|
||||
await heartbeat.beat()
|
||||
await heartbeat.deregister()
|
||||
|
||||
assert await count_live_proxy_workers(prisma) is None
|
||||
prisma.db.execute_raw.assert_not_awaited()
|
||||
prisma.db.query_raw.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured_interval", "expected_interval"),
|
||||
((None, 60), ("15", 15), ("1800", 1800), ("-1", 60), ("invalid", 60), ("", 60), ("1.5", 60)),
|
||||
)
|
||||
async def test_heartbeat_uses_configured_liveness_and_retention(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_interval: str | None, expected_interval: int
|
||||
) -> None:
|
||||
if configured_interval is None:
|
||||
monkeypatch.delenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", configured_interval)
|
||||
prisma: Final = _prisma()
|
||||
prisma.db.query_raw.return_value = [{"live_workers": 2}]
|
||||
heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="configured-worker")
|
||||
|
||||
await heartbeat.beat()
|
||||
|
||||
assert prisma.db.execute_raw.call_args_list[0].args == (BEAT_SQL, "configured-worker", heartbeat.hostname)
|
||||
assert prisma.db.execute_raw.call_args_list[1].args == (
|
||||
PRUNE_SQL,
|
||||
max(STALE_ROW_RETENTION_SECONDS, 3 * expected_interval),
|
||||
)
|
||||
assert await count_live_proxy_workers(prisma) == 2
|
||||
prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, 3 * expected_interval)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("interval_seconds", (0, 15, 60, 1800))
|
||||
async def test_start_registers_only_enabled_heartbeats(monkeypatch: pytest.MonkeyPatch, interval_seconds: int) -> None:
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(interval_seconds))
|
||||
prisma: Final = _prisma()
|
||||
heartbeat: Final = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="scheduled-worker")
|
||||
scheduler: Final = AsyncIOScheduler()
|
||||
|
||||
await heartbeat.start(scheduler)
|
||||
|
||||
job: Final = scheduler.get_job("proxy_worker_heartbeat_job")
|
||||
if interval_seconds == 0:
|
||||
assert job is None
|
||||
prisma.db.execute_raw.assert_not_awaited()
|
||||
return
|
||||
|
||||
assert job is not None
|
||||
assert isinstance(job.trigger, IntervalTrigger)
|
||||
assert job.trigger.interval.total_seconds() == interval_seconds
|
||||
assert prisma.db.execute_raw.await_count == 2
|
||||
await job.func()
|
||||
assert prisma.db.execute_raw.await_count == 4
|
||||
assert prisma.db.execute_raw.call_args_list[2].args == (BEAT_SQL, "scheduled-worker", heartbeat.hostname)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_beat_upserts_own_row_then_prunes_stale_rows():
|
||||
prisma = _prisma()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue