fix(proxy): preserve per-worker heartbeat expiry

This commit is contained in:
haiyangbg 2026-10-09 19:18:32 +08:00
parent c8eb7757f9
commit b405e86c07
8 changed files with 158 additions and 21 deletions

View file

@ -31,7 +31,4 @@ LITELLM_MASTER_KEY = ""
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
STORE_MODEL_IN_DB = "True"
# Worker heartbeat interval in seconds; 0 disables heartbeat writes and worker counts
# Set the same value on all workers sharing a database, then restart them
# Invalid values fall back to 60; other background jobs must be configured separately for idle databases
PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS = "60"

View file

@ -0,0 +1 @@
ALTER TABLE "LiteLLM_ProxyWorkerHeartbeat" ADD COLUMN "expires_at" TIMESTAMP(3);

View file

@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat {
hostname String
started_at DateTime @default(now())
last_heartbeat_at DateTime @default(now())
expires_at DateTime?
}
// Track the status of cron jobs running. Only allow one pod to run the job at a time

View file

@ -207,13 +207,14 @@ def _window_for(*, job_id: str, period_seconds: int | None, settings: ScheduledJ
return min(limit for limit in limits if limit is not None)
def _clamped_override(*, job_id: str, requested: int) -> int:
horizon: Final = DEFAULT_CRON_DEDUPE_SECONDS.get(job_id)
def _clamped_override(*, job_id: str, requested: int, period_seconds: int | None) -> int:
horizon: Final = (
period_seconds if job_id == "proxy_worker_heartbeat_job" else DEFAULT_CRON_DEDUPE_SECONDS.get(job_id)
)
if horizon is None or requested < horizon:
return requested
verbose_proxy_logger.warning(
"general_settings.%s.offsets[%s]=%ss would place replicas more than %ss apart, "
"which is long enough for a second replica to redo the run; using %ss instead",
"general_settings.%s.offsets[%s]=%ss exceeds the job's %ss scheduling window; using %ss instead",
GENERAL_SETTINGS_KEY,
job_id,
requested,
@ -233,7 +234,7 @@ def _offset_for(
) -> int:
override: Final = settings.offsets.get(job_id)
if override is not None:
return _clamped_override(job_id=job_id, requested=max(0, override))
return _clamped_override(job_id=job_id, requested=max(0, override), period_seconds=period_seconds)
if not staggerable:
return 0
return offset_seconds(

View file

@ -34,19 +34,20 @@ PROXY_WORKER_LIVENESS_WINDOW_SECONDS: Final = 3 * PROXY_WORKER_HEARTBEAT_INTERVA
STALE_ROW_RETENTION_SECONDS: Final = 3600
BEAT_SQL: Final = """
INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at)
VALUES ($1, $2, NOW())
ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW()
INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at, expires_at)
VALUES ($1, $2, NOW(), NOW() + make_interval(secs => $3))
ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW(), expires_at = EXCLUDED.expires_at
"""
PRUNE_SQL: Final = """
DELETE FROM "LiteLLM_ProxyWorkerHeartbeat"
WHERE last_heartbeat_at < NOW() - make_interval(secs => $1)
AND COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $2)) <= NOW()
"""
COUNT_SQL: Final = """
SELECT COUNT(*)::int AS live_workers FROM "LiteLLM_ProxyWorkerHeartbeat"
WHERE last_heartbeat_at > NOW() - make_interval(secs => $1)
WHERE COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $1)) > NOW()
"""
DEREGISTER_SQL: Final = """
@ -107,10 +108,12 @@ class ProxyWorkerHeartbeat:
return
try:
async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"):
await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname)
await self.prisma_client.db.execute_raw(
BEAT_SQL, self.worker_id, self.hostname, 3 * self.interval_seconds
)
async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"):
await self.prisma_client.db.execute_raw(
PRUNE_SQL, max(STALE_ROW_RETENTION_SECONDS, 3 * self.interval_seconds)
PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS
)
except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker
verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err)
@ -138,7 +141,7 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None:
db: Final = prisma_client.db
primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db
async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"):
rows: Final = await primary_db.query_raw(COUNT_SQL, 3 * interval_seconds)
rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"]
except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503
verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err)

View file

@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat {
hostname String
started_at DateTime @default(now())
last_heartbeat_at DateTime @default(now())
expires_at DateTime?
}
// Track the status of cron jobs running. Only allow one pod to run the job at a time

View file

@ -0,0 +1,97 @@
from collections.abc import AsyncIterator
from pathlib import Path
from typing import Final
import pytest
import pytest_asyncio
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.proxy_worker_heartbeat import ProxyWorkerHeartbeat, count_live_proxy_workers
from litellm.proxy.utils import PrismaClient, ProxyLogging
from tests.integration._support.database import read_rows, scratch_database, write_rows
@pytest_asyncio.fixture(loop_scope="function")
async def heartbeat_database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[PrismaClient, str]]:
with scratch_database() as url:
write_rows(
'CREATE TABLE "LiteLLM_ProxyWorkerHeartbeat" (worker_id TEXT PRIMARY KEY, hostname TEXT NOT NULL, '
"started_at TIMESTAMP NOT NULL DEFAULT NOW(), last_heartbeat_at TIMESTAMP NOT NULL DEFAULT NOW())",
(),
database_url=url,
)
monkeypatch.setenv("DATABASE_URL", url)
monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False)
client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache()))
await client.connect()
try:
migration: Final = (
Path(__file__).resolve().parents[3]
/ "litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql"
)
await client.db.execute_raw(migration.read_text())
yield client, url
finally:
await client.disconnect()
@pytest.mark.asyncio
@pytest.mark.parametrize(("slow_interval", "elapsed"), ((1800, 600), (7200, 3700)))
async def test_fast_worker_neither_hides_nor_prunes_a_live_slow_worker(
heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch, slow_interval: int, elapsed: int
) -> None:
client, url = heartbeat_database
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval))
slow: Final = ProxyWorkerHeartbeat(client, worker_id="slow")
await slow.beat()
write_rows(
'UPDATE "LiteLLM_ProxyWorkerHeartbeat" '
"SET last_heartbeat_at = last_heartbeat_at - make_interval(secs => %s::int), "
"expires_at = expires_at - make_interval(secs => %s::int)",
(str(elapsed), str(elapsed)),
database_url=url,
)
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "60")
await ProxyWorkerHeartbeat(client, worker_id="fast").beat()
assert await count_live_proxy_workers(client) == 2
assert read_rows(
'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url
) == [{"worker_id": "fast"}, {"worker_id": "slow"}]
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval))
assert await count_live_proxy_workers(client) == 2
@pytest.mark.asyncio
async def test_expired_workers_are_excluded_and_legacy_rows_keep_the_default_window(
heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch
) -> None:
client, url = heartbeat_database
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15")
await ProxyWorkerHeartbeat(client, worker_id="expired").beat()
write_rows(
'UPDATE "LiteLLM_ProxyWorkerHeartbeat" '
"SET last_heartbeat_at = NOW() - INTERVAL '50 seconds', expires_at = NOW() - INTERVAL '5 seconds'",
(),
database_url=url,
)
write_rows(
'INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at) '
"VALUES ('legacy-live', 'old-worker', NOW() - INTERVAL '120 seconds'), "
"('legacy-expired', 'old-worker', NOW() - INTERVAL '181 seconds'), "
"('legacy-stale', 'old-worker', NOW() - INTERVAL '2 hours')",
(),
database_url=url,
)
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "1800")
await ProxyWorkerHeartbeat(client, worker_id="current").beat()
assert await count_live_proxy_workers(client) == 2
assert read_rows(
'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url
) == [
{"worker_id": "current"},
{"worker_id": "expired"},
{"worker_id": "legacy-expired"},
{"worker_id": "legacy-live"},
]

View file

@ -1,4 +1,5 @@
from collections.abc import Awaitable, Callable
from datetime import timedelta
from typing import Final
from unittest.mock import AsyncMock, MagicMock
@ -6,6 +7,9 @@ import pytest
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.interval import IntervalTrigger
from litellm.proxy._types import ScheduledJobStaggerSettings
from litellm.proxy.common_utils.scheduled_job_stagger import apply_scheduled_job_stagger
from litellm.proxy.db.proxy_worker_heartbeat import (
BEAT_SQL,
COUNT_SQL,
@ -20,6 +24,27 @@ from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@pytest.mark.asyncio
@pytest.mark.parametrize("offset", (0, 14, 15, 120))
async def test_startup_heartbeat_survives_an_explicit_stagger_offset(
monkeypatch: pytest.MonkeyPatch, offset: int
) -> None:
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15")
scheduler: Final = AsyncIOScheduler()
await ProxyWorkerHeartbeat(_prisma(), worker_id="staggered").start(scheduler)
job: Final = scheduler.get_job("proxy_worker_heartbeat_job")
assert job is not None
started_at: Final = job.trigger.start_date - timedelta(seconds=15)
apply_scheduled_job_stagger(
scheduler=scheduler,
settings=ScheduledJobStaggerSettings(offsets={"proxy_worker_heartbeat_job": offset}),
identity="test-worker",
)
next_fire: Final = job.trigger.get_next_fire_time(None, started_at)
assert next_fire is not None
assert next_fire - started_at < timedelta(seconds=30)
def _prisma():
prisma = MagicMock()
prisma.db.execute_raw = AsyncMock()
@ -59,13 +84,19 @@ async def test_heartbeat_uses_configured_liveness_and_retention(
await heartbeat.beat()
assert prisma.db.execute_raw.call_args_list[0].args == (BEAT_SQL, "configured-worker", heartbeat.hostname)
assert prisma.db.execute_raw.call_args_list[0].args == (
BEAT_SQL,
"configured-worker",
heartbeat.hostname,
3 * expected_interval,
)
assert prisma.db.execute_raw.call_args_list[1].args == (
PRUNE_SQL,
max(STALE_ROW_RETENTION_SECONDS, 3 * expected_interval),
STALE_ROW_RETENTION_SECONDS,
PROXY_WORKER_LIVENESS_WINDOW_SECONDS,
)
assert await count_live_proxy_workers(prisma) == 2
prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, 3 * expected_interval)
prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
@pytest.mark.asyncio
@ -90,7 +121,12 @@ async def test_start_registers_only_enabled_heartbeats(monkeypatch: pytest.Monke
assert prisma.db.execute_raw.await_count == 2
await job.func()
assert prisma.db.execute_raw.await_count == 4
assert prisma.db.execute_raw.call_args_list[2].args == (BEAT_SQL, "scheduled-worker", heartbeat.hostname)
assert prisma.db.execute_raw.call_args_list[2].args == (
BEAT_SQL,
"scheduled-worker",
heartbeat.hostname,
3 * interval_seconds,
)
@pytest.mark.asyncio
@ -99,8 +135,8 @@ async def test_beat_upserts_own_row_then_prunes_stale_rows():
heartbeat = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1")
await heartbeat.beat()
calls = prisma.db.execute_raw.call_args_list
assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname)
assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS)
assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
@pytest.mark.asyncio