mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(proxy): preserve per-worker heartbeat expiry
This commit is contained in:
parent
c8eb7757f9
commit
b405e86c07
8 changed files with 158 additions and 21 deletions
|
|
@ -31,7 +31,4 @@ LITELLM_MASTER_KEY = ""
|
|||
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
|
||||
STORE_MODEL_IN_DB = "True"
|
||||
|
||||
# Worker heartbeat interval in seconds; 0 disables heartbeat writes and worker counts
|
||||
# Set the same value on all workers sharing a database, then restart them
|
||||
# Invalid values fall back to 60; other background jobs must be configured separately for idle databases
|
||||
PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS = "60"
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE "LiteLLM_ProxyWorkerHeartbeat" ADD COLUMN "expires_at" TIMESTAMP(3);
|
||||
|
|
@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat {
|
|||
hostname String
|
||||
started_at DateTime @default(now())
|
||||
last_heartbeat_at DateTime @default(now())
|
||||
expires_at DateTime?
|
||||
}
|
||||
|
||||
// Track the status of cron jobs running. Only allow one pod to run the job at a time
|
||||
|
|
|
|||
|
|
@ -207,13 +207,14 @@ def _window_for(*, job_id: str, period_seconds: int | None, settings: ScheduledJ
|
|||
return min(limit for limit in limits if limit is not None)
|
||||
|
||||
|
||||
def _clamped_override(*, job_id: str, requested: int) -> int:
|
||||
horizon: Final = DEFAULT_CRON_DEDUPE_SECONDS.get(job_id)
|
||||
def _clamped_override(*, job_id: str, requested: int, period_seconds: int | None) -> int:
|
||||
horizon: Final = (
|
||||
period_seconds if job_id == "proxy_worker_heartbeat_job" else DEFAULT_CRON_DEDUPE_SECONDS.get(job_id)
|
||||
)
|
||||
if horizon is None or requested < horizon:
|
||||
return requested
|
||||
verbose_proxy_logger.warning(
|
||||
"general_settings.%s.offsets[%s]=%ss would place replicas more than %ss apart, "
|
||||
"which is long enough for a second replica to redo the run; using %ss instead",
|
||||
"general_settings.%s.offsets[%s]=%ss exceeds the job's %ss scheduling window; using %ss instead",
|
||||
GENERAL_SETTINGS_KEY,
|
||||
job_id,
|
||||
requested,
|
||||
|
|
@ -233,7 +234,7 @@ def _offset_for(
|
|||
) -> int:
|
||||
override: Final = settings.offsets.get(job_id)
|
||||
if override is not None:
|
||||
return _clamped_override(job_id=job_id, requested=max(0, override))
|
||||
return _clamped_override(job_id=job_id, requested=max(0, override), period_seconds=period_seconds)
|
||||
if not staggerable:
|
||||
return 0
|
||||
return offset_seconds(
|
||||
|
|
|
|||
|
|
@ -34,19 +34,20 @@ PROXY_WORKER_LIVENESS_WINDOW_SECONDS: Final = 3 * PROXY_WORKER_HEARTBEAT_INTERVA
|
|||
STALE_ROW_RETENTION_SECONDS: Final = 3600
|
||||
|
||||
BEAT_SQL: Final = """
|
||||
INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at)
|
||||
VALUES ($1, $2, NOW())
|
||||
ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW()
|
||||
INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at, expires_at)
|
||||
VALUES ($1, $2, NOW(), NOW() + make_interval(secs => $3))
|
||||
ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW(), expires_at = EXCLUDED.expires_at
|
||||
"""
|
||||
|
||||
PRUNE_SQL: Final = """
|
||||
DELETE FROM "LiteLLM_ProxyWorkerHeartbeat"
|
||||
WHERE last_heartbeat_at < NOW() - make_interval(secs => $1)
|
||||
AND COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $2)) <= NOW()
|
||||
"""
|
||||
|
||||
COUNT_SQL: Final = """
|
||||
SELECT COUNT(*)::int AS live_workers FROM "LiteLLM_ProxyWorkerHeartbeat"
|
||||
WHERE last_heartbeat_at > NOW() - make_interval(secs => $1)
|
||||
WHERE COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $1)) > NOW()
|
||||
"""
|
||||
|
||||
DEREGISTER_SQL: Final = """
|
||||
|
|
@ -107,10 +108,12 @@ class ProxyWorkerHeartbeat:
|
|||
return
|
||||
try:
|
||||
async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname)
|
||||
await self.prisma_client.db.execute_raw(
|
||||
BEAT_SQL, self.worker_id, self.hostname, 3 * self.interval_seconds
|
||||
)
|
||||
async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
await self.prisma_client.db.execute_raw(
|
||||
PRUNE_SQL, max(STALE_ROW_RETENTION_SECONDS, 3 * self.interval_seconds)
|
||||
PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS
|
||||
)
|
||||
except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker
|
||||
verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err)
|
||||
|
|
@ -138,7 +141,7 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None:
|
|||
db: Final = prisma_client.db
|
||||
primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db
|
||||
async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"):
|
||||
rows: Final = await primary_db.query_raw(COUNT_SQL, 3 * interval_seconds)
|
||||
rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
|
||||
return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"]
|
||||
except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503
|
||||
verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err)
|
||||
|
|
|
|||
|
|
@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat {
|
|||
hostname String
|
||||
started_at DateTime @default(now())
|
||||
last_heartbeat_at DateTime @default(now())
|
||||
expires_at DateTime?
|
||||
}
|
||||
|
||||
// Track the status of cron jobs running. Only allow one pod to run the job at a time
|
||||
|
|
|
|||
97
tests/integration/database/test_proxy_worker_heartbeat.py
Normal file
97
tests/integration/database/test_proxy_worker_heartbeat.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.proxy_worker_heartbeat import ProxyWorkerHeartbeat, count_live_proxy_workers
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from tests.integration._support.database import read_rows, scratch_database, write_rows
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(loop_scope="function")
|
||||
async def heartbeat_database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[PrismaClient, str]]:
|
||||
with scratch_database() as url:
|
||||
write_rows(
|
||||
'CREATE TABLE "LiteLLM_ProxyWorkerHeartbeat" (worker_id TEXT PRIMARY KEY, hostname TEXT NOT NULL, '
|
||||
"started_at TIMESTAMP NOT NULL DEFAULT NOW(), last_heartbeat_at TIMESTAMP NOT NULL DEFAULT NOW())",
|
||||
(),
|
||||
database_url=url,
|
||||
)
|
||||
monkeypatch.setenv("DATABASE_URL", url)
|
||||
monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False)
|
||||
client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache()))
|
||||
await client.connect()
|
||||
try:
|
||||
migration: Final = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql"
|
||||
)
|
||||
await client.db.execute_raw(migration.read_text())
|
||||
yield client, url
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("slow_interval", "elapsed"), ((1800, 600), (7200, 3700)))
|
||||
async def test_fast_worker_neither_hides_nor_prunes_a_live_slow_worker(
|
||||
heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch, slow_interval: int, elapsed: int
|
||||
) -> None:
|
||||
client, url = heartbeat_database
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval))
|
||||
slow: Final = ProxyWorkerHeartbeat(client, worker_id="slow")
|
||||
await slow.beat()
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_ProxyWorkerHeartbeat" '
|
||||
"SET last_heartbeat_at = last_heartbeat_at - make_interval(secs => %s::int), "
|
||||
"expires_at = expires_at - make_interval(secs => %s::int)",
|
||||
(str(elapsed), str(elapsed)),
|
||||
database_url=url,
|
||||
)
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "60")
|
||||
await ProxyWorkerHeartbeat(client, worker_id="fast").beat()
|
||||
|
||||
assert await count_live_proxy_workers(client) == 2
|
||||
assert read_rows(
|
||||
'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url
|
||||
) == [{"worker_id": "fast"}, {"worker_id": "slow"}]
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval))
|
||||
assert await count_live_proxy_workers(client) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_workers_are_excluded_and_legacy_rows_keep_the_default_window(
|
||||
heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
client, url = heartbeat_database
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15")
|
||||
await ProxyWorkerHeartbeat(client, worker_id="expired").beat()
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_ProxyWorkerHeartbeat" '
|
||||
"SET last_heartbeat_at = NOW() - INTERVAL '50 seconds', expires_at = NOW() - INTERVAL '5 seconds'",
|
||||
(),
|
||||
database_url=url,
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at) '
|
||||
"VALUES ('legacy-live', 'old-worker', NOW() - INTERVAL '120 seconds'), "
|
||||
"('legacy-expired', 'old-worker', NOW() - INTERVAL '181 seconds'), "
|
||||
"('legacy-stale', 'old-worker', NOW() - INTERVAL '2 hours')",
|
||||
(),
|
||||
database_url=url,
|
||||
)
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "1800")
|
||||
await ProxyWorkerHeartbeat(client, worker_id="current").beat()
|
||||
|
||||
assert await count_live_proxy_workers(client) == 2
|
||||
assert read_rows(
|
||||
'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url
|
||||
) == [
|
||||
{"worker_id": "current"},
|
||||
{"worker_id": "expired"},
|
||||
{"worker_id": "legacy-expired"},
|
||||
{"worker_id": "legacy-live"},
|
||||
]
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from datetime import timedelta
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -6,6 +7,9 @@ import pytest
|
|||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from litellm.proxy._types import ScheduledJobStaggerSettings
|
||||
from litellm.proxy.common_utils.scheduled_job_stagger import apply_scheduled_job_stagger
|
||||
|
||||
from litellm.proxy.db.proxy_worker_heartbeat import (
|
||||
BEAT_SQL,
|
||||
COUNT_SQL,
|
||||
|
|
@ -20,6 +24,27 @@ from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
|
|||
from tests.unit.proxy.db.fake_prisma_engine import engine_call
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("offset", (0, 14, 15, 120))
|
||||
async def test_startup_heartbeat_survives_an_explicit_stagger_offset(
|
||||
monkeypatch: pytest.MonkeyPatch, offset: int
|
||||
) -> None:
|
||||
monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15")
|
||||
scheduler: Final = AsyncIOScheduler()
|
||||
await ProxyWorkerHeartbeat(_prisma(), worker_id="staggered").start(scheduler)
|
||||
job: Final = scheduler.get_job("proxy_worker_heartbeat_job")
|
||||
assert job is not None
|
||||
started_at: Final = job.trigger.start_date - timedelta(seconds=15)
|
||||
apply_scheduled_job_stagger(
|
||||
scheduler=scheduler,
|
||||
settings=ScheduledJobStaggerSettings(offsets={"proxy_worker_heartbeat_job": offset}),
|
||||
identity="test-worker",
|
||||
)
|
||||
next_fire: Final = job.trigger.get_next_fire_time(None, started_at)
|
||||
assert next_fire is not None
|
||||
assert next_fire - started_at < timedelta(seconds=30)
|
||||
|
||||
|
||||
def _prisma():
|
||||
prisma = MagicMock()
|
||||
prisma.db.execute_raw = AsyncMock()
|
||||
|
|
@ -59,13 +84,19 @@ async def test_heartbeat_uses_configured_liveness_and_retention(
|
|||
|
||||
await heartbeat.beat()
|
||||
|
||||
assert prisma.db.execute_raw.call_args_list[0].args == (BEAT_SQL, "configured-worker", heartbeat.hostname)
|
||||
assert prisma.db.execute_raw.call_args_list[0].args == (
|
||||
BEAT_SQL,
|
||||
"configured-worker",
|
||||
heartbeat.hostname,
|
||||
3 * expected_interval,
|
||||
)
|
||||
assert prisma.db.execute_raw.call_args_list[1].args == (
|
||||
PRUNE_SQL,
|
||||
max(STALE_ROW_RETENTION_SECONDS, 3 * expected_interval),
|
||||
STALE_ROW_RETENTION_SECONDS,
|
||||
PROXY_WORKER_LIVENESS_WINDOW_SECONDS,
|
||||
)
|
||||
assert await count_live_proxy_workers(prisma) == 2
|
||||
prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, 3 * expected_interval)
|
||||
prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -90,7 +121,12 @@ async def test_start_registers_only_enabled_heartbeats(monkeypatch: pytest.Monke
|
|||
assert prisma.db.execute_raw.await_count == 2
|
||||
await job.func()
|
||||
assert prisma.db.execute_raw.await_count == 4
|
||||
assert prisma.db.execute_raw.call_args_list[2].args == (BEAT_SQL, "scheduled-worker", heartbeat.hostname)
|
||||
assert prisma.db.execute_raw.call_args_list[2].args == (
|
||||
BEAT_SQL,
|
||||
"scheduled-worker",
|
||||
heartbeat.hostname,
|
||||
3 * interval_seconds,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -99,8 +135,8 @@ async def test_beat_upserts_own_row_then_prunes_stale_rows():
|
|||
heartbeat = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1")
|
||||
await heartbeat.beat()
|
||||
calls = prisma.db.execute_raw.call_args_list
|
||||
assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname)
|
||||
assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS)
|
||||
assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
|
||||
assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue