From b405e86c0728a2570f17a37a39f7a8fb31377643 Mon Sep 17 00:00:00 2001 From: haiyangbg Date: Fri, 9 Oct 2026 19:18:32 +0800 Subject: [PATCH] fix(proxy): preserve per-worker heartbeat expiry --- .env.example | 3 - .../migration.sql | 1 + .../litellm_proxy_extras/schema.prisma | 1 + .../common_utils/scheduled_job_stagger.py | 11 ++- litellm/proxy/db/proxy_worker_heartbeat.py | 17 ++-- litellm/proxy/schema.prisma | 1 + .../database/test_proxy_worker_heartbeat.py | 97 +++++++++++++++++++ .../proxy/db/test_proxy_worker_heartbeat.py | 48 +++++++-- 8 files changed, 158 insertions(+), 21 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql create mode 100644 tests/integration/database/test_proxy_worker_heartbeat.py diff --git a/.env.example b/.env.example index 23de0084e9e..eb659202d25 100644 --- a/.env.example +++ b/.env.example @@ -31,7 +31,4 @@ LITELLM_MASTER_KEY = "" DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm" STORE_MODEL_IN_DB = "True" -# Worker heartbeat interval in seconds; 0 disables heartbeat writes and worker counts -# Set the same value on all workers sharing a database, then restart them -# Invalid values fall back to 60; other background jobs must be configured separately for idle databases PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS = "60" diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql new file mode 100644 index 00000000000..39df8e03409 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "LiteLLM_ProxyWorkerHeartbeat" ADD COLUMN "expires_at" TIMESTAMP(3); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 59ffb037177..8442b1ee6cc 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat { hostname String started_at DateTime @default(now()) last_heartbeat_at DateTime @default(now()) + expires_at DateTime? } // Track the status of cron jobs running. Only allow one pod to run the job at a time diff --git a/litellm/proxy/common_utils/scheduled_job_stagger.py b/litellm/proxy/common_utils/scheduled_job_stagger.py index e48e9686f13..91c613cb215 100644 --- a/litellm/proxy/common_utils/scheduled_job_stagger.py +++ b/litellm/proxy/common_utils/scheduled_job_stagger.py @@ -207,13 +207,14 @@ def _window_for(*, job_id: str, period_seconds: int | None, settings: ScheduledJ return min(limit for limit in limits if limit is not None) -def _clamped_override(*, job_id: str, requested: int) -> int: - horizon: Final = DEFAULT_CRON_DEDUPE_SECONDS.get(job_id) +def _clamped_override(*, job_id: str, requested: int, period_seconds: int | None) -> int: + horizon: Final = ( + period_seconds if job_id == "proxy_worker_heartbeat_job" else DEFAULT_CRON_DEDUPE_SECONDS.get(job_id) + ) if horizon is None or requested < horizon: return requested verbose_proxy_logger.warning( - "general_settings.%s.offsets[%s]=%ss would place replicas more than %ss apart, " - "which is long enough for a second replica to redo the run; using %ss instead", + "general_settings.%s.offsets[%s]=%ss exceeds the job's %ss scheduling window; using %ss instead", GENERAL_SETTINGS_KEY, job_id, requested, @@ -233,7 +234,7 @@ def _offset_for( ) -> int: override: Final = settings.offsets.get(job_id) if override is not None: - return _clamped_override(job_id=job_id, requested=max(0, override)) + return _clamped_override(job_id=job_id, requested=max(0, override), period_seconds=period_seconds) if not staggerable: return 0 return offset_seconds( diff --git a/litellm/proxy/db/proxy_worker_heartbeat.py b/litellm/proxy/db/proxy_worker_heartbeat.py index 6c026538d72..3e525efcc43 100644 --- a/litellm/proxy/db/proxy_worker_heartbeat.py +++ b/litellm/proxy/db/proxy_worker_heartbeat.py @@ -34,19 +34,20 @@ PROXY_WORKER_LIVENESS_WINDOW_SECONDS: Final = 3 * PROXY_WORKER_HEARTBEAT_INTERVA STALE_ROW_RETENTION_SECONDS: Final = 3600 BEAT_SQL: Final = """ -INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at) -VALUES ($1, $2, NOW()) -ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW() +INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at, expires_at) +VALUES ($1, $2, NOW(), NOW() + make_interval(secs => $3)) +ON CONFLICT (worker_id) DO UPDATE SET last_heartbeat_at = NOW(), expires_at = EXCLUDED.expires_at """ PRUNE_SQL: Final = """ DELETE FROM "LiteLLM_ProxyWorkerHeartbeat" WHERE last_heartbeat_at < NOW() - make_interval(secs => $1) +AND COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $2)) <= NOW() """ COUNT_SQL: Final = """ SELECT COUNT(*)::int AS live_workers FROM "LiteLLM_ProxyWorkerHeartbeat" -WHERE last_heartbeat_at > NOW() - make_interval(secs => $1) +WHERE COALESCE(expires_at, last_heartbeat_at + make_interval(secs => $1)) > NOW() """ DEREGISTER_SQL: Final = """ @@ -107,10 +108,12 @@ class ProxyWorkerHeartbeat: return try: async with db_span("proxy_worker_heartbeat", "LiteLLM_ProxyWorkerHeartbeat"): - await self.prisma_client.db.execute_raw(BEAT_SQL, self.worker_id, self.hostname) + await self.prisma_client.db.execute_raw( + BEAT_SQL, self.worker_id, self.hostname, 3 * self.interval_seconds + ) async with db_span("prune_proxy_worker_heartbeats", "LiteLLM_ProxyWorkerHeartbeat"): await self.prisma_client.db.execute_raw( - PRUNE_SQL, max(STALE_ROW_RETENTION_SECONDS, 3 * self.interval_seconds) + PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS ) except Exception as beat_err: # noqa: BLE001 # a missed heartbeat must never take down the worker verbose_proxy_logger.debug("Proxy worker heartbeat write failed: %s", beat_err) @@ -138,7 +141,7 @@ async def count_live_proxy_workers(prisma_client: PrismaClient) -> int | None: db: Final = prisma_client.db primary_db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) else db async with db_span("count_live_proxy_workers", "LiteLLM_ProxyWorkerHeartbeat"): - rows: Final = await primary_db.query_raw(COUNT_SQL, 3 * interval_seconds) + rows: Final = await primary_db.query_raw(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) return _COUNT_ROWS_ADAPTER.validate_python(rows)[0]["live_workers"] except Exception as count_err: # noqa: BLE001 # an unknown count must degrade to "warn", never to a 503 verbose_proxy_logger.debug("Live proxy worker count unavailable: %s", count_err) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 59ffb037177..8442b1ee6cc 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1116,6 +1116,7 @@ model LiteLLM_ProxyWorkerHeartbeat { hostname String started_at DateTime @default(now()) last_heartbeat_at DateTime @default(now()) + expires_at DateTime? } // Track the status of cron jobs running. Only allow one pod to run the job at a time diff --git a/tests/integration/database/test_proxy_worker_heartbeat.py b/tests/integration/database/test_proxy_worker_heartbeat.py new file mode 100644 index 00000000000..a28390a5bf0 --- /dev/null +++ b/tests/integration/database/test_proxy_worker_heartbeat.py @@ -0,0 +1,97 @@ +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Final + +import pytest +import pytest_asyncio + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db.proxy_worker_heartbeat import ProxyWorkerHeartbeat, count_live_proxy_workers +from litellm.proxy.utils import PrismaClient, ProxyLogging +from tests.integration._support.database import read_rows, scratch_database, write_rows + + +@pytest_asyncio.fixture(loop_scope="function") +async def heartbeat_database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[tuple[PrismaClient, str]]: + with scratch_database() as url: + write_rows( + 'CREATE TABLE "LiteLLM_ProxyWorkerHeartbeat" (worker_id TEXT PRIMARY KEY, hostname TEXT NOT NULL, ' + "started_at TIMESTAMP NOT NULL DEFAULT NOW(), last_heartbeat_at TIMESTAMP NOT NULL DEFAULT NOW())", + (), + database_url=url, + ) + monkeypatch.setenv("DATABASE_URL", url) + monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) + client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + migration: Final = ( + Path(__file__).resolve().parents[3] + / "litellm-proxy-extras/litellm_proxy_extras/migrations/20261009120000_proxy_worker_expiry/migration.sql" + ) + await client.db.execute_raw(migration.read_text()) + yield client, url + finally: + await client.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("slow_interval", "elapsed"), ((1800, 600), (7200, 3700))) +async def test_fast_worker_neither_hides_nor_prunes_a_live_slow_worker( + heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch, slow_interval: int, elapsed: int +) -> None: + client, url = heartbeat_database + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval)) + slow: Final = ProxyWorkerHeartbeat(client, worker_id="slow") + await slow.beat() + write_rows( + 'UPDATE "LiteLLM_ProxyWorkerHeartbeat" ' + "SET last_heartbeat_at = last_heartbeat_at - make_interval(secs => %s::int), " + "expires_at = expires_at - make_interval(secs => %s::int)", + (str(elapsed), str(elapsed)), + database_url=url, + ) + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "60") + await ProxyWorkerHeartbeat(client, worker_id="fast").beat() + + assert await count_live_proxy_workers(client) == 2 + assert read_rows( + 'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url + ) == [{"worker_id": "fast"}, {"worker_id": "slow"}] + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", str(slow_interval)) + assert await count_live_proxy_workers(client) == 2 + + +@pytest.mark.asyncio +async def test_expired_workers_are_excluded_and_legacy_rows_keep_the_default_window( + heartbeat_database: tuple[PrismaClient, str], monkeypatch: pytest.MonkeyPatch +) -> None: + client, url = heartbeat_database + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15") + await ProxyWorkerHeartbeat(client, worker_id="expired").beat() + write_rows( + 'UPDATE "LiteLLM_ProxyWorkerHeartbeat" ' + "SET last_heartbeat_at = NOW() - INTERVAL '50 seconds', expires_at = NOW() - INTERVAL '5 seconds'", + (), + database_url=url, + ) + write_rows( + 'INSERT INTO "LiteLLM_ProxyWorkerHeartbeat" (worker_id, hostname, last_heartbeat_at) ' + "VALUES ('legacy-live', 'old-worker', NOW() - INTERVAL '120 seconds'), " + "('legacy-expired', 'old-worker', NOW() - INTERVAL '181 seconds'), " + "('legacy-stale', 'old-worker', NOW() - INTERVAL '2 hours')", + (), + database_url=url, + ) + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "1800") + await ProxyWorkerHeartbeat(client, worker_id="current").beat() + + assert await count_live_proxy_workers(client) == 2 + assert read_rows( + 'SELECT worker_id FROM "LiteLLM_ProxyWorkerHeartbeat" ORDER BY worker_id', (), database_url=url + ) == [ + {"worker_id": "current"}, + {"worker_id": "expired"}, + {"worker_id": "legacy-expired"}, + {"worker_id": "legacy-live"}, + ] diff --git a/tests/unit/proxy/db/test_proxy_worker_heartbeat.py b/tests/unit/proxy/db/test_proxy_worker_heartbeat.py index 89e922ecf25..87952f0d608 100644 --- a/tests/unit/proxy/db/test_proxy_worker_heartbeat.py +++ b/tests/unit/proxy/db/test_proxy_worker_heartbeat.py @@ -1,4 +1,5 @@ from collections.abc import Awaitable, Callable +from datetime import timedelta from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -6,6 +7,9 @@ import pytest from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.interval import IntervalTrigger +from litellm.proxy._types import ScheduledJobStaggerSettings +from litellm.proxy.common_utils.scheduled_job_stagger import apply_scheduled_job_stagger + from litellm.proxy.db.proxy_worker_heartbeat import ( BEAT_SQL, COUNT_SQL, @@ -20,6 +24,27 @@ from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper from tests.unit.proxy.db.fake_prisma_engine import engine_call +@pytest.mark.asyncio +@pytest.mark.parametrize("offset", (0, 14, 15, 120)) +async def test_startup_heartbeat_survives_an_explicit_stagger_offset( + monkeypatch: pytest.MonkeyPatch, offset: int +) -> None: + monkeypatch.setenv("PROXY_WORKER_HEARTBEAT_INTERVAL_SECONDS", "15") + scheduler: Final = AsyncIOScheduler() + await ProxyWorkerHeartbeat(_prisma(), worker_id="staggered").start(scheduler) + job: Final = scheduler.get_job("proxy_worker_heartbeat_job") + assert job is not None + started_at: Final = job.trigger.start_date - timedelta(seconds=15) + apply_scheduled_job_stagger( + scheduler=scheduler, + settings=ScheduledJobStaggerSettings(offsets={"proxy_worker_heartbeat_job": offset}), + identity="test-worker", + ) + next_fire: Final = job.trigger.get_next_fire_time(None, started_at) + assert next_fire is not None + assert next_fire - started_at < timedelta(seconds=30) + + def _prisma(): prisma = MagicMock() prisma.db.execute_raw = AsyncMock() @@ -59,13 +84,19 @@ async def test_heartbeat_uses_configured_liveness_and_retention( await heartbeat.beat() - assert prisma.db.execute_raw.call_args_list[0].args == (BEAT_SQL, "configured-worker", heartbeat.hostname) + assert prisma.db.execute_raw.call_args_list[0].args == ( + BEAT_SQL, + "configured-worker", + heartbeat.hostname, + 3 * expected_interval, + ) assert prisma.db.execute_raw.call_args_list[1].args == ( PRUNE_SQL, - max(STALE_ROW_RETENTION_SECONDS, 3 * expected_interval), + STALE_ROW_RETENTION_SECONDS, + PROXY_WORKER_LIVENESS_WINDOW_SECONDS, ) assert await count_live_proxy_workers(prisma) == 2 - prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, 3 * expected_interval) + prisma.db.query_raw.assert_awaited_once_with(COUNT_SQL, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) @pytest.mark.asyncio @@ -90,7 +121,12 @@ async def test_start_registers_only_enabled_heartbeats(monkeypatch: pytest.Monke assert prisma.db.execute_raw.await_count == 2 await job.func() assert prisma.db.execute_raw.await_count == 4 - assert prisma.db.execute_raw.call_args_list[2].args == (BEAT_SQL, "scheduled-worker", heartbeat.hostname) + assert prisma.db.execute_raw.call_args_list[2].args == ( + BEAT_SQL, + "scheduled-worker", + heartbeat.hostname, + 3 * interval_seconds, + ) @pytest.mark.asyncio @@ -99,8 +135,8 @@ async def test_beat_upserts_own_row_then_prunes_stale_rows(): heartbeat = ProxyWorkerHeartbeat(prisma_client=prisma, worker_id="worker-1") await heartbeat.beat() calls = prisma.db.execute_raw.call_args_list - assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname) - assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS) + assert calls[0].args == (BEAT_SQL, "worker-1", heartbeat.hostname, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) + assert calls[1].args == (PRUNE_SQL, STALE_ROW_RETENTION_SECONDS, PROXY_WORKER_LIVENESS_WINDOW_SECONDS) @pytest.mark.asyncio