From e7a004c78be56655011513ef7c7c74bef113d485 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:01:40 +0200 Subject: [PATCH] fix: expire task registry entries left behind by a killed worker Running tasks are recorded in Redis so that any worker can list or stop them, but the cleanup that removes a record only ever runs in the worker that owns the task. A worker killed by an OOM, a crash or a rolling restart therefore leaves its records behind permanently. The chat they belong to keeps reporting a generation in progress, the stop control acts on a task nobody is running any more, and the registry grows for the life of the deployment with no way to clear it short of deleting the keys by hand. Records now carry a TTL, and the worker that owns a task re-stamps its own records on an interval, so the only ones that age out belong to a worker that is gone. The per-item set becomes a sorted set, because it needs a per-entry timestamp that a plain set cannot hold. Scores and expiry read the Redis clock, so workers whose own clocks drift cannot expire each other's live tasks. `REDIS_TASK_TTL` defaults to 300 seconds, and 0 keeps the previous behaviour with no timestamps written at all. The sorted set lives under a new key name so that a rolling deploy never has old and new workers writing two different shapes into one key. A compatibility shim that read both shapes was tried first and dropped: it left the old key to leak forever, which is the bug this fixes. --- backend/open_webui/env.py | 9 +++++ backend/open_webui/main.py | 7 ++++ backend/open_webui/tasks.py | 78 ++++++++++++++++++++++++++++++------- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index b17df7340a..36d2cb9fbb 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -381,6 +381,15 @@ try: except ValueError: REDIS_RESPONSE_STREAM_TTL = 3600 +# Seconds a task entry survives without a heartbeat. 0 disables. +REDIS_TASK_TTL = os.getenv('REDIS_TASK_TTL', '300') +try: + REDIS_TASK_TTL = int(REDIS_TASK_TTL) + if REDIS_TASK_TTL != 0 and REDIS_TASK_TTL < 60: + REDIS_TASK_TTL = 300 +except ValueError: + REDIS_TASK_TTL = 300 + REDIS_SENTINEL_HOSTS = os.getenv('REDIS_SENTINEL_HOSTS', '') REDIS_SENTINEL_PORT = os.getenv('REDIS_SENTINEL_PORT', '26379') diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c84a394b87..68d2e777a7 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -106,6 +106,7 @@ from open_webui.env import ( MAX_BODY_LOG_SIZE, # Redis REDIS_KEY_PREFIX, + REDIS_TASK_TTL, REDIS_URL, RESET_CONFIG_ON_START, SAFE_MODE, @@ -199,6 +200,7 @@ from open_webui.tasks import ( list_task_ids_by_item_id, list_tasks, redis_task_command_listener, + redis_task_heartbeat_loop, stop_item_tasks, stop_task, ) # Import from tasks.py @@ -385,6 +387,8 @@ async def lifespan(app: FastAPI): if app.state.redis is not None: app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app)) + if REDIS_TASK_TTL > 0: + app.state.redis_task_heartbeat_loop = asyncio.create_task(redis_task_heartbeat_loop(app)) app.state.periodic_usage_pool_cleanup = asyncio.create_task(periodic_usage_pool_cleanup()) app.state.periodic_session_pool_cleanup = asyncio.create_task(periodic_session_pool_cleanup()) @@ -472,6 +476,9 @@ async def lifespan(app: FastAPI): if hasattr(app.state, 'redis_task_command_listener'): app.state.redis_task_command_listener.cancel() + if hasattr(app.state, 'redis_task_heartbeat_loop'): + app.state.redis_task_heartbeat_loop.cancel() + app.state.periodic_usage_pool_cleanup.cancel() app.state.periodic_session_pool_cleanup.cancel() app.state.scheduler_worker_loop.cancel() diff --git a/backend/open_webui/tasks.py b/backend/open_webui/tasks.py index ef15c57a18..e54700c2e0 100644 --- a/backend/open_webui/tasks.py +++ b/backend/open_webui/tasks.py @@ -6,7 +6,7 @@ from uuid import uuid4 from redis.asyncio import Redis -from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL +from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL from open_webui.utils.json_codec import JSONCodec, dumps_bytes log = logging.getLogger(__name__) @@ -18,11 +18,12 @@ response_streams: dict[str, dict] = {} REDIS_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks' -REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item' +REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item_active' REDIS_RESPONSE_STREAMS_KEY = f'{REDIS_KEY_PREFIX}:tasks:response_streams' REDIS_PUBSUB_CHANNEL = f'{REDIS_KEY_PREFIX}:tasks:commands' REDIS_PUBSUB_RECONNECT_INTERVAL = 1.0 REDIS_PUBSUB_MAX_RECONNECT_INTERVAL = 30.0 +REDIS_TASK_HEARTBEAT_INTERVAL = REDIS_TASK_TTL // 4 async def redis_task_command_listener(app): @@ -63,17 +64,66 @@ async def redis_task_command_listener(app): reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL) +async def redis_task_heartbeat_loop(app): + """ + Re-stamp this worker's tasks so only entries left behind by a dead worker age out. + """ + if REDIS_TASK_TTL <= 0: + return + + redis: Redis = app.state.redis + + while True: + await asyncio.sleep(REDIS_TASK_HEARTBEAT_INTERVAL) + try: + now = await redis_now(redis) + held_task_ids = [] + pipe = redis.pipeline(transaction=False) + for item_id, task_ids in list(item_tasks.items()): + held_task_ids.extend(task_ids) + if item_id: + key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}' + # xx=True so a task that finishes while this flush is in flight is not re-added. + pipe.zadd(key, {task_id: now for task_id in task_ids}, xx=True) + pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL) + pipe.expire(key, REDIS_TASK_TTL) + await pipe.execute() + if held_task_ids: + with suppress(Exception): + await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, *held_task_ids) + except asyncio.CancelledError: + raise + except Exception: + log.exception('Redis task heartbeat failed') + + ### ------------------------------ ### REDIS-ENABLED HANDLERS ### ------------------------------ +async def redis_now(redis: Redis) -> float: + """Read the Redis clock, so every worker scores and expires task entries against the same one.""" + seconds, microseconds = await redis.time() + return seconds + microseconds / 1_000_000 + + async def redis_save_task(redis: Redis, task_id: str, item_id: str | None): pipe = redis.pipeline() pipe.hset(REDIS_TASKS_KEY, task_id, item_id or '') if item_id: - pipe.sadd(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) + key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}' + if REDIS_TASK_TTL > 0: + now = await redis_now(redis) + pipe.zadd(key, {task_id: now}) + pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL) + pipe.expire(key, REDIS_TASK_TTL) + else: + pipe.zadd(key, {task_id: 0}) await pipe.execute() + if REDIS_TASK_TTL > 0: + with suppress(Exception): + await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, task_id) async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): @@ -81,13 +131,8 @@ async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): pipe.hdel(REDIS_TASKS_KEY, task_id) pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) if item_id: - pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) - await pipe.execute() - # Remove the set key entirely if no tasks remain for this item - if await redis.scard(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') == 0: - await redis.delete(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') - else: - await pipe.execute() + pipe.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) + await pipe.execute() async def redis_list_tasks(redis: Redis) -> list[str]: @@ -95,7 +140,7 @@ async def redis_list_tasks(redis: Redis) -> list[str]: async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]: - return list(await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}')) + return list(await redis.zrange(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', 0, -1)) async def redis_send_command(redis: Redis, command: dict): @@ -118,8 +163,8 @@ async def cleanup_task(redis, task_id: str, id=None): tasks.pop(task_id, None) # Remove the task if it exists response_streams.pop(task_id, None) - # If an ID is provided, remove the task from the item_tasks dictionary - if id and task_id in item_tasks.get(id, []): + # Remove the task from the item_tasks dictionary + if task_id in item_tasks.get(id, []): item_tasks[id].remove(task_id) if not item_tasks[id]: # If no tasks left for this ID, remove the entry item_tasks.pop(id, None) @@ -231,7 +276,7 @@ async def stop_task(redis, task_id: str): Cancel a running task and remove it from the global task list. """ if redis: - # Look up the item_id before cleanup so we can remove the set entry too + # Look up the item_id before cleanup so we can remove the item tasks entry too item_id = await redis.hget(REDIS_TASKS_KEY, task_id) # PUBSUB: All instances check if they have this task, and stop if so. await redis_send_command( @@ -241,7 +286,7 @@ async def stop_task(redis, task_id: str): 'task_id': task_id, }, ) - # Always clean Redis directly — hdel/srem are idempotent, safe even + # Always clean Redis directly — hdel/zrem are idempotent, safe even # if the done_callback on the owning process also fires cleanup. await redis_cleanup_task(redis, task_id, item_id or None) return {'status': True, 'message': f'Task {task_id} stopped.'} @@ -276,6 +321,9 @@ async def stop_item_tasks(redis: Redis, item_id: str): if not result['status']: return result # Return the first failure + if redis: + await redis.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', *task_ids) + return {'status': True, 'message': f'All tasks for item {item_id} stopped.'}