diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index b17df7340a..36d2cb9fbb 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -381,6 +381,15 @@ try: except ValueError: REDIS_RESPONSE_STREAM_TTL = 3600 +# Seconds a task entry survives without a heartbeat. 0 disables. +REDIS_TASK_TTL = os.getenv('REDIS_TASK_TTL', '300') +try: + REDIS_TASK_TTL = int(REDIS_TASK_TTL) + if REDIS_TASK_TTL != 0 and REDIS_TASK_TTL < 60: + REDIS_TASK_TTL = 300 +except ValueError: + REDIS_TASK_TTL = 300 + REDIS_SENTINEL_HOSTS = os.getenv('REDIS_SENTINEL_HOSTS', '') REDIS_SENTINEL_PORT = os.getenv('REDIS_SENTINEL_PORT', '26379') diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index c84a394b87..68d2e777a7 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -106,6 +106,7 @@ from open_webui.env import ( MAX_BODY_LOG_SIZE, # Redis REDIS_KEY_PREFIX, + REDIS_TASK_TTL, REDIS_URL, RESET_CONFIG_ON_START, SAFE_MODE, @@ -199,6 +200,7 @@ from open_webui.tasks import ( list_task_ids_by_item_id, list_tasks, redis_task_command_listener, + redis_task_heartbeat_loop, stop_item_tasks, stop_task, ) # Import from tasks.py @@ -385,6 +387,8 @@ async def lifespan(app: FastAPI): if app.state.redis is not None: app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app)) + if REDIS_TASK_TTL > 0: + app.state.redis_task_heartbeat_loop = asyncio.create_task(redis_task_heartbeat_loop(app)) app.state.periodic_usage_pool_cleanup = asyncio.create_task(periodic_usage_pool_cleanup()) app.state.periodic_session_pool_cleanup = asyncio.create_task(periodic_session_pool_cleanup()) @@ -472,6 +476,9 @@ async def lifespan(app: FastAPI): if hasattr(app.state, 'redis_task_command_listener'): app.state.redis_task_command_listener.cancel() + if hasattr(app.state, 'redis_task_heartbeat_loop'): + app.state.redis_task_heartbeat_loop.cancel() + app.state.periodic_usage_pool_cleanup.cancel() app.state.periodic_session_pool_cleanup.cancel() app.state.scheduler_worker_loop.cancel() diff --git a/backend/open_webui/tasks.py b/backend/open_webui/tasks.py index ef15c57a18..e54700c2e0 100644 --- a/backend/open_webui/tasks.py +++ b/backend/open_webui/tasks.py @@ -6,7 +6,7 @@ from uuid import uuid4 from redis.asyncio import Redis -from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL +from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL from open_webui.utils.json_codec import JSONCodec, dumps_bytes log = logging.getLogger(__name__) @@ -18,11 +18,12 @@ response_streams: dict[str, dict] = {} REDIS_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks' -REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item' +REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item_active' REDIS_RESPONSE_STREAMS_KEY = f'{REDIS_KEY_PREFIX}:tasks:response_streams' REDIS_PUBSUB_CHANNEL = f'{REDIS_KEY_PREFIX}:tasks:commands' REDIS_PUBSUB_RECONNECT_INTERVAL = 1.0 REDIS_PUBSUB_MAX_RECONNECT_INTERVAL = 30.0 +REDIS_TASK_HEARTBEAT_INTERVAL = REDIS_TASK_TTL // 4 async def redis_task_command_listener(app): @@ -63,17 +64,66 @@ async def redis_task_command_listener(app): reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL) +async def redis_task_heartbeat_loop(app): + """ + Re-stamp this worker's tasks so only entries left behind by a dead worker age out. + """ + if REDIS_TASK_TTL <= 0: + return + + redis: Redis = app.state.redis + + while True: + await asyncio.sleep(REDIS_TASK_HEARTBEAT_INTERVAL) + try: + now = await redis_now(redis) + held_task_ids = [] + pipe = redis.pipeline(transaction=False) + for item_id, task_ids in list(item_tasks.items()): + held_task_ids.extend(task_ids) + if item_id: + key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}' + # xx=True so a task that finishes while this flush is in flight is not re-added. + pipe.zadd(key, {task_id: now for task_id in task_ids}, xx=True) + pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL) + pipe.expire(key, REDIS_TASK_TTL) + await pipe.execute() + if held_task_ids: + with suppress(Exception): + await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, *held_task_ids) + except asyncio.CancelledError: + raise + except Exception: + log.exception('Redis task heartbeat failed') + + ### ------------------------------ ### REDIS-ENABLED HANDLERS ### ------------------------------ +async def redis_now(redis: Redis) -> float: + """Read the Redis clock, so every worker scores and expires task entries against the same one.""" + seconds, microseconds = await redis.time() + return seconds + microseconds / 1_000_000 + + async def redis_save_task(redis: Redis, task_id: str, item_id: str | None): pipe = redis.pipeline() pipe.hset(REDIS_TASKS_KEY, task_id, item_id or '') if item_id: - pipe.sadd(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) + key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}' + if REDIS_TASK_TTL > 0: + now = await redis_now(redis) + pipe.zadd(key, {task_id: now}) + pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL) + pipe.expire(key, REDIS_TASK_TTL) + else: + pipe.zadd(key, {task_id: 0}) await pipe.execute() + if REDIS_TASK_TTL > 0: + with suppress(Exception): + await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, task_id) async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): @@ -81,13 +131,8 @@ async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None): pipe.hdel(REDIS_TASKS_KEY, task_id) pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id) if item_id: - pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) - await pipe.execute() - # Remove the set key entirely if no tasks remain for this item - if await redis.scard(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') == 0: - await redis.delete(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') - else: - await pipe.execute() + pipe.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id) + await pipe.execute() async def redis_list_tasks(redis: Redis) -> list[str]: @@ -95,7 +140,7 @@ async def redis_list_tasks(redis: Redis) -> list[str]: async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]: - return list(await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}')) + return list(await redis.zrange(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', 0, -1)) async def redis_send_command(redis: Redis, command: dict): @@ -118,8 +163,8 @@ async def cleanup_task(redis, task_id: str, id=None): tasks.pop(task_id, None) # Remove the task if it exists response_streams.pop(task_id, None) - # If an ID is provided, remove the task from the item_tasks dictionary - if id and task_id in item_tasks.get(id, []): + # Remove the task from the item_tasks dictionary + if task_id in item_tasks.get(id, []): item_tasks[id].remove(task_id) if not item_tasks[id]: # If no tasks left for this ID, remove the entry item_tasks.pop(id, None) @@ -231,7 +276,7 @@ async def stop_task(redis, task_id: str): Cancel a running task and remove it from the global task list. """ if redis: - # Look up the item_id before cleanup so we can remove the set entry too + # Look up the item_id before cleanup so we can remove the item tasks entry too item_id = await redis.hget(REDIS_TASKS_KEY, task_id) # PUBSUB: All instances check if they have this task, and stop if so. await redis_send_command( @@ -241,7 +286,7 @@ async def stop_task(redis, task_id: str): 'task_id': task_id, }, ) - # Always clean Redis directly — hdel/srem are idempotent, safe even + # Always clean Redis directly — hdel/zrem are idempotent, safe even # if the done_callback on the owning process also fires cleanup. await redis_cleanup_task(redis, task_id, item_id or None) return {'status': True, 'message': f'Task {task_id} stopped.'} @@ -276,6 +321,9 @@ async def stop_item_tasks(redis: Redis, item_id: str): if not result['status']: return result # Return the first failure + if redis: + await redis.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', *task_ids) + return {'status': True, 'message': f'All tasks for item {item_id} stopped.'}