mirror of
https://github.com/open-webui/open-webui.git
synced 2026-11-16 09:41:23 +00:00
fix: expire task registry entries left behind by a killed worker
Running tasks are recorded in Redis so that any worker can list or stop them, but the cleanup that removes a record only ever runs in the worker that owns the task. A worker killed by an OOM, a crash or a rolling restart therefore leaves its records behind permanently. The chat they belong to keeps reporting a generation in progress, the stop control acts on a task nobody is running any more, and the registry grows for the life of the deployment with no way to clear it short of deleting the keys by hand. Records now carry a TTL, and the worker that owns a task re-stamps its own records on an interval, so the only ones that age out belong to a worker that is gone. The per-item set becomes a sorted set, because it needs a per-entry timestamp that a plain set cannot hold. Scores and expiry read the Redis clock, so workers whose own clocks drift cannot expire each other's live tasks. `REDIS_TASK_TTL` defaults to 300 seconds, and 0 keeps the previous behaviour with no timestamps written at all. The sorted set lives under a new key name so that a rolling deploy never has old and new workers writing two different shapes into one key. A compatibility shim that read both shapes was tried first and dropped: it left the old key to leak forever, which is the bug this fixes.
This commit is contained in:
parent
233681464f
commit
e7a004c78b
3 changed files with 79 additions and 15 deletions
|
|
@ -381,6 +381,15 @@ try:
|
|||
except ValueError:
|
||||
REDIS_RESPONSE_STREAM_TTL = 3600
|
||||
|
||||
# Seconds a task entry survives without a heartbeat. 0 disables.
|
||||
REDIS_TASK_TTL = os.getenv('REDIS_TASK_TTL', '300')
|
||||
try:
|
||||
REDIS_TASK_TTL = int(REDIS_TASK_TTL)
|
||||
if REDIS_TASK_TTL != 0 and REDIS_TASK_TTL < 60:
|
||||
REDIS_TASK_TTL = 300
|
||||
except ValueError:
|
||||
REDIS_TASK_TTL = 300
|
||||
|
||||
REDIS_SENTINEL_HOSTS = os.getenv('REDIS_SENTINEL_HOSTS', '')
|
||||
REDIS_SENTINEL_PORT = os.getenv('REDIS_SENTINEL_PORT', '26379')
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ from open_webui.env import (
|
|||
MAX_BODY_LOG_SIZE,
|
||||
# Redis
|
||||
REDIS_KEY_PREFIX,
|
||||
REDIS_TASK_TTL,
|
||||
REDIS_URL,
|
||||
RESET_CONFIG_ON_START,
|
||||
SAFE_MODE,
|
||||
|
|
@ -199,6 +200,7 @@ from open_webui.tasks import (
|
|||
list_task_ids_by_item_id,
|
||||
list_tasks,
|
||||
redis_task_command_listener,
|
||||
redis_task_heartbeat_loop,
|
||||
stop_item_tasks,
|
||||
stop_task,
|
||||
) # Import from tasks.py
|
||||
|
|
@ -385,6 +387,8 @@ async def lifespan(app: FastAPI):
|
|||
|
||||
if app.state.redis is not None:
|
||||
app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app))
|
||||
if REDIS_TASK_TTL > 0:
|
||||
app.state.redis_task_heartbeat_loop = asyncio.create_task(redis_task_heartbeat_loop(app))
|
||||
|
||||
app.state.periodic_usage_pool_cleanup = asyncio.create_task(periodic_usage_pool_cleanup())
|
||||
app.state.periodic_session_pool_cleanup = asyncio.create_task(periodic_session_pool_cleanup())
|
||||
|
|
@ -472,6 +476,9 @@ async def lifespan(app: FastAPI):
|
|||
if hasattr(app.state, 'redis_task_command_listener'):
|
||||
app.state.redis_task_command_listener.cancel()
|
||||
|
||||
if hasattr(app.state, 'redis_task_heartbeat_loop'):
|
||||
app.state.redis_task_heartbeat_loop.cancel()
|
||||
|
||||
app.state.periodic_usage_pool_cleanup.cancel()
|
||||
app.state.periodic_session_pool_cleanup.cancel()
|
||||
app.state.scheduler_worker_loop.cancel()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from uuid import uuid4
|
|||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL
|
||||
from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL
|
||||
from open_webui.utils.json_codec import JSONCodec, dumps_bytes
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -18,11 +18,12 @@ response_streams: dict[str, dict] = {}
|
|||
|
||||
|
||||
REDIS_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks'
|
||||
REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item'
|
||||
REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item_active'
|
||||
REDIS_RESPONSE_STREAMS_KEY = f'{REDIS_KEY_PREFIX}:tasks:response_streams'
|
||||
REDIS_PUBSUB_CHANNEL = f'{REDIS_KEY_PREFIX}:tasks:commands'
|
||||
REDIS_PUBSUB_RECONNECT_INTERVAL = 1.0
|
||||
REDIS_PUBSUB_MAX_RECONNECT_INTERVAL = 30.0
|
||||
REDIS_TASK_HEARTBEAT_INTERVAL = REDIS_TASK_TTL // 4
|
||||
|
||||
|
||||
async def redis_task_command_listener(app):
|
||||
|
|
@ -63,17 +64,66 @@ async def redis_task_command_listener(app):
|
|||
reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL)
|
||||
|
||||
|
||||
async def redis_task_heartbeat_loop(app):
|
||||
"""
|
||||
Re-stamp this worker's tasks so only entries left behind by a dead worker age out.
|
||||
"""
|
||||
if REDIS_TASK_TTL <= 0:
|
||||
return
|
||||
|
||||
redis: Redis = app.state.redis
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(REDIS_TASK_HEARTBEAT_INTERVAL)
|
||||
try:
|
||||
now = await redis_now(redis)
|
||||
held_task_ids = []
|
||||
pipe = redis.pipeline(transaction=False)
|
||||
for item_id, task_ids in list(item_tasks.items()):
|
||||
held_task_ids.extend(task_ids)
|
||||
if item_id:
|
||||
key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}'
|
||||
# xx=True so a task that finishes while this flush is in flight is not re-added.
|
||||
pipe.zadd(key, {task_id: now for task_id in task_ids}, xx=True)
|
||||
pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL)
|
||||
pipe.expire(key, REDIS_TASK_TTL)
|
||||
await pipe.execute()
|
||||
if held_task_ids:
|
||||
with suppress(Exception):
|
||||
await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, *held_task_ids)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
log.exception('Redis task heartbeat failed')
|
||||
|
||||
|
||||
### ------------------------------
|
||||
### REDIS-ENABLED HANDLERS
|
||||
### ------------------------------
|
||||
|
||||
|
||||
async def redis_now(redis: Redis) -> float:
|
||||
"""Read the Redis clock, so every worker scores and expires task entries against the same one."""
|
||||
seconds, microseconds = await redis.time()
|
||||
return seconds + microseconds / 1_000_000
|
||||
|
||||
|
||||
async def redis_save_task(redis: Redis, task_id: str, item_id: str | None):
|
||||
pipe = redis.pipeline()
|
||||
pipe.hset(REDIS_TASKS_KEY, task_id, item_id or '')
|
||||
if item_id:
|
||||
pipe.sadd(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
|
||||
key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}'
|
||||
if REDIS_TASK_TTL > 0:
|
||||
now = await redis_now(redis)
|
||||
pipe.zadd(key, {task_id: now})
|
||||
pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL)
|
||||
pipe.expire(key, REDIS_TASK_TTL)
|
||||
else:
|
||||
pipe.zadd(key, {task_id: 0})
|
||||
await pipe.execute()
|
||||
if REDIS_TASK_TTL > 0:
|
||||
with suppress(Exception):
|
||||
await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, task_id)
|
||||
|
||||
|
||||
async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None):
|
||||
|
|
@ -81,13 +131,8 @@ async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None):
|
|||
pipe.hdel(REDIS_TASKS_KEY, task_id)
|
||||
pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id)
|
||||
if item_id:
|
||||
pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
|
||||
await pipe.execute()
|
||||
# Remove the set key entirely if no tasks remain for this item
|
||||
if await redis.scard(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') == 0:
|
||||
await redis.delete(f'{REDIS_ITEM_TASKS_KEY}:{item_id}')
|
||||
else:
|
||||
await pipe.execute()
|
||||
pipe.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
|
||||
await pipe.execute()
|
||||
|
||||
|
||||
async def redis_list_tasks(redis: Redis) -> list[str]:
|
||||
|
|
@ -95,7 +140,7 @@ async def redis_list_tasks(redis: Redis) -> list[str]:
|
|||
|
||||
|
||||
async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]:
|
||||
return list(await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}'))
|
||||
return list(await redis.zrange(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', 0, -1))
|
||||
|
||||
|
||||
async def redis_send_command(redis: Redis, command: dict):
|
||||
|
|
@ -118,8 +163,8 @@ async def cleanup_task(redis, task_id: str, id=None):
|
|||
tasks.pop(task_id, None) # Remove the task if it exists
|
||||
response_streams.pop(task_id, None)
|
||||
|
||||
# If an ID is provided, remove the task from the item_tasks dictionary
|
||||
if id and task_id in item_tasks.get(id, []):
|
||||
# Remove the task from the item_tasks dictionary
|
||||
if task_id in item_tasks.get(id, []):
|
||||
item_tasks[id].remove(task_id)
|
||||
if not item_tasks[id]: # If no tasks left for this ID, remove the entry
|
||||
item_tasks.pop(id, None)
|
||||
|
|
@ -231,7 +276,7 @@ async def stop_task(redis, task_id: str):
|
|||
Cancel a running task and remove it from the global task list.
|
||||
"""
|
||||
if redis:
|
||||
# Look up the item_id before cleanup so we can remove the set entry too
|
||||
# Look up the item_id before cleanup so we can remove the item tasks entry too
|
||||
item_id = await redis.hget(REDIS_TASKS_KEY, task_id)
|
||||
# PUBSUB: All instances check if they have this task, and stop if so.
|
||||
await redis_send_command(
|
||||
|
|
@ -241,7 +286,7 @@ async def stop_task(redis, task_id: str):
|
|||
'task_id': task_id,
|
||||
},
|
||||
)
|
||||
# Always clean Redis directly — hdel/srem are idempotent, safe even
|
||||
# Always clean Redis directly — hdel/zrem are idempotent, safe even
|
||||
# if the done_callback on the owning process also fires cleanup.
|
||||
await redis_cleanup_task(redis, task_id, item_id or None)
|
||||
return {'status': True, 'message': f'Task {task_id} stopped.'}
|
||||
|
|
@ -276,6 +321,9 @@ async def stop_item_tasks(redis: Redis, item_id: str):
|
|||
if not result['status']:
|
||||
return result # Return the first failure
|
||||
|
||||
if redis:
|
||||
await redis.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', *task_ids)
|
||||
|
||||
return {'status': True, 'message': f'All tasks for item {item_id} stopped.'}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue