fix: expire task registry entries left behind by a killed worker

Running tasks are recorded in Redis so that any worker can list or stop them, but the cleanup that removes a record only ever runs in the worker that owns the task. A worker killed by an OOM, a crash or a rolling restart therefore leaves its records behind permanently. The chat they belong to keeps reporting a generation in progress, the stop control acts on a task nobody is running any more, and the registry grows for the life of the deployment with no way to clear it short of deleting the keys by hand.

Records now carry a TTL, and the worker that owns a task re-stamps its own records on an interval, so the only ones that age out belong to a worker that is gone. The per-item set becomes a sorted set, because it needs a per-entry timestamp that a plain set cannot hold. Scores and expiry read the Redis clock, so workers whose own clocks drift cannot expire each other's live tasks. `REDIS_TASK_TTL` defaults to 300 seconds, and 0 keeps the previous behaviour with no timestamps written at all.

The sorted set lives under a new key name so that a rolling deploy never has old and new workers writing two different shapes into one key. A compatibility shim that read both shapes was tried first and dropped: it left the old key to leak forever, which is the bug this fixes.
This commit is contained in:
Classic298 2026-09-02 00:01:40 +02:00
parent 233681464f
commit e7a004c78b
3 changed files with 79 additions and 15 deletions

View file

@ -381,6 +381,15 @@ try:
except ValueError:
REDIS_RESPONSE_STREAM_TTL = 3600
# Seconds a task entry survives without a heartbeat. 0 disables.
REDIS_TASK_TTL = os.getenv('REDIS_TASK_TTL', '300')
try:
REDIS_TASK_TTL = int(REDIS_TASK_TTL)
if REDIS_TASK_TTL != 0 and REDIS_TASK_TTL < 60:
REDIS_TASK_TTL = 300
except ValueError:
REDIS_TASK_TTL = 300
REDIS_SENTINEL_HOSTS = os.getenv('REDIS_SENTINEL_HOSTS', '')
REDIS_SENTINEL_PORT = os.getenv('REDIS_SENTINEL_PORT', '26379')

View file

@ -106,6 +106,7 @@ from open_webui.env import (
MAX_BODY_LOG_SIZE,
# Redis
REDIS_KEY_PREFIX,
REDIS_TASK_TTL,
REDIS_URL,
RESET_CONFIG_ON_START,
SAFE_MODE,
@ -199,6 +200,7 @@ from open_webui.tasks import (
list_task_ids_by_item_id,
list_tasks,
redis_task_command_listener,
redis_task_heartbeat_loop,
stop_item_tasks,
stop_task,
) # Import from tasks.py
@ -385,6 +387,8 @@ async def lifespan(app: FastAPI):
if app.state.redis is not None:
app.state.redis_task_command_listener = asyncio.create_task(redis_task_command_listener(app))
if REDIS_TASK_TTL > 0:
app.state.redis_task_heartbeat_loop = asyncio.create_task(redis_task_heartbeat_loop(app))
app.state.periodic_usage_pool_cleanup = asyncio.create_task(periodic_usage_pool_cleanup())
app.state.periodic_session_pool_cleanup = asyncio.create_task(periodic_session_pool_cleanup())
@ -472,6 +476,9 @@ async def lifespan(app: FastAPI):
if hasattr(app.state, 'redis_task_command_listener'):
app.state.redis_task_command_listener.cancel()
if hasattr(app.state, 'redis_task_heartbeat_loop'):
app.state.redis_task_heartbeat_loop.cancel()
app.state.periodic_usage_pool_cleanup.cancel()
app.state.periodic_session_pool_cleanup.cancel()
app.state.scheduler_worker_loop.cancel()

View file

@ -6,7 +6,7 @@ from uuid import uuid4
from redis.asyncio import Redis
from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL
from open_webui.env import REDIS_KEY_PREFIX, REDIS_RESPONSE_STREAM_TTL, REDIS_TASK_TTL
from open_webui.utils.json_codec import JSONCodec, dumps_bytes
log = logging.getLogger(__name__)
@ -18,11 +18,12 @@ response_streams: dict[str, dict] = {}
REDIS_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks'
REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item'
REDIS_ITEM_TASKS_KEY = f'{REDIS_KEY_PREFIX}:tasks:item_active'
REDIS_RESPONSE_STREAMS_KEY = f'{REDIS_KEY_PREFIX}:tasks:response_streams'
REDIS_PUBSUB_CHANNEL = f'{REDIS_KEY_PREFIX}:tasks:commands'
REDIS_PUBSUB_RECONNECT_INTERVAL = 1.0
REDIS_PUBSUB_MAX_RECONNECT_INTERVAL = 30.0
REDIS_TASK_HEARTBEAT_INTERVAL = REDIS_TASK_TTL // 4
async def redis_task_command_listener(app):
@ -63,17 +64,66 @@ async def redis_task_command_listener(app):
reconnect_interval = min(reconnect_interval * 2, REDIS_PUBSUB_MAX_RECONNECT_INTERVAL)
async def redis_task_heartbeat_loop(app):
"""
Re-stamp this worker's tasks so only entries left behind by a dead worker age out.
"""
if REDIS_TASK_TTL <= 0:
return
redis: Redis = app.state.redis
while True:
await asyncio.sleep(REDIS_TASK_HEARTBEAT_INTERVAL)
try:
now = await redis_now(redis)
held_task_ids = []
pipe = redis.pipeline(transaction=False)
for item_id, task_ids in list(item_tasks.items()):
held_task_ids.extend(task_ids)
if item_id:
key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}'
# xx=True so a task that finishes while this flush is in flight is not re-added.
pipe.zadd(key, {task_id: now for task_id in task_ids}, xx=True)
pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL)
pipe.expire(key, REDIS_TASK_TTL)
await pipe.execute()
if held_task_ids:
with suppress(Exception):
await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, *held_task_ids)
except asyncio.CancelledError:
raise
except Exception:
log.exception('Redis task heartbeat failed')
### ------------------------------
### REDIS-ENABLED HANDLERS
### ------------------------------
async def redis_now(redis: Redis) -> float:
"""Read the Redis clock, so every worker scores and expires task entries against the same one."""
seconds, microseconds = await redis.time()
return seconds + microseconds / 1_000_000
async def redis_save_task(redis: Redis, task_id: str, item_id: str | None):
pipe = redis.pipeline()
pipe.hset(REDIS_TASKS_KEY, task_id, item_id or '')
if item_id:
pipe.sadd(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
key = f'{REDIS_ITEM_TASKS_KEY}:{item_id}'
if REDIS_TASK_TTL > 0:
now = await redis_now(redis)
pipe.zadd(key, {task_id: now})
pipe.zremrangebyscore(key, '-inf', now - REDIS_TASK_TTL)
pipe.expire(key, REDIS_TASK_TTL)
else:
pipe.zadd(key, {task_id: 0})
await pipe.execute()
if REDIS_TASK_TTL > 0:
with suppress(Exception):
await redis.hexpire(REDIS_TASKS_KEY, REDIS_TASK_TTL, task_id)
async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None):
@ -81,13 +131,8 @@ async def redis_cleanup_task(redis: Redis, task_id: str, item_id: str | None):
pipe.hdel(REDIS_TASKS_KEY, task_id)
pipe.hdel(REDIS_RESPONSE_STREAMS_KEY, task_id)
if item_id:
pipe.srem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
await pipe.execute()
# Remove the set key entirely if no tasks remain for this item
if await redis.scard(f'{REDIS_ITEM_TASKS_KEY}:{item_id}') == 0:
await redis.delete(f'{REDIS_ITEM_TASKS_KEY}:{item_id}')
else:
await pipe.execute()
pipe.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', task_id)
await pipe.execute()
async def redis_list_tasks(redis: Redis) -> list[str]:
@ -95,7 +140,7 @@ async def redis_list_tasks(redis: Redis) -> list[str]:
async def redis_list_item_tasks(redis: Redis, item_id: str) -> list[str]:
return list(await redis.smembers(f'{REDIS_ITEM_TASKS_KEY}:{item_id}'))
return list(await redis.zrange(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', 0, -1))
async def redis_send_command(redis: Redis, command: dict):
@ -118,8 +163,8 @@ async def cleanup_task(redis, task_id: str, id=None):
tasks.pop(task_id, None) # Remove the task if it exists
response_streams.pop(task_id, None)
# If an ID is provided, remove the task from the item_tasks dictionary
if id and task_id in item_tasks.get(id, []):
# Remove the task from the item_tasks dictionary
if task_id in item_tasks.get(id, []):
item_tasks[id].remove(task_id)
if not item_tasks[id]: # If no tasks left for this ID, remove the entry
item_tasks.pop(id, None)
@ -231,7 +276,7 @@ async def stop_task(redis, task_id: str):
Cancel a running task and remove it from the global task list.
"""
if redis:
# Look up the item_id before cleanup so we can remove the set entry too
# Look up the item_id before cleanup so we can remove the item tasks entry too
item_id = await redis.hget(REDIS_TASKS_KEY, task_id)
# PUBSUB: All instances check if they have this task, and stop if so.
await redis_send_command(
@ -241,7 +286,7 @@ async def stop_task(redis, task_id: str):
'task_id': task_id,
},
)
# Always clean Redis directly — hdel/srem are idempotent, safe even
# Always clean Redis directly — hdel/zrem are idempotent, safe even
# if the done_callback on the owning process also fires cleanup.
await redis_cleanup_task(redis, task_id, item_id or None)
return {'status': True, 'message': f'Task {task_id} stopped.'}
@ -276,6 +321,9 @@ async def stop_item_tasks(redis: Redis, item_id: str):
if not result['status']:
return result # Return the first failure
if redis:
await redis.zrem(f'{REDIS_ITEM_TASKS_KEY}:{item_id}', *task_ids)
return {'status': True, 'message': f'All tasks for item {item_id} stopped.'}