From ba53749d9f3f0764486bf0994a3002208492e765 Mon Sep 17 00:00:00 2001 From: jimbo-p Date: Thu, 4 Jun 2026 12:55:00 -0500 Subject: [PATCH] fix: build pending-knowledge index concurrently and fix SQLite expression Use CREATE INDEX CONCURRENTLY (in an autocommit block) on PostgreSQL so the build does not take an ACCESS EXCLUSIVE lock on the large `file` table during migration; DROP INDEX CONCURRENTLY on downgrade. Correct the SQLite index expression/JSON path to match exactly what SQLAlchemy emits for get_pending_files_for_knowledge (nested JSON_EXTRACT/JSON_QUOTE with double-quoted path keys). The previous form did not match, so SQLite fell back to a full table scan instead of using the index. Refs #25717 --- ...d7a2e4_add_file_pending_knowledge_index.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/backend/open_webui/migrations/versions/b9f3c1d7a2e4_add_file_pending_knowledge_index.py b/backend/open_webui/migrations/versions/b9f3c1d7a2e4_add_file_pending_knowledge_index.py index 6d83564d76..7649646e61 100644 --- a/backend/open_webui/migrations/versions/b9f3c1d7a2e4_add_file_pending_knowledge_index.py +++ b/backend/open_webui/migrations/versions/b9f3c1d7a2e4_add_file_pending_knowledge_index.py @@ -34,22 +34,37 @@ def upgrade(): return if dialect == 'postgresql': - op.execute( - f'CREATE INDEX {INDEX_NAME} ON file ' - "((meta -> 'data' ->> 'knowledge_id')) " - "WHERE (data ->> 'status') IN ('pending', 'processing')" - ) + # CONCURRENTLY avoids an ACCESS EXCLUSIVE lock while the index builds, + # which matters on large `file` tables (file.data holds full extracted + # document content). It cannot run inside a transaction, so use an + # autocommit block. + with op.get_context().autocommit_block(): + op.execute( + f'CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} ON file ' + "((meta -> 'data' ->> 'knowledge_id')) " + "WHERE (data ->> 'status') IN ('pending', 'processing')" + ) elif dialect == 'sqlite': + # Expression/path must match exactly what SQLAlchemy emits for the + # SQLite dialect (nested JSON_EXTRACT/JSON_QUOTE, double-quoted path + # keys), otherwise SQLite will not use this index for the ORM query. op.execute( f'CREATE INDEX {INDEX_NAME} ON file ' - "(json_extract(meta, '$.data.knowledge_id')) " - "WHERE json_extract(data, '$.status') IN ('pending', 'processing')" + '(JSON_EXTRACT(JSON_QUOTE(JSON_EXTRACT(meta, \'$."data"\')), \'$."knowledge_id"\')) ' + 'WHERE JSON_EXTRACT(data, \'$."status"\') IN (\'pending\', \'processing\')' ) def downgrade(): conn = op.get_bind() + dialect = conn.dialect.name existing_indexes = {idx['name'] for idx in sa.inspect(conn).get_indexes('file')} - if INDEX_NAME in existing_indexes: + if INDEX_NAME not in existing_indexes: + return + + if dialect == 'postgresql': + with op.get_context().autocommit_block(): + op.execute(f'DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}') + else: op.drop_index(INDEX_NAME, table_name='file')