From a59cab0aa990c32dc3b633cb9eda01384b683115 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:00:29 +0200 Subject: [PATCH] perf: knowledge base changes slow down as the knowledge base grows (#33335) Adding, updating or removing knowledge base files through the API sent back the metadata of every file in the knowledge base, and so did renaming a knowledge base or changing its access. These calls got slower as the knowledge base grew, so bulk ingestion through the API lost speed with every batch. Adding or updating files now returns only those files, while removing a file, renaming or changing access returns an empty files list. API clients that read the files list from these responses now get only the affected files; the full list stays available, paginated, from `GET /api/v1/knowledge/{id}/files`. Timings with a local mock embedding server that answers instantly, median of 5 runs (at 1,000 files the per-file work of a batch still hides the difference): | Files already in the knowledge base | Call | Before | After | Response size before | Response size after | |---|---|---|---|---|---| | 1,000 | batch add of 100 files | 519 ms | 528 ms | 437 KB | 39 KB | | 10,000 | batch add of 100 files | 679 ms | 476 ms | 4.0 MB | 39 KB | | 10,000 | single file add | 184 ms | 36 ms | 4.0 MB | 0.6 KB | Closes #31397 --- backend/open_webui/models/knowledge.py | 25 ------------------------- backend/open_webui/routers/knowledge.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index e4c97bcecc..b053344bc9 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -671,31 +671,6 @@ class KnowledgeTable: except Exception: return [] - async def get_file_metadatas_by_id( - self, knowledge_id: str, db: Optional[AsyncSession] = None - ) -> list[FileMetadataResponse]: - """Column-only listing: File.data holds each file's full extracted - text, which metadata views must never load.""" - try: - async with get_async_db_context(db) as db: - result = await db.execute( - select(File.id, File.hash, File.meta, File.created_at, File.updated_at) - .join(KnowledgeFile, File.id == KnowledgeFile.file_id) - .filter(KnowledgeFile.knowledge_id == knowledge_id) - ) - return [ - FileMetadataResponse( - id=row.id, - hash=row.hash, - meta=row.meta, - created_at=row.created_at, - updated_at=row.updated_at, - ) - for row in result.all() - ] - except Exception: - return [] - async def add_file_to_knowledge_by_id( self, knowledge_id: str, diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 2c7a9f410d..6b13b798cd 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -1180,7 +1180,7 @@ async def update_knowledge_by_id( ) response = KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id), + files=[], ) await publish_event( request, @@ -1249,7 +1249,7 @@ async def update_knowledge_access_by_id( response = KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(id, db=db), + files=[], ) await publish_event( request, @@ -1489,7 +1489,7 @@ async def add_file_to_knowledge_by_id( if knowledge: response = KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=await Files.get_file_metadatas_by_ids([form_data.file_id], db=db), ) await publish_event( request, @@ -1573,7 +1573,7 @@ async def update_file_from_knowledge_by_id( if knowledge: response = KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=await Files.get_file_metadatas_by_ids([form_data.file_id], db=db), ) await publish_event( request, @@ -1662,7 +1662,7 @@ async def remove_file_from_knowledge_by_id( if knowledge: response = KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=[], ) await publish_event( request, @@ -2105,7 +2105,7 @@ async def add_files_to_knowledge_batch( if not new_entries: return KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=[], ) # Narrow the file list to only new files for processing @@ -2141,7 +2141,7 @@ async def add_files_to_knowledge_batch( error_details = [f'{err.file_id}: {err.error}' for err in result.errors] return KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=await Files.get_file_metadatas_by_ids(successful_file_ids, db=db), warnings={ 'message': 'Some files failed to process', 'errors': error_details, @@ -2150,7 +2150,7 @@ async def add_files_to_knowledge_batch( return KnowledgeFilesResponse( **knowledge.model_dump(), - files=await Knowledges.get_file_metadatas_by_id(knowledge.id, db=db), + files=await Files.get_file_metadatas_by_ids(successful_file_ids, db=db), )