From 363ad352fec9553469852d111bc0506b896504a6 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 24 Aug 2026 17:12:56 -0400 Subject: [PATCH] refac --- .env.example | 3 ++ backend/open_webui/config.py | 2 + backend/open_webui/routers/knowledge.py | 66 +++++++++++++++---------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.env.example b/.env.example index 40f5b61fc5..22b128ce8f 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,9 @@ ENABLE_MEMORY_SYSTEM_CONTEXT=true # Set to true to add compact row/column stats to parsed CSV retrieval context. ENABLE_RAG_CSV_SUMMARY=false +# Set to true to preserve backing file records, storage blobs, and per-file vectors when files are removed from knowledge bases. +ENABLE_KNOWLEDGE_FILE_RETENTION=false + # Set to false to disable workspace Tools and Functions. ENABLE_PLUGINS=true diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 33d3840e28..bc00611565 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -973,6 +973,8 @@ RAG_FILE_MAX_COUNT = int(os.getenv('RAG_FILE_MAX_COUNT')) if os.getenv('RAG_FILE RAG_FILE_MAX_SIZE = int(os.getenv('RAG_FILE_MAX_SIZE')) if os.getenv('RAG_FILE_MAX_SIZE') else None +ENABLE_KNOWLEDGE_FILE_RETENTION = os.getenv('ENABLE_KNOWLEDGE_FILE_RETENTION', 'False').lower() == 'true' + RAG_FILE_CONTENT_SEARCH_MAX_CHARS = int(os.getenv('RAG_FILE_CONTENT_SEARCH_MAX_CHARS', str(64 * 1024 * 1024))) FILE_IMAGE_COMPRESSION_WIDTH = ( diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 6c1fcf9fb8..3e75bf6193 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -11,7 +11,11 @@ from urllib.parse import quote from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from fastapi.responses import StreamingResponse -from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL, RAG_EMBEDDING_CONTENT_PREFIX +from open_webui.config import ( + BYPASS_ADMIN_ACCESS_CONTROL, + ENABLE_KNOWLEDGE_FILE_RETENTION, + RAG_EMBEDDING_CONTENT_PREFIX, +) from open_webui.constants import ERROR_MESSAGES from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_session @@ -56,6 +60,25 @@ router = APIRouter() PAGE_ITEM_COUNT = 30 + +async def delete_file_resource(file: FileModel, db: AsyncSession) -> bool: + try: + file_collection = f'file-{file.id}' + if await ASYNC_VECTOR_DB_CLIENT.has_collection(collection_name=file_collection): + await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=file_collection) + except Exception as e: + log.debug('This was most likely caused by bypassing embedding processing') + log.debug(e) + + result = await Files.delete_file_by_id(file.id, db=db) + if result and file.path: + try: + await asyncio.to_thread(Storage.delete_file, file.path) + except Exception as e: + log.debug(e) + + return result + ############################ # Knowledge Base Embedding ############################ @@ -1569,7 +1592,7 @@ async def remove_file_from_knowledge_by_id( request: Request, id: str, form_data: KnowledgeFileIdForm, - delete_file: bool = Query(True), + delete_file: bool = Query(not ENABLE_KNOWLEDGE_FILE_RETENTION), user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session), ): @@ -1630,18 +1653,7 @@ async def remove_file_from_knowledge_by_id( # Anyone with write permission or higher can delete files if delete_file and (file.user_id == user.id or user.role == 'admin'): - try: - # Remove the file's collection from vector database - file_collection = f'file-{form_data.file_id}' - if await ASYNC_VECTOR_DB_CLIENT.has_collection(collection_name=file_collection): - await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=file_collection) - except Exception as e: - log.debug('This was most likely caused by bypassing embedding processing') - log.debug(e) - pass - - # Delete file from database - await Files.delete_file_by_id(form_data.file_id, db=db) + await delete_file_resource(file, db) if knowledge: response = KnowledgeFilesResponse( @@ -1786,12 +1798,18 @@ async def reset_knowledge_by_id( detail=ERROR_MESSAGES.ACCESS_PROHIBITED, ) + files = await Knowledges.get_files_by_id(id, db=db) if not ENABLE_KNOWLEDGE_FILE_RETENTION else [] + try: await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id) except Exception as e: log.debug(e) pass + for file in files: + if file.user_id == user.id or user.role == 'admin': + await delete_file_resource(file, db) + knowledge = await Knowledges.reset_knowledge_by_id(id=id, include_directories=include_directories, db=db) if knowledge: await publish_event( @@ -1966,19 +1984,13 @@ async def sync_knowledge_cleanup( except Exception: pass - try: - collection_name = f'file-{file_id}' - if await ASYNC_VECTOR_DB_CLIENT.has_collection(collection_name): - await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name) - except Exception: - pass - - if file.user_id == user.id or user.role == 'admin': - await Files.delete_file_by_id(file_id, db=db) - try: - await asyncio.to_thread(Storage.delete_file, file.path) - except Exception: - pass + linked_knowledges = await Knowledges.get_knowledges_by_file_id(file_id, db=db) + if ( + not ENABLE_KNOWLEDGE_FILE_RETENTION + and not linked_knowledges + and (file.user_id == user.id or user.role == 'admin') + ): + await delete_file_resource(file, db) # ── Remove orphaned directories (children before parents) ── for dir_id in reversed(form_data.dir_ids):