From 6b2c927828bc5986610677c9c5b7ffa79b1b5846 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:14:54 +0100 Subject: [PATCH] feat: Smart sync for knowledge base directory uploads (#146) Co-authored-by: Stoyan Zlatev <47269030+Stoyan-Zlatev@users.noreply.github.com> --- backend/open_webui/routers/files.py | 6 + backend/open_webui/routers/knowledge.py | 313 ++++++++++++- backend/open_webui/routers/retrieval.py | 1 - backend/open_webui/utils/misc.py | 5 + src/lib/apis/knowledge/index.ts | 96 ++++ .../workspace/Knowledge/KnowledgeBase.svelte | 417 +++++++++++------- .../KnowledgeBase/AddContentMenu.svelte | 2 +- 7 files changed, 667 insertions(+), 173 deletions(-) diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index 8f1ee13f7f..b3f5d1aa0a 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -50,6 +50,8 @@ from open_webui.storage.provider import Storage from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL from open_webui.utils.auth import get_admin_user, get_verified_user +from open_webui.utils.access_control import has_access +from open_webui.utils.misc import calculate_sha256_bytes from open_webui.utils.misc import strict_match_mime_type from pydantic import BaseModel @@ -236,6 +238,9 @@ async def upload_file_handler( }, ) + # Calculate SHA-256 hash of the raw file bytes for sync comparison + file_hash = calculate_sha256_bytes(contents) + file_item = await Files.insert_new_file( user.id, FileForm( @@ -250,6 +255,7 @@ async def upload_file_handler( 'name': name, 'content_type': (file.content_type if isinstance(file.content_type, str) else None), 'size': len(contents), + 'file_hash': file_hash, 'data': file_metadata, }, } diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index f6c3416c8d..6c5c4230fa 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -1,6 +1,6 @@ from typing import List, Optional from pydantic import BaseModel -from fastapi import APIRouter, Depends, HTTPException, status, Request, Query +from fastapi import APIRouter, Depends, HTTPException, status, Request, Query, UploadFile, File, Form from fastapi.responses import StreamingResponse from fastapi.concurrency import run_in_threadpool import logging @@ -27,10 +27,12 @@ from open_webui.routers.retrieval import ( BatchProcessFilesForm, ) from open_webui.storage.provider import Storage +from open_webui.routers.files import upload_file_handler from open_webui.constants import ERROR_MESSAGES from open_webui.utils.auth import get_verified_user, get_admin_user -from open_webui.utils.access_control import has_permission, filter_allowed_access_grants +from open_webui.utils.access_control import has_access, has_permission, filter_allowed_access_grants +from open_webui.utils.misc import calculate_sha256 from open_webui.models.access_grants import AccessGrants @@ -687,6 +689,132 @@ async def add_file_to_knowledge_by_id( ) +############################ +# UploadAndReplaceFile +############################ + + +class UploadAndReplaceResponse(BaseModel): + """Response from upload_and_replace endpoint.""" + + new_file_id: str + old_file_id: str + filename: str + + +@router.post('/{id}/file/upload_and_replace', response_model=UploadAndReplaceResponse) +async def upload_and_replace_file( + request: Request, + id: str, + file: UploadFile = File(...), + old_file_id: str = Form(...), + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + """ + Atomically upload a new file and replace an existing file in the knowledge base. + """ + # Validate knowledge base exists and user has access + knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + if ( + knowledge.user_id != user.id + and not await AccessGrants.has_access( + user_id=user.id, + resource_type='knowledge', + resource_id=knowledge.id, + permission='write', + db=db, + ) + and user.role != 'admin' + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + # Validate old file exists + old_file = await Files.get_file_by_id(old_file_id, db=db) + if not old_file: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + # Step 1: Upload the new file (reuses existing upload_file_handler) + try: + new_file_result = await upload_file_handler( + request, + file=file, + process=True, + process_in_background=False, + user=user, + db=db, + ) + new_file_id = new_file_result['id'] + except Exception as e: + log.error(f'Failed to upload new file: {e}') + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f'Failed to upload file: {str(e)}', + ) + + # Step 2: Verify new file was processed + new_file = await Files.get_file_by_id(new_file_id, db=db) + if not new_file or not new_file.data: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.FILE_NOT_PROCESSED, + ) + + # Step 3: Add new file to knowledge base (reuses existing process_file) + try: + process_file( + request, + ProcessFileForm(file_id=new_file_id, collection_name=id), + user=user, + db=db, + ) + await Knowledges.add_file_to_knowledge_by_id( + knowledge_id=id, file_id=new_file_id, user_id=user.id, db=db + ) + except Exception as e: + log.error(f'Failed to add new file to knowledge base: {e}') + # Clean up: delete the uploaded file since we couldn't add it to KB + try: + await Files.delete_file_by_id(new_file_id, db=db) + except Exception: + pass + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f'Failed to add file to knowledge base: {str(e)}', + ) + + # Step 4: Remove old file (reuses existing remove_file_from_knowledge_by_id) + try: + await remove_file_from_knowledge_by_id( + id=id, + form_data=KnowledgeFileIdForm(file_id=old_file_id), + delete_file=True, + user=user, + db=db, + ) + except Exception as e: + log.error(f'Failed to remove old file (new file is already added): {e}') + # Don't fail - new file is already added successfully + + return UploadAndReplaceResponse( + new_file_id=new_file_id, + old_file_id=old_file_id, + filename=new_file.filename, + ) + + @router.post('/{id}/file/update', response_model=Optional[KnowledgeFilesResponse]) async def update_file_from_knowledge_by_id( request: Request, @@ -966,6 +1094,187 @@ async def reset_knowledge_by_id(id: str, user=Depends(get_verified_user), db: As return knowledge +############################ +# SyncCompare +############################ + + +class FileSyncCompareItem(BaseModel): + """Item for comparing a file during sync.""" + + file_path: str # Relative path within the directory (e.g., "docs/readme.md") + file_hash: str # SHA-256 hash of the raw file bytes + size: int # File size in bytes + + +class SyncCompareForm(BaseModel): + """Form for comparing files for sync.""" + + files: List[FileSyncCompareItem] + + +class ChangedFileInfo(BaseModel): + """Info about a changed file that needs to be replaced.""" + + file_path: str # Path of the new file to upload + old_file_id: str # ID of the old file to delete after upload + + +class SyncCompareResponse(BaseModel): + """Response from sync compare endpoint.""" + + new_files: List[str] # file_paths for new files (no old version exists) + changed_files: List[ChangedFileInfo] # files that changed (upload new, delete old) + removed_file_ids: List[str] # file_ids to remove (no new version exists) + unchanged: List[str] # file_paths that are already up to date + + +async def get_file_hash(file: FileModel, persist: bool = False) -> Optional[str]: + """ + Get the file hash from meta, or calculate it on-demand from the stored file. + This provides backwards compatibility for files uploaded before file_hash was stored. + + Args: + file: The file model to get hash for + persist: If True and hash was calculated (not from meta), persist it to database. + Only set to True when you know the file will NOT be replaced. + """ + # First check if file_hash is already stored in meta + if file.meta and file.meta.get('file_hash'): + return file.meta.get('file_hash') + + # If not, calculate it from the stored file + if not file.path: + log.warning(f'File {file.id} has no path, cannot calculate hash') + return None + + try: + # Get the local file path (downloads from cloud storage if needed) + local_path = Storage.get_file(file.path) + # Calculate hash with 8KB chunks + file_hash = calculate_sha256(local_path, 8192) + + # Only persist if explicitly requested (when file is confirmed unchanged) + if persist: + await Files.update_file_metadata_by_id( + file.id, + {'file_hash': file_hash}, + ) + + return file_hash + except Exception as e: + log.error(f'Failed to calculate hash for file {file.id}: {e}') + return None + + +@router.post('/{id}/sync/compare', response_model=SyncCompareResponse) +async def compare_files_for_sync( + id: str, + form_data: SyncCompareForm, + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + """ + Compare uploaded files against existing knowledge base files. + Returns lists of files that need to be uploaded, deleted, or are unchanged. + """ + knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db) + if not knowledge: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.NOT_FOUND, + ) + + if ( + knowledge.user_id != user.id + and not await AccessGrants.has_access( + user_id=user.id, + resource_type='knowledge', + resource_id=knowledge.id, + permission='write', + db=db, + ) + and user.role != 'admin' + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=ERROR_MESSAGES.ACCESS_PROHIBITED, + ) + + # Get all files currently in the knowledge base + existing_files = await Knowledges.get_files_by_id(id, db=db) + + # Build a map of existing files by filename for quick lookup + existing_by_filename: dict[str, FileModel] = {} + for file in existing_files: + # Use the original filename from meta if available, otherwise use filename field + filename = file.meta.get('name', file.filename) if file.meta else file.filename + existing_by_filename[filename] = file + + # Track files from the incoming directory + incoming_filenames = set() + new_files: List[str] = [] # New files (no old version) + changed_files: List[ChangedFileInfo] = [] # Changed files (need upload + delete old) + removed_file_ids: List[str] = [] # Removed files (no new version) + unchanged: List[str] = [] + # Track files that need hash persisted (unchanged files without stored hash) + files_needing_hash_persist: list[tuple[str, str]] = [] # [(file_id, hash), ...] + + for incoming_file in form_data.files: + incoming_filenames.add(incoming_file.file_path) + + # Check if file exists in knowledge base + existing_file = existing_by_filename.get(incoming_file.file_path) + + if existing_file: + # Check if hash was already stored in meta + had_stored_hash = bool( + existing_file.meta and existing_file.meta.get('file_hash') + ) + + # File exists - check if it has changed (don't persist yet) + existing_hash = await get_file_hash(existing_file, persist=False) + + if existing_hash and existing_hash == incoming_file.file_hash: + # File unchanged + unchanged.append(incoming_file.file_path) + # If hash was calculated on-demand (not from meta), queue for persistence + if not had_stored_hash: + files_needing_hash_persist.append((existing_file.id, existing_hash)) + else: + # File changed or hash calculation failed - need to upload new and delete old + changed_files.append( + ChangedFileInfo( + file_path=incoming_file.file_path, + old_file_id=existing_file.id, + ) + ) + else: + # New file - needs to be uploaded + new_files.append(incoming_file.file_path) + + # Find files to delete (exist in KB but not in incoming directory) + for filename, file in existing_by_filename.items(): + if filename not in incoming_filenames: + removed_file_ids.append(file.id) + + # Now persist hashes only for unchanged files that were calculated on-demand + # These files are confirmed to stay, so caching their hash is beneficial + for file_id, file_hash in files_needing_hash_persist: + try: + await Files.update_file_metadata_by_id(file_id, {'file_hash': file_hash}) + except Exception as e: + log.warning(f'Failed to persist hash for file {file_id}: {e}') + # Non-critical, continue + + return SyncCompareResponse( + new_files=new_files, + changed_files=changed_files, + removed_file_ids=removed_file_ids, + unchanged=unchanged, + ) + + ############################ # AddFilesToKnowledge ############################ diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 77261a6f2b..398d164236 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -1534,7 +1534,6 @@ async def process_file( ): """ Process a file and save its content to the vector database. - Process a file and save its content to the vector database. Note: granular session management is used to prevent connection pool exhaustion. The session is committed before external API calls, and updates use a fresh session. """ diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 787ef4d6e8..6c99f02905 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -557,6 +557,11 @@ def calculate_sha256(file_path, chunk_size): return sha256.hexdigest() +def calculate_sha256_bytes(data: bytes) -> str: + """Compute SHA-256 hash of raw bytes.""" + return hashlib.sha256(data).hexdigest() + + def calculate_sha256_string(string): # Create a new SHA-256 hash object sha256_hash = hashlib.sha256() diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index f314bae634..b568dabc6d 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -544,3 +544,99 @@ export const exportKnowledgeById = async (token: string, id: string) => { return res; }; + +export type FileSyncCompareItem = { + file_path: string; + file_hash: string; + size: number; +}; + +export type ChangedFileInfo = { + file_path: string; + old_file_id: string; +}; + +export type SyncCompareResponse = { + new_files: string[]; + changed_files: ChangedFileInfo[]; + removed_file_ids: string[]; + unchanged: string[]; +}; + +export const compareFilesForSync = async ( + token: string, + id: string, + files: FileSyncCompareItem[] +): Promise => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/compare`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ files }) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export type UploadAndReplaceResponse = { + new_file_id: string; + old_file_id: string; + filename: string; +}; + +export const uploadAndReplaceFile = async ( + token: string, + knowledgeId: string, + file: File, + oldFileId: string +): Promise => { + let error = null; + + const formData = new FormData(); + formData.append('file', file); + formData.append('old_file_id', oldFileId); + + const res = await fetch( + `${WEBUI_API_BASE_URL}/knowledge/${knowledgeId}/file/upload_and_replace`, + { + method: 'POST', + headers: { + authorization: `Bearer ${token}` + }, + body: formData + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index 64438e23fc..e0344f5aa6 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -32,7 +32,10 @@ updateFileFromKnowledgeById, updateKnowledgeById, updateKnowledgeAccessGrants, - searchKnowledgeFilesById + searchKnowledgeFilesById, + compareFilesForSync, + uploadAndReplaceFile, + type FileSyncCompareItem } from '$lib/apis/knowledge'; import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval'; @@ -339,21 +342,40 @@ } }; + // Upload directory handler - uses shared utility const uploadDirectoryHandler = async () => { - // Check if File System Access API is supported - const isFileSystemAccessSupported = 'showDirectoryPicker' in window; - - try { - if (isFileSystemAccessSupported) { - // Modern browsers (Chrome, Edge) implementation - await handleModernBrowserUpload(); - } else { - // Firefox fallback - await handleFirefoxUpload(); - } - } catch (error) { - handleUploadError(error); - } + try { + const files = await collectDirectoryFiles(); + + if (files.length === 0) { + toast.info($i18n.t('No files found in directory')); + return; + } + + const totalFiles = files.length; + let uploadedFiles = 0; + + const updateProgress = () => { + const percentage = (uploadedFiles / totalFiles) * 100; + toast.info( + $i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', { + uploadedFiles, + totalFiles, + percentage: percentage.toFixed(2) + }) + ); + }; + + updateProgress(); + + for (const { file } of files) { + await uploadFileHandler(file); + uploadedFiles++; + updateProgress(); + } + } catch (error) { + handleUploadError(error); + } }; // Helper function to check if a path contains hidden folders @@ -361,144 +383,116 @@ return path.split('/').some((part) => part.startsWith('.')); }; - // Modern browsers implementation using File System Access API - const handleModernBrowserUpload = async () => { - const dirHandle = await window.showDirectoryPicker(); - let totalFiles = 0; - let uploadedFiles = 0; - - // Function to update the UI with the progress - const updateProgress = () => { - const percentage = (uploadedFiles / totalFiles) * 100; - toast.info( - $i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', { - uploadedFiles: uploadedFiles, - totalFiles: totalFiles, - percentage: percentage.toFixed(2) - }) - ); - }; - - // Recursive function to count all files excluding hidden ones - async function countFiles(dirHandle) { - for await (const entry of dirHandle.values()) { - // Skip hidden files and directories - if (entry.name.startsWith('.')) continue; - - if (entry.kind === 'file') { - totalFiles++; - } else if (entry.kind === 'directory') { - // Only process non-hidden directories - if (!entry.name.startsWith('.')) { - await countFiles(entry); - } - } - } - } - - // Recursive function to process directories excluding hidden files and folders - async function processDirectory(dirHandle, path = '') { - for await (const entry of dirHandle.values()) { - // Skip hidden files and directories - if (entry.name.startsWith('.')) continue; - - const entryPath = path ? `${path}/${entry.name}` : entry.name; - - // Skip if the path contains any hidden folders - if (hasHiddenFolder(entryPath)) continue; - - if (entry.kind === 'file') { - const file = await entry.getFile(); - const fileWithPath = new File([file], entryPath, { type: file.type }); - - await uploadFileHandler(fileWithPath); - uploadedFiles++; - updateProgress(); - } else if (entry.kind === 'directory') { - // Only process non-hidden directories - if (!entry.name.startsWith('.')) { - await processDirectory(entry, entryPath); - } - } - } - } - - await countFiles(dirHandle); - updateProgress(); - - if (totalFiles > 0) { - await processDirectory(dirHandle); - } else { - console.log('No files to upload.'); - } + // Calculate SHA-256 hash of a file in the browser + const calculateFileHash = async (file: File): Promise => { + const buffer = await file.arrayBuffer(); + const hashBuffer = await crypto.subtle.digest('SHA-256', buffer); + return Array.from(new Uint8Array(hashBuffer)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); }; - - // Firefox fallback implementation using traditional file input - const handleFirefoxUpload = async () => { - return new Promise((resolve, reject) => { - // Create hidden file input - const input = document.createElement('input'); - input.type = 'file'; - input.webkitdirectory = true; - input.directory = true; - input.multiple = true; - input.style.display = 'none'; - - // Add input to DOM temporarily - document.body.appendChild(input); - - input.onchange = async () => { - try { - const files = Array.from(input.files) - // Filter out files from hidden folders - .filter((file) => !hasHiddenFolder(file.webkitRelativePath)); - - let totalFiles = files.length; - let uploadedFiles = 0; - - // Function to update the UI with the progress - const updateProgress = () => { - const percentage = (uploadedFiles / totalFiles) * 100; - toast.info( - $i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', { - uploadedFiles: uploadedFiles, - totalFiles: totalFiles, - percentage: percentage.toFixed(2) - }) - ); - }; - - updateProgress(); - - // Process all files - for (const file of files) { - // Skip hidden files (additional check) - if (!file.name.startsWith('.')) { - const relativePath = file.webkitRelativePath || file.name; - const fileWithPath = new File([file], relativePath, { type: file.type }); - - await uploadFileHandler(fileWithPath); - uploadedFiles++; - updateProgress(); - } - } - - // Clean up - document.body.removeChild(input); - resolve(); - } catch (error) { - reject(error); - } - }; - - input.onerror = (error) => { - document.body.removeChild(input); - reject(error); - }; - - // Trigger file picker - input.click(); - }); + + // Shared type for collected files + type CollectedFile = { + file: File; + path: string; + size: number; + hash?: string; + }; + + // Shared utility to collect all files from a directory + const collectDirectoryFiles = async (options?: { withHashes?: boolean }): Promise => { + const withHashes = options?.withHashes ?? false; + const files: CollectedFile[] = []; + + const isFileSystemAccessSupported = 'showDirectoryPicker' in window; + + if (isFileSystemAccessSupported) { + const dirHandle = await window.showDirectoryPicker(); + + async function processDirectory(dirHandle: FileSystemDirectoryHandle, path = '') { + for await (const entry of dirHandle.values()) { + if (entry.name.startsWith('.')) continue; + + const entryPath = path ? `${path}/${entry.name}` : entry.name; + + if (hasHiddenFolder(entryPath)) continue; + + if (entry.kind === 'file') { + const file = await (entry as FileSystemFileHandle).getFile(); + const fileWithPath = new File([file], entryPath, { type: file.type }); + + const collectedFile: CollectedFile = { + file: fileWithPath, + path: entryPath, + size: file.size + }; + + if (withHashes) { + collectedFile.hash = await calculateFileHash(file); + } + + files.push(collectedFile); + } else if (entry.kind === 'directory') { + await processDirectory(entry as FileSystemDirectoryHandle, entryPath); + } + } + } + + await processDirectory(dirHandle); + } else { + // Firefox fallback + await new Promise((resolve, reject) => { + const input = document.createElement('input'); + input.type = 'file'; + input.webkitdirectory = true; + input.directory = true; + input.multiple = true; + input.style.display = 'none'; + + document.body.appendChild(input); + + input.onchange = async () => { + try { + const inputFiles = Array.from(input.files || []).filter( + (file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.') + ); + + for (const file of inputFiles) { + const relativePath = file.webkitRelativePath || file.name; + const fileWithPath = new File([file], relativePath, { type: file.type }); + + const collectedFile: CollectedFile = { + file: fileWithPath, + path: relativePath, + size: file.size + }; + + if (withHashes) { + collectedFile.hash = await calculateFileHash(file); + } + + files.push(collectedFile); + } + + document.body.removeChild(input); + resolve(); + } catch (error) { + document.body.removeChild(input); + reject(error); + } + }; + + input.onerror = (error) => { + document.body.removeChild(input); + reject(error); + }; + + input.click(); + }); + } + + return files; }; // Error handler @@ -511,23 +505,108 @@ } }; - // Helper function to maintain file paths within zip + // Smart sync: only upload changed files, delete removed files const syncDirectoryHandler = async () => { - if (fileItems.length > 0) { - const res = await resetKnowledgeById(localStorage.token, id).catch((e) => { - toast.error(`${e}`); - }); - - if (res) { - fileItems = []; - toast.success($i18n.t('Knowledge reset successfully.')); - - // Upload directory - uploadDirectoryHandler(); - } - } else { - uploadDirectoryHandler(); - } + try { + toast.info($i18n.t('Scanning directory...')); + const directoryFiles = await collectDirectoryFiles({ withHashes: true }); + + if (directoryFiles.length === 0) { + toast.info($i18n.t('No files found in directory')); + return; + } + + toast.info( + $i18n.t('Found {{count}} files, comparing with knowledge base...', { + count: directoryFiles.length + }) + ); + + // Prepare comparison data + const compareData: FileSyncCompareItem[] = directoryFiles.map((f) => ({ + file_path: f.path, + file_hash: f.hash!, + size: f.size + })); + + // Call compare endpoint to get sync plan + const comparison = await compareFilesForSync(localStorage.token, id, compareData); + + if (!comparison) { + toast.error($i18n.t('Failed to compare files')); + return; + } + + const { new_files, changed_files, removed_file_ids, unchanged } = comparison; + + const totalToProcess = new_files.length + changed_files.length; + let processedCount = 0; + + // Upload new files (no old version to delete) + for (const filePath of new_files) { + const fileData = directoryFiles.find((f) => f.path === filePath); + if (fileData) { + await uploadFileHandler(fileData.file); + processedCount++; + toast.info( + $i18n.t('Uploading new: {{current}}/{{total}}', { + current: processedCount, + total: totalToProcess + }) + ); + } + } + + // Upload changed files using atomic upload_and_replace endpoint + for (const changedFile of changed_files) { + const fileData = directoryFiles.find((f) => f.path === changedFile.file_path); + if (fileData) { + await uploadAndReplaceFile( + localStorage.token, + id, + fileData.file, + changedFile.old_file_id + ); + processedCount++; + toast.info( + $i18n.t('Updating: {{current}}/{{total}}', { + current: processedCount, + total: totalToProcess + }) + ); + } + } + + // Delete removed files (files that no longer exist in directory) + if (removed_file_ids.length > 0) { + toast.info( + $i18n.t('Removing {{count}} deleted files...', { + count: removed_file_ids.length + }) + ); + for (const fileId of removed_file_ids) { + await removeFileFromKnowledgeById(localStorage.token, id, fileId); + } + } + + // Show summary + toast.success( + $i18n.t( + 'Sync complete: {{newCount}} new, {{changedCount}} updated, {{removedCount}} removed, {{unchangedCount}} unchanged', + { + newCount: new_files.length, + changedCount: changed_files.length, + removedCount: removed_file_ids.length, + unchangedCount: unchanged.length + } + ) + ); + + // Refresh the file list + await init(); + } catch (error) { + handleUploadError(error); + } }; const addFileHandler = async (fileId) => { @@ -784,7 +863,7 @@ { syncDirectoryHandler(); diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte index 1b5a8bafe1..da0a2ff301 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase/AddContentMenu.svelte @@ -76,7 +76,7 @@