mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-05 02:41:34 +00:00
feat: Smart sync for knowledge base directory uploads (#146)
Co-authored-by: Stoyan Zlatev <47269030+Stoyan-Zlatev@users.noreply.github.com>
This commit is contained in:
Classic298
DrMelone
parent
f6b85700ea
commit
6b2c927828
7 changed files with 667 additions and 173 deletions
|
|
@ -50,6 +50,8 @@ from open_webui.storage.provider import Storage
|
|||
|
||||
from open_webui.config import BYPASS_ADMIN_ACCESS_CONTROL
|
||||
from open_webui.utils.auth import get_admin_user, get_verified_user
|
||||
from open_webui.utils.access_control import has_access
|
||||
from open_webui.utils.misc import calculate_sha256_bytes
|
||||
from open_webui.utils.misc import strict_match_mime_type
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
|
@ -236,6 +238,9 @@ async def upload_file_handler(
|
|||
},
|
||||
)
|
||||
|
||||
# Calculate SHA-256 hash of the raw file bytes for sync comparison
|
||||
file_hash = calculate_sha256_bytes(contents)
|
||||
|
||||
file_item = await Files.insert_new_file(
|
||||
user.id,
|
||||
FileForm(
|
||||
|
|
@ -250,6 +255,7 @@ async def upload_file_handler(
|
|||
'name': name,
|
||||
'content_type': (file.content_type if isinstance(file.content_type, str) else None),
|
||||
'size': len(contents),
|
||||
'file_hash': file_hash,
|
||||
'data': file_metadata,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import List, Optional
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Request, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Request, Query, UploadFile, File, Form
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
import logging
|
||||
|
|
@ -27,10 +27,12 @@ from open_webui.routers.retrieval import (
|
|||
BatchProcessFilesForm,
|
||||
)
|
||||
from open_webui.storage.provider import Storage
|
||||
from open_webui.routers.files import upload_file_handler
|
||||
|
||||
from open_webui.constants import ERROR_MESSAGES
|
||||
from open_webui.utils.auth import get_verified_user, get_admin_user
|
||||
from open_webui.utils.access_control import has_permission, filter_allowed_access_grants
|
||||
from open_webui.utils.access_control import has_access, has_permission, filter_allowed_access_grants
|
||||
from open_webui.utils.misc import calculate_sha256
|
||||
from open_webui.models.access_grants import AccessGrants
|
||||
|
||||
|
||||
|
|
@ -687,6 +689,132 @@ async def add_file_to_knowledge_by_id(
|
|||
)
|
||||
|
||||
|
||||
############################
|
||||
# UploadAndReplaceFile
|
||||
############################
|
||||
|
||||
|
||||
class UploadAndReplaceResponse(BaseModel):
|
||||
"""Response from upload_and_replace endpoint."""
|
||||
|
||||
new_file_id: str
|
||||
old_file_id: str
|
||||
filename: str
|
||||
|
||||
|
||||
@router.post('/{id}/file/upload_and_replace', response_model=UploadAndReplaceResponse)
|
||||
async def upload_and_replace_file(
|
||||
request: Request,
|
||||
id: str,
|
||||
file: UploadFile = File(...),
|
||||
old_file_id: str = Form(...),
|
||||
user=Depends(get_verified_user),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
"""
|
||||
Atomically upload a new file and replace an existing file in the knowledge base.
|
||||
"""
|
||||
# Validate knowledge base exists and user has access
|
||||
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
|
||||
if not knowledge:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if (
|
||||
knowledge.user_id != user.id
|
||||
and not await AccessGrants.has_access(
|
||||
user_id=user.id,
|
||||
resource_type='knowledge',
|
||||
resource_id=knowledge.id,
|
||||
permission='write',
|
||||
db=db,
|
||||
)
|
||||
and user.role != 'admin'
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
|
||||
)
|
||||
|
||||
# Validate old file exists
|
||||
old_file = await Files.get_file_by_id(old_file_id, db=db)
|
||||
if not old_file:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
# Step 1: Upload the new file (reuses existing upload_file_handler)
|
||||
try:
|
||||
new_file_result = await upload_file_handler(
|
||||
request,
|
||||
file=file,
|
||||
process=True,
|
||||
process_in_background=False,
|
||||
user=user,
|
||||
db=db,
|
||||
)
|
||||
new_file_id = new_file_result['id']
|
||||
except Exception as e:
|
||||
log.error(f'Failed to upload new file: {e}')
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f'Failed to upload file: {str(e)}',
|
||||
)
|
||||
|
||||
# Step 2: Verify new file was processed
|
||||
new_file = await Files.get_file_by_id(new_file_id, db=db)
|
||||
if not new_file or not new_file.data:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.FILE_NOT_PROCESSED,
|
||||
)
|
||||
|
||||
# Step 3: Add new file to knowledge base (reuses existing process_file)
|
||||
try:
|
||||
process_file(
|
||||
request,
|
||||
ProcessFileForm(file_id=new_file_id, collection_name=id),
|
||||
user=user,
|
||||
db=db,
|
||||
)
|
||||
await Knowledges.add_file_to_knowledge_by_id(
|
||||
knowledge_id=id, file_id=new_file_id, user_id=user.id, db=db
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(f'Failed to add new file to knowledge base: {e}')
|
||||
# Clean up: delete the uploaded file since we couldn't add it to KB
|
||||
try:
|
||||
await Files.delete_file_by_id(new_file_id, db=db)
|
||||
except Exception:
|
||||
pass
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f'Failed to add file to knowledge base: {str(e)}',
|
||||
)
|
||||
|
||||
# Step 4: Remove old file (reuses existing remove_file_from_knowledge_by_id)
|
||||
try:
|
||||
await remove_file_from_knowledge_by_id(
|
||||
id=id,
|
||||
form_data=KnowledgeFileIdForm(file_id=old_file_id),
|
||||
delete_file=True,
|
||||
user=user,
|
||||
db=db,
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(f'Failed to remove old file (new file is already added): {e}')
|
||||
# Don't fail - new file is already added successfully
|
||||
|
||||
return UploadAndReplaceResponse(
|
||||
new_file_id=new_file_id,
|
||||
old_file_id=old_file_id,
|
||||
filename=new_file.filename,
|
||||
)
|
||||
|
||||
|
||||
@router.post('/{id}/file/update', response_model=Optional[KnowledgeFilesResponse])
|
||||
async def update_file_from_knowledge_by_id(
|
||||
request: Request,
|
||||
|
|
@ -966,6 +1094,187 @@ async def reset_knowledge_by_id(id: str, user=Depends(get_verified_user), db: As
|
|||
return knowledge
|
||||
|
||||
|
||||
############################
|
||||
# SyncCompare
|
||||
############################
|
||||
|
||||
|
||||
class FileSyncCompareItem(BaseModel):
|
||||
"""Item for comparing a file during sync."""
|
||||
|
||||
file_path: str # Relative path within the directory (e.g., "docs/readme.md")
|
||||
file_hash: str # SHA-256 hash of the raw file bytes
|
||||
size: int # File size in bytes
|
||||
|
||||
|
||||
class SyncCompareForm(BaseModel):
|
||||
"""Form for comparing files for sync."""
|
||||
|
||||
files: List[FileSyncCompareItem]
|
||||
|
||||
|
||||
class ChangedFileInfo(BaseModel):
|
||||
"""Info about a changed file that needs to be replaced."""
|
||||
|
||||
file_path: str # Path of the new file to upload
|
||||
old_file_id: str # ID of the old file to delete after upload
|
||||
|
||||
|
||||
class SyncCompareResponse(BaseModel):
|
||||
"""Response from sync compare endpoint."""
|
||||
|
||||
new_files: List[str] # file_paths for new files (no old version exists)
|
||||
changed_files: List[ChangedFileInfo] # files that changed (upload new, delete old)
|
||||
removed_file_ids: List[str] # file_ids to remove (no new version exists)
|
||||
unchanged: List[str] # file_paths that are already up to date
|
||||
|
||||
|
||||
async def get_file_hash(file: FileModel, persist: bool = False) -> Optional[str]:
|
||||
"""
|
||||
Get the file hash from meta, or calculate it on-demand from the stored file.
|
||||
This provides backwards compatibility for files uploaded before file_hash was stored.
|
||||
|
||||
Args:
|
||||
file: The file model to get hash for
|
||||
persist: If True and hash was calculated (not from meta), persist it to database.
|
||||
Only set to True when you know the file will NOT be replaced.
|
||||
"""
|
||||
# First check if file_hash is already stored in meta
|
||||
if file.meta and file.meta.get('file_hash'):
|
||||
return file.meta.get('file_hash')
|
||||
|
||||
# If not, calculate it from the stored file
|
||||
if not file.path:
|
||||
log.warning(f'File {file.id} has no path, cannot calculate hash')
|
||||
return None
|
||||
|
||||
try:
|
||||
# Get the local file path (downloads from cloud storage if needed)
|
||||
local_path = Storage.get_file(file.path)
|
||||
# Calculate hash with 8KB chunks
|
||||
file_hash = calculate_sha256(local_path, 8192)
|
||||
|
||||
# Only persist if explicitly requested (when file is confirmed unchanged)
|
||||
if persist:
|
||||
await Files.update_file_metadata_by_id(
|
||||
file.id,
|
||||
{'file_hash': file_hash},
|
||||
)
|
||||
|
||||
return file_hash
|
||||
except Exception as e:
|
||||
log.error(f'Failed to calculate hash for file {file.id}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
@router.post('/{id}/sync/compare', response_model=SyncCompareResponse)
|
||||
async def compare_files_for_sync(
|
||||
id: str,
|
||||
form_data: SyncCompareForm,
|
||||
user=Depends(get_verified_user),
|
||||
db: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
"""
|
||||
Compare uploaded files against existing knowledge base files.
|
||||
Returns lists of files that need to be uploaded, deleted, or are unchanged.
|
||||
"""
|
||||
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
|
||||
if not knowledge:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.NOT_FOUND,
|
||||
)
|
||||
|
||||
if (
|
||||
knowledge.user_id != user.id
|
||||
and not await AccessGrants.has_access(
|
||||
user_id=user.id,
|
||||
resource_type='knowledge',
|
||||
resource_id=knowledge.id,
|
||||
permission='write',
|
||||
db=db,
|
||||
)
|
||||
and user.role != 'admin'
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
|
||||
)
|
||||
|
||||
# Get all files currently in the knowledge base
|
||||
existing_files = await Knowledges.get_files_by_id(id, db=db)
|
||||
|
||||
# Build a map of existing files by filename for quick lookup
|
||||
existing_by_filename: dict[str, FileModel] = {}
|
||||
for file in existing_files:
|
||||
# Use the original filename from meta if available, otherwise use filename field
|
||||
filename = file.meta.get('name', file.filename) if file.meta else file.filename
|
||||
existing_by_filename[filename] = file
|
||||
|
||||
# Track files from the incoming directory
|
||||
incoming_filenames = set()
|
||||
new_files: List[str] = [] # New files (no old version)
|
||||
changed_files: List[ChangedFileInfo] = [] # Changed files (need upload + delete old)
|
||||
removed_file_ids: List[str] = [] # Removed files (no new version)
|
||||
unchanged: List[str] = []
|
||||
# Track files that need hash persisted (unchanged files without stored hash)
|
||||
files_needing_hash_persist: list[tuple[str, str]] = [] # [(file_id, hash), ...]
|
||||
|
||||
for incoming_file in form_data.files:
|
||||
incoming_filenames.add(incoming_file.file_path)
|
||||
|
||||
# Check if file exists in knowledge base
|
||||
existing_file = existing_by_filename.get(incoming_file.file_path)
|
||||
|
||||
if existing_file:
|
||||
# Check if hash was already stored in meta
|
||||
had_stored_hash = bool(
|
||||
existing_file.meta and existing_file.meta.get('file_hash')
|
||||
)
|
||||
|
||||
# File exists - check if it has changed (don't persist yet)
|
||||
existing_hash = await get_file_hash(existing_file, persist=False)
|
||||
|
||||
if existing_hash and existing_hash == incoming_file.file_hash:
|
||||
# File unchanged
|
||||
unchanged.append(incoming_file.file_path)
|
||||
# If hash was calculated on-demand (not from meta), queue for persistence
|
||||
if not had_stored_hash:
|
||||
files_needing_hash_persist.append((existing_file.id, existing_hash))
|
||||
else:
|
||||
# File changed or hash calculation failed - need to upload new and delete old
|
||||
changed_files.append(
|
||||
ChangedFileInfo(
|
||||
file_path=incoming_file.file_path,
|
||||
old_file_id=existing_file.id,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# New file - needs to be uploaded
|
||||
new_files.append(incoming_file.file_path)
|
||||
|
||||
# Find files to delete (exist in KB but not in incoming directory)
|
||||
for filename, file in existing_by_filename.items():
|
||||
if filename not in incoming_filenames:
|
||||
removed_file_ids.append(file.id)
|
||||
|
||||
# Now persist hashes only for unchanged files that were calculated on-demand
|
||||
# These files are confirmed to stay, so caching their hash is beneficial
|
||||
for file_id, file_hash in files_needing_hash_persist:
|
||||
try:
|
||||
await Files.update_file_metadata_by_id(file_id, {'file_hash': file_hash})
|
||||
except Exception as e:
|
||||
log.warning(f'Failed to persist hash for file {file_id}: {e}')
|
||||
# Non-critical, continue
|
||||
|
||||
return SyncCompareResponse(
|
||||
new_files=new_files,
|
||||
changed_files=changed_files,
|
||||
removed_file_ids=removed_file_ids,
|
||||
unchanged=unchanged,
|
||||
)
|
||||
|
||||
|
||||
############################
|
||||
# AddFilesToKnowledge
|
||||
############################
|
||||
|
|
|
|||
|
|
@ -1534,7 +1534,6 @@ async def process_file(
|
|||
):
|
||||
"""
|
||||
Process a file and save its content to the vector database.
|
||||
Process a file and save its content to the vector database.
|
||||
Note: granular session management is used to prevent connection pool exhaustion.
|
||||
The session is committed before external API calls, and updates use a fresh session.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -557,6 +557,11 @@ def calculate_sha256(file_path, chunk_size):
|
|||
return sha256.hexdigest()
|
||||
|
||||
|
||||
def calculate_sha256_bytes(data: bytes) -> str:
|
||||
"""Compute SHA-256 hash of raw bytes."""
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def calculate_sha256_string(string):
|
||||
# Create a new SHA-256 hash object
|
||||
sha256_hash = hashlib.sha256()
|
||||
|
|
|
|||
|
|
@ -544,3 +544,99 @@ export const exportKnowledgeById = async (token: string, id: string) => {
|
|||
|
||||
return res;
|
||||
};
|
||||
|
||||
export type FileSyncCompareItem = {
|
||||
file_path: string;
|
||||
file_hash: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export type ChangedFileInfo = {
|
||||
file_path: string;
|
||||
old_file_id: string;
|
||||
};
|
||||
|
||||
export type SyncCompareResponse = {
|
||||
new_files: string[];
|
||||
changed_files: ChangedFileInfo[];
|
||||
removed_file_ids: string[];
|
||||
unchanged: string[];
|
||||
};
|
||||
|
||||
export const compareFilesForSync = async (
|
||||
token: string,
|
||||
id: string,
|
||||
files: FileSyncCompareItem[]
|
||||
): Promise<SyncCompareResponse> => {
|
||||
let error = null;
|
||||
|
||||
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/compare`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
authorization: `Bearer ${token}`
|
||||
},
|
||||
body: JSON.stringify({ files })
|
||||
})
|
||||
.then(async (res) => {
|
||||
if (!res.ok) throw await res.json();
|
||||
return res.json();
|
||||
})
|
||||
.catch((err) => {
|
||||
error = err.detail;
|
||||
console.error(err);
|
||||
return null;
|
||||
});
|
||||
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
export type UploadAndReplaceResponse = {
|
||||
new_file_id: string;
|
||||
old_file_id: string;
|
||||
filename: string;
|
||||
};
|
||||
|
||||
export const uploadAndReplaceFile = async (
|
||||
token: string,
|
||||
knowledgeId: string,
|
||||
file: File,
|
||||
oldFileId: string
|
||||
): Promise<UploadAndReplaceResponse> => {
|
||||
let error = null;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
formData.append('old_file_id', oldFileId);
|
||||
|
||||
const res = await fetch(
|
||||
`${WEBUI_API_BASE_URL}/knowledge/${knowledgeId}/file/upload_and_replace`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`
|
||||
},
|
||||
body: formData
|
||||
}
|
||||
)
|
||||
.then(async (res) => {
|
||||
if (!res.ok) throw await res.json();
|
||||
return res.json();
|
||||
})
|
||||
.catch((err) => {
|
||||
error = err.detail;
|
||||
console.error(err);
|
||||
return null;
|
||||
});
|
||||
|
||||
if (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
return res;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -32,7 +32,10 @@
|
|||
updateFileFromKnowledgeById,
|
||||
updateKnowledgeById,
|
||||
updateKnowledgeAccessGrants,
|
||||
searchKnowledgeFilesById
|
||||
searchKnowledgeFilesById,
|
||||
compareFilesForSync,
|
||||
uploadAndReplaceFile,
|
||||
type FileSyncCompareItem
|
||||
} from '$lib/apis/knowledge';
|
||||
import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
|
||||
|
||||
|
|
@ -339,21 +342,40 @@
|
|||
}
|
||||
};
|
||||
|
||||
// Upload directory handler - uses shared utility
|
||||
const uploadDirectoryHandler = async () => {
|
||||
// Check if File System Access API is supported
|
||||
const isFileSystemAccessSupported = 'showDirectoryPicker' in window;
|
||||
|
||||
try {
|
||||
if (isFileSystemAccessSupported) {
|
||||
// Modern browsers (Chrome, Edge) implementation
|
||||
await handleModernBrowserUpload();
|
||||
} else {
|
||||
// Firefox fallback
|
||||
await handleFirefoxUpload();
|
||||
}
|
||||
} catch (error) {
|
||||
handleUploadError(error);
|
||||
}
|
||||
try {
|
||||
const files = await collectDirectoryFiles();
|
||||
|
||||
if (files.length === 0) {
|
||||
toast.info($i18n.t('No files found in directory'));
|
||||
return;
|
||||
}
|
||||
|
||||
const totalFiles = files.length;
|
||||
let uploadedFiles = 0;
|
||||
|
||||
const updateProgress = () => {
|
||||
const percentage = (uploadedFiles / totalFiles) * 100;
|
||||
toast.info(
|
||||
$i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', {
|
||||
uploadedFiles,
|
||||
totalFiles,
|
||||
percentage: percentage.toFixed(2)
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
updateProgress();
|
||||
|
||||
for (const { file } of files) {
|
||||
await uploadFileHandler(file);
|
||||
uploadedFiles++;
|
||||
updateProgress();
|
||||
}
|
||||
} catch (error) {
|
||||
handleUploadError(error);
|
||||
}
|
||||
};
|
||||
|
||||
// Helper function to check if a path contains hidden folders
|
||||
|
|
@ -361,144 +383,116 @@
|
|||
return path.split('/').some((part) => part.startsWith('.'));
|
||||
};
|
||||
|
||||
// Modern browsers implementation using File System Access API
|
||||
const handleModernBrowserUpload = async () => {
|
||||
const dirHandle = await window.showDirectoryPicker();
|
||||
let totalFiles = 0;
|
||||
let uploadedFiles = 0;
|
||||
|
||||
// Function to update the UI with the progress
|
||||
const updateProgress = () => {
|
||||
const percentage = (uploadedFiles / totalFiles) * 100;
|
||||
toast.info(
|
||||
$i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', {
|
||||
uploadedFiles: uploadedFiles,
|
||||
totalFiles: totalFiles,
|
||||
percentage: percentage.toFixed(2)
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
// Recursive function to count all files excluding hidden ones
|
||||
async function countFiles(dirHandle) {
|
||||
for await (const entry of dirHandle.values()) {
|
||||
// Skip hidden files and directories
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
|
||||
if (entry.kind === 'file') {
|
||||
totalFiles++;
|
||||
} else if (entry.kind === 'directory') {
|
||||
// Only process non-hidden directories
|
||||
if (!entry.name.startsWith('.')) {
|
||||
await countFiles(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Recursive function to process directories excluding hidden files and folders
|
||||
async function processDirectory(dirHandle, path = '') {
|
||||
for await (const entry of dirHandle.values()) {
|
||||
// Skip hidden files and directories
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
|
||||
const entryPath = path ? `${path}/${entry.name}` : entry.name;
|
||||
|
||||
// Skip if the path contains any hidden folders
|
||||
if (hasHiddenFolder(entryPath)) continue;
|
||||
|
||||
if (entry.kind === 'file') {
|
||||
const file = await entry.getFile();
|
||||
const fileWithPath = new File([file], entryPath, { type: file.type });
|
||||
|
||||
await uploadFileHandler(fileWithPath);
|
||||
uploadedFiles++;
|
||||
updateProgress();
|
||||
} else if (entry.kind === 'directory') {
|
||||
// Only process non-hidden directories
|
||||
if (!entry.name.startsWith('.')) {
|
||||
await processDirectory(entry, entryPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await countFiles(dirHandle);
|
||||
updateProgress();
|
||||
|
||||
if (totalFiles > 0) {
|
||||
await processDirectory(dirHandle);
|
||||
} else {
|
||||
console.log('No files to upload.');
|
||||
}
|
||||
// Calculate SHA-256 hash of a file in the browser
|
||||
const calculateFileHash = async (file: File): Promise<string> => {
|
||||
const buffer = await file.arrayBuffer();
|
||||
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
|
||||
return Array.from(new Uint8Array(hashBuffer))
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
};
|
||||
|
||||
// Firefox fallback implementation using traditional file input
|
||||
const handleFirefoxUpload = async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Create hidden file input
|
||||
const input = document.createElement('input');
|
||||
input.type = 'file';
|
||||
input.webkitdirectory = true;
|
||||
input.directory = true;
|
||||
input.multiple = true;
|
||||
input.style.display = 'none';
|
||||
|
||||
// Add input to DOM temporarily
|
||||
document.body.appendChild(input);
|
||||
|
||||
input.onchange = async () => {
|
||||
try {
|
||||
const files = Array.from(input.files)
|
||||
// Filter out files from hidden folders
|
||||
.filter((file) => !hasHiddenFolder(file.webkitRelativePath));
|
||||
|
||||
let totalFiles = files.length;
|
||||
let uploadedFiles = 0;
|
||||
|
||||
// Function to update the UI with the progress
|
||||
const updateProgress = () => {
|
||||
const percentage = (uploadedFiles / totalFiles) * 100;
|
||||
toast.info(
|
||||
$i18n.t('Upload Progress: {{uploadedFiles}}/{{totalFiles}} ({{percentage}}%)', {
|
||||
uploadedFiles: uploadedFiles,
|
||||
totalFiles: totalFiles,
|
||||
percentage: percentage.toFixed(2)
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
updateProgress();
|
||||
|
||||
// Process all files
|
||||
for (const file of files) {
|
||||
// Skip hidden files (additional check)
|
||||
if (!file.name.startsWith('.')) {
|
||||
const relativePath = file.webkitRelativePath || file.name;
|
||||
const fileWithPath = new File([file], relativePath, { type: file.type });
|
||||
|
||||
await uploadFileHandler(fileWithPath);
|
||||
uploadedFiles++;
|
||||
updateProgress();
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
document.body.removeChild(input);
|
||||
resolve();
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
|
||||
input.onerror = (error) => {
|
||||
document.body.removeChild(input);
|
||||
reject(error);
|
||||
};
|
||||
|
||||
// Trigger file picker
|
||||
input.click();
|
||||
});
|
||||
|
||||
// Shared type for collected files
|
||||
type CollectedFile = {
|
||||
file: File;
|
||||
path: string;
|
||||
size: number;
|
||||
hash?: string;
|
||||
};
|
||||
|
||||
// Shared utility to collect all files from a directory
|
||||
const collectDirectoryFiles = async (options?: { withHashes?: boolean }): Promise<CollectedFile[]> => {
|
||||
const withHashes = options?.withHashes ?? false;
|
||||
const files: CollectedFile[] = [];
|
||||
|
||||
const isFileSystemAccessSupported = 'showDirectoryPicker' in window;
|
||||
|
||||
if (isFileSystemAccessSupported) {
|
||||
const dirHandle = await window.showDirectoryPicker();
|
||||
|
||||
async function processDirectory(dirHandle: FileSystemDirectoryHandle, path = '') {
|
||||
for await (const entry of dirHandle.values()) {
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
|
||||
const entryPath = path ? `${path}/${entry.name}` : entry.name;
|
||||
|
||||
if (hasHiddenFolder(entryPath)) continue;
|
||||
|
||||
if (entry.kind === 'file') {
|
||||
const file = await (entry as FileSystemFileHandle).getFile();
|
||||
const fileWithPath = new File([file], entryPath, { type: file.type });
|
||||
|
||||
const collectedFile: CollectedFile = {
|
||||
file: fileWithPath,
|
||||
path: entryPath,
|
||||
size: file.size
|
||||
};
|
||||
|
||||
if (withHashes) {
|
||||
collectedFile.hash = await calculateFileHash(file);
|
||||
}
|
||||
|
||||
files.push(collectedFile);
|
||||
} else if (entry.kind === 'directory') {
|
||||
await processDirectory(entry as FileSystemDirectoryHandle, entryPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await processDirectory(dirHandle);
|
||||
} else {
|
||||
// Firefox fallback
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const input = document.createElement('input');
|
||||
input.type = 'file';
|
||||
input.webkitdirectory = true;
|
||||
input.directory = true;
|
||||
input.multiple = true;
|
||||
input.style.display = 'none';
|
||||
|
||||
document.body.appendChild(input);
|
||||
|
||||
input.onchange = async () => {
|
||||
try {
|
||||
const inputFiles = Array.from(input.files || []).filter(
|
||||
(file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.')
|
||||
);
|
||||
|
||||
for (const file of inputFiles) {
|
||||
const relativePath = file.webkitRelativePath || file.name;
|
||||
const fileWithPath = new File([file], relativePath, { type: file.type });
|
||||
|
||||
const collectedFile: CollectedFile = {
|
||||
file: fileWithPath,
|
||||
path: relativePath,
|
||||
size: file.size
|
||||
};
|
||||
|
||||
if (withHashes) {
|
||||
collectedFile.hash = await calculateFileHash(file);
|
||||
}
|
||||
|
||||
files.push(collectedFile);
|
||||
}
|
||||
|
||||
document.body.removeChild(input);
|
||||
resolve();
|
||||
} catch (error) {
|
||||
document.body.removeChild(input);
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
|
||||
input.onerror = (error) => {
|
||||
document.body.removeChild(input);
|
||||
reject(error);
|
||||
};
|
||||
|
||||
input.click();
|
||||
});
|
||||
}
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
// Error handler
|
||||
|
|
@ -511,23 +505,108 @@
|
|||
}
|
||||
};
|
||||
|
||||
// Helper function to maintain file paths within zip
|
||||
// Smart sync: only upload changed files, delete removed files
|
||||
const syncDirectoryHandler = async () => {
|
||||
if (fileItems.length > 0) {
|
||||
const res = await resetKnowledgeById(localStorage.token, id).catch((e) => {
|
||||
toast.error(`${e}`);
|
||||
});
|
||||
|
||||
if (res) {
|
||||
fileItems = [];
|
||||
toast.success($i18n.t('Knowledge reset successfully.'));
|
||||
|
||||
// Upload directory
|
||||
uploadDirectoryHandler();
|
||||
}
|
||||
} else {
|
||||
uploadDirectoryHandler();
|
||||
}
|
||||
try {
|
||||
toast.info($i18n.t('Scanning directory...'));
|
||||
const directoryFiles = await collectDirectoryFiles({ withHashes: true });
|
||||
|
||||
if (directoryFiles.length === 0) {
|
||||
toast.info($i18n.t('No files found in directory'));
|
||||
return;
|
||||
}
|
||||
|
||||
toast.info(
|
||||
$i18n.t('Found {{count}} files, comparing with knowledge base...', {
|
||||
count: directoryFiles.length
|
||||
})
|
||||
);
|
||||
|
||||
// Prepare comparison data
|
||||
const compareData: FileSyncCompareItem[] = directoryFiles.map((f) => ({
|
||||
file_path: f.path,
|
||||
file_hash: f.hash!,
|
||||
size: f.size
|
||||
}));
|
||||
|
||||
// Call compare endpoint to get sync plan
|
||||
const comparison = await compareFilesForSync(localStorage.token, id, compareData);
|
||||
|
||||
if (!comparison) {
|
||||
toast.error($i18n.t('Failed to compare files'));
|
||||
return;
|
||||
}
|
||||
|
||||
const { new_files, changed_files, removed_file_ids, unchanged } = comparison;
|
||||
|
||||
const totalToProcess = new_files.length + changed_files.length;
|
||||
let processedCount = 0;
|
||||
|
||||
// Upload new files (no old version to delete)
|
||||
for (const filePath of new_files) {
|
||||
const fileData = directoryFiles.find((f) => f.path === filePath);
|
||||
if (fileData) {
|
||||
await uploadFileHandler(fileData.file);
|
||||
processedCount++;
|
||||
toast.info(
|
||||
$i18n.t('Uploading new: {{current}}/{{total}}', {
|
||||
current: processedCount,
|
||||
total: totalToProcess
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Upload changed files using atomic upload_and_replace endpoint
|
||||
for (const changedFile of changed_files) {
|
||||
const fileData = directoryFiles.find((f) => f.path === changedFile.file_path);
|
||||
if (fileData) {
|
||||
await uploadAndReplaceFile(
|
||||
localStorage.token,
|
||||
id,
|
||||
fileData.file,
|
||||
changedFile.old_file_id
|
||||
);
|
||||
processedCount++;
|
||||
toast.info(
|
||||
$i18n.t('Updating: {{current}}/{{total}}', {
|
||||
current: processedCount,
|
||||
total: totalToProcess
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete removed files (files that no longer exist in directory)
|
||||
if (removed_file_ids.length > 0) {
|
||||
toast.info(
|
||||
$i18n.t('Removing {{count}} deleted files...', {
|
||||
count: removed_file_ids.length
|
||||
})
|
||||
);
|
||||
for (const fileId of removed_file_ids) {
|
||||
await removeFileFromKnowledgeById(localStorage.token, id, fileId);
|
||||
}
|
||||
}
|
||||
|
||||
// Show summary
|
||||
toast.success(
|
||||
$i18n.t(
|
||||
'Sync complete: {{newCount}} new, {{changedCount}} updated, {{removedCount}} removed, {{unchangedCount}} unchanged',
|
||||
{
|
||||
newCount: new_files.length,
|
||||
changedCount: changed_files.length,
|
||||
removedCount: removed_file_ids.length,
|
||||
unchangedCount: unchanged.length
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
// Refresh the file list
|
||||
await init();
|
||||
} catch (error) {
|
||||
handleUploadError(error);
|
||||
}
|
||||
};
|
||||
|
||||
const addFileHandler = async (fileId) => {
|
||||
|
|
@ -784,7 +863,7 @@
|
|||
<SyncConfirmDialog
|
||||
bind:show={showSyncConfirmModal}
|
||||
message={$i18n.t(
|
||||
'This will reset the knowledge base and sync all files. Do you wish to continue?'
|
||||
'This will sync the knowledge base with the selected directory. New and changed files will be uploaded, and files removed from the directory will be deleted from the knowledge base. Continue?'
|
||||
)}
|
||||
on:confirm={() => {
|
||||
syncDirectoryHandler();
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@
|
|||
|
||||
<Tooltip
|
||||
content={$i18n.t(
|
||||
'This option will delete all existing files in the collection and replace them with newly uploaded files.'
|
||||
'Mirrors the local directory to the knowledge base. New or changed files will be uploaded, while files that no longer exist in the local directory will be removed. Unchanged files are skipped.'
|
||||
)}
|
||||
className="w-full"
|
||||
>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue