From 7618df3f53a8db2ad5352a789c88673a736b07a4 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Fri, 30 Oct 2026 07:08:28 +0900 Subject: [PATCH] refac --- backend/open_webui/config.py | 8 + backend/open_webui/models/knowledge.py | 68 +++++++- backend/open_webui/routers/files.py | 45 +++++- backend/open_webui/routers/knowledge.py | 150 +++++++++++++++--- backend/open_webui/routers/retrieval.py | 33 +++- src/lib/apis/knowledge/index.ts | 34 +++- .../admin/Settings/Documents.svelte | 33 ++++ src/lib/components/workspace/Knowledge.svelte | 4 +- .../workspace/Knowledge/KnowledgeBase.svelte | 137 +++++++++++++++- ...edgeBase.svelte => KnowledgeEditor.svelte} | 142 +++++++++++++---- src/lib/i18n/locales/en-US/translation.json | 13 +- 11 files changed, 588 insertions(+), 79 deletions(-) rename src/lib/components/workspace/Knowledge/{CreateKnowledgeBase.svelte => KnowledgeEditor.svelte} (52%) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index fc9c5543b7..baf1ea4594 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -996,6 +996,12 @@ ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = ( RAG_FULL_CONTEXT = os.getenv('RAG_FULL_CONTEXT', 'False').lower() == 'true' RAG_FILE_MAX_COUNT = int(os.getenv('RAG_FILE_MAX_COUNT')) if os.getenv('RAG_FILE_MAX_COUNT') else None +KNOWLEDGE_MAX_FILE_COUNT = int(os.getenv('KNOWLEDGE_MAX_FILE_COUNT')) if os.getenv('KNOWLEDGE_MAX_FILE_COUNT') else None +KNOWLEDGE_MAX_FILE_SIZE = int(os.getenv('KNOWLEDGE_MAX_FILE_SIZE')) if os.getenv('KNOWLEDGE_MAX_FILE_SIZE') else None +if KNOWLEDGE_MAX_FILE_COUNT is not None and KNOWLEDGE_MAX_FILE_COUNT <= 0: + raise ValueError('KNOWLEDGE_MAX_FILE_COUNT must be a positive integer or unset') +if KNOWLEDGE_MAX_FILE_SIZE is not None and KNOWLEDGE_MAX_FILE_SIZE <= 0: + raise ValueError('KNOWLEDGE_MAX_FILE_SIZE must be a positive integer or unset') RAG_FILE_MAX_SIZE = int(os.getenv('RAG_FILE_MAX_SIZE')) if os.getenv('RAG_FILE_MAX_SIZE') else None @@ -3013,6 +3019,8 @@ DEFAULT_CONFIG = { 'rag.enable_hybrid_search_enriched_texts': ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS, 'rag.full_context': RAG_FULL_CONTEXT, 'rag.file.max_count': RAG_FILE_MAX_COUNT, + 'knowledge.max_file_count': KNOWLEDGE_MAX_FILE_COUNT, + 'knowledge.max_file_size': KNOWLEDGE_MAX_FILE_SIZE, 'rag.file.max_size': RAG_FILE_MAX_SIZE, 'file.image_compression_width': FILE_IMAGE_COMPRESSION_WIDTH, 'file.image_compression_height': FILE_IMAGE_COMPRESSION_HEIGHT, diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index b053344bc9..8e52cea70d 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -6,6 +6,7 @@ from typing import Optional from open_webui.config import RAG_FILE_CONTENT_SEARCH_MAX_CHARS from open_webui.internal.db import Base, JSONField, get_async_db_context from open_webui.models.access_grants import AccessGrantModel, AccessGrants +from open_webui.models.config import Config from open_webui.models.files import ( File, FileMetadataResponse, @@ -162,10 +163,16 @@ class KnowledgeUserResponse(KnowledgeUserModel): pass +class KnowledgeMeta(BaseModel): + max_file_count: int | None = Field(default=None, gt=0) + max_file_size: int | None = Field(default=None, gt=0) + + class KnowledgeForm(BaseModel): name: str description: str access_grants: Optional[list[dict]] = None + meta: KnowledgeMeta | None = None class FileUserResponse(FileModelResponse): @@ -209,7 +216,8 @@ class KnowledgeTable: async with get_async_db_context(db) as db: knowledge = KnowledgeModel( **{ - **form_data.model_dump(exclude={'access_grants'}), + **form_data.model_dump(exclude={'access_grants', 'meta'}), + 'meta': form_data.meta.model_dump() if form_data.meta else None, 'id': str(uuid.uuid4()), 'user_id': user_id, 'created_at': int(time.time()), @@ -671,6 +679,37 @@ class KnowledgeTable: except Exception: return [] + async def get_file_limits(self, meta: dict | None) -> dict[str, int | None]: + local = meta or {} + counts = [ + int(value) for value in (await Config.get('knowledge.max_file_count'), local.get('max_file_count')) if value + ] + sizes = [ + int(value) for value in (await Config.get('knowledge.max_file_size'), local.get('max_file_size')) if value + ] + return {'max_file_count': min(counts, default=None), 'max_file_size': min(sizes, default=None)} + + async def get_file_count_by_id( + self, + knowledge_id: str, + include_pending: bool = False, + exclude_file_ids: list[str] | None = None, + db: Optional[AsyncSession] = None, + ) -> int: + async with get_async_db_context(db) as db: + linked = select(KnowledgeFile.file_id.label('file_id')).where(KnowledgeFile.knowledge_id == knowledge_id) + if include_pending: + pending = select(File.id.label('file_id')).where( + File.meta['data']['knowledge_id'].as_string() == knowledge_id, + File.data['status'].as_string().in_(['pending', 'processing']), + ) + linked = linked.union(pending) + files = linked.subquery() + stmt = select(func.count()).select_from(files) + if exclude_file_ids: + stmt = stmt.where(files.c.file_id.notin_(exclude_file_ids)) + return (await db.execute(stmt)).scalar_one() + async def add_file_to_knowledge_by_id( self, knowledge_id: str, @@ -679,7 +718,27 @@ class KnowledgeTable: directory_id: Optional[str] = None, db: Optional[AsyncSession] = None, ) -> Optional[KnowledgeFileModel]: + global_limit = await Config.get('knowledge.max_file_count') async with get_async_db_context(db) as db: + # Read the local limit under the same write lock as count + insert. + await db.execute( + update(Knowledge).where(Knowledge.id == knowledge_id).values(updated_at=Knowledge.updated_at) + ) + meta = await db.scalar(select(Knowledge.meta).where(Knowledge.id == knowledge_id)) + local_limit = (meta or {}).get('max_file_count') + limit = min((int(value) for value in (global_limit, local_limit) if value), default=None) + if limit is not None: + existing = await db.scalar(select(KnowledgeFile).filter_by(knowledge_id=knowledge_id, file_id=file_id)) + if existing: + await db.commit() + # Preserve duplicate-insert semantics; callers must not release another request's slot. + return None + count = await db.scalar( + select(func.count()).select_from(KnowledgeFile).where(KnowledgeFile.knowledge_id == knowledge_id) + ) + if count >= int(limit): + await db.rollback() + raise ValueError(f'Knowledge base file limit exceeded ({limit} files maximum).') knowledge_file = KnowledgeFileModel( **{ 'id': str(uuid.uuid4()), @@ -702,6 +761,7 @@ class KnowledgeTable: else: return None except Exception: + await db.rollback() return None async def has_file(self, knowledge_id: str, file_id: str, db: Optional[AsyncSession] = None) -> bool: @@ -758,11 +818,15 @@ class KnowledgeTable: ) -> Optional[KnowledgeModel]: try: async with get_async_db_context(db) as db: + values = form_data.model_dump(exclude={'access_grants', 'meta'}) + if form_data.meta is not None: + meta = await db.scalar(select(Knowledge.meta).where(Knowledge.id == id)) or {} + values['meta'] = {**meta, **form_data.meta.model_dump(exclude_unset=True)} await db.execute( update(Knowledge) .filter_by(id=id) .values( - **form_data.model_dump(exclude={'access_grants'}), + **values, updated_at=int(time.time()), ) ) diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index 1f2292e753..f583b38880 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -41,6 +41,7 @@ from open_webui.models.knowledge import Knowledges from open_webui.models.users import Users from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.routers.audio import transcribe +from open_webui.routers.knowledge import check_knowledge_upload_limits from open_webui.routers.retrieval import ProcessFileForm, index_file, process_file from open_webui.storage.provider import Storage from open_webui.utils.auth import get_admin_user, get_verified_user @@ -136,6 +137,14 @@ async def process_uploaded_file( ): async def _process_handler(db_session): try: + if file_metadata.get('knowledge_id'): + await check_knowledge_upload_limits( + file_metadata['knowledge_id'], + 1, + file_size=(file_item.meta or {}).get('size'), + exclude_file_ids=[file_item.id], + db=db_session, + ) await Files.update_file_data_by_id(file_item.id, {'status': 'processing'}, db=db_session) content_type = file.content_type @@ -229,12 +238,6 @@ async def process_uploaded_file( ) directory_id = None - await index_file( - request, - ProcessFileForm(file_id=file_item.id, collection_name=knowledge_id), - user=user, - db=db_session, - ) knowledge_file = await Knowledges.add_file_to_knowledge_by_id( knowledge_id=knowledge_id, file_id=file_item.id, @@ -244,6 +247,16 @@ async def process_uploaded_file( ) if not knowledge_file: raise Exception(f'Failed to link file {file_item.id} to knowledge {knowledge_id}') + try: + await index_file( + request, + ProcessFileForm(file_id=file_item.id, collection_name=knowledge_id), + user=user, + db=db_session, + ) + except Exception: + await Knowledges.remove_file_from_knowledge_by_id(knowledge_id, file_item.id, db=db_session) + raise log.info('Linked file %s to knowledge %s', file_item.id, knowledge_id) except Exception as e: log.warning(f'Failed to link file {file_item.id} to knowledge {knowledge_id}: {e}') @@ -357,6 +370,19 @@ async def upload_file_handler( detail=ERROR_MESSAGES.DEFAULT(f'File type {file_extension} is not allowed'), ) + knowledge_id = file_metadata.get('knowledge_id') + if knowledge_id: + knowledge = await Knowledges.get_knowledge_by_id(knowledge_id, db=db) + if not knowledge or not ( + knowledge.user_id == user.id + or user.role == 'admin' + or await AccessGrants.has_access( + user_id=user.id, resource_type='knowledge', resource_id=knowledge_id, permission='write', db=db + ) + ): + raise HTTPException(403, ERROR_MESSAGES.ACCESS_PROHIBITED) + await check_knowledge_upload_limits(knowledge_id, 1, batch=True, file_size=file.size, db=db) + # Prefer readable storage names for admins, but fall back if the filesystem rejects it. id = str(uuid.uuid4()) name = filename @@ -395,6 +421,13 @@ async def upload_file_handler( detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB'), ) + if knowledge_id: + try: + await check_knowledge_upload_limits(knowledge_id, 0, file_size=len(contents), db=db) + except HTTPException: + await asyncio.to_thread(Storage.delete_file, file_path) + raise + # SHA-256 of raw uploaded bytes for incremental sync diffing. # If the client pre-computed and sent file_hash, use that. file_hash = file_metadata.get('file_hash') or await asyncio.to_thread( diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 6b13b798cd..7b6e88feec 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -46,7 +46,7 @@ from open_webui.utils.access_control import filter_allowed_access_grants, has_pe from open_webui.utils.access_control.files import has_access_to_file from open_webui.utils.auth import get_admin_user, get_verified_user from open_webui.utils.json_codec import JSONCodec -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession log = logging.getLogger(__name__) @@ -1169,6 +1169,12 @@ async def update_knowledge_by_id( detail=ERROR_MESSAGES.ACCESS_PROHIBITED, ) + if form_data.meta is not None: + if knowledge.user_id != user.id and user.role != 'admin': + raise HTTPException(403, ERROR_MESSAGES.ACCESS_PROHIBITED) + if is_external_knowledge(knowledge): + external_knowledge_error() + knowledge = await Knowledges.update_knowledge_by_id(id=id, form_data=form_data) if knowledge: # Re-embed knowledge base for semantic search @@ -1402,6 +1408,65 @@ async def get_knowledge_files_by_id( ############################ +async def check_knowledge_upload_limits( + knowledge_id: str, + count: int, + *, + batch: bool = False, + file_size: int | None = None, + exclude_file_ids: list[str] | None = None, + db=None, +) -> None: + if batch: + batch_limit = await Config.get('rag.file.max_count') + if batch_limit is not None and count > int(batch_limit): + raise HTTPException(400, f'You can only upload a maximum of {batch_limit} file(s) per batch.') + + knowledge = await Knowledges.get_knowledge_by_id(knowledge_id, db=db) + if not knowledge: + raise HTTPException(404, ERROR_MESSAGES.NOT_FOUND) + limits = await Knowledges.get_file_limits(knowledge.meta) + + if file_size is not None: + for max_size in (await Config.get('rag.file.max_size'), limits['max_file_size']): + if max_size and file_size > int(max_size) * 1024 * 1024: + raise HTTPException(413, ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB')) + + limit = limits['max_file_count'] + if limit is not None and count > 0: + current_count = await Knowledges.get_file_count_by_id( + knowledge_id, include_pending=True, exclude_file_ids=exclude_file_ids, db=db + ) + if current_count + count > int(limit): + raise HTTPException(400, f'Knowledge base file limit exceeded ({limit} files maximum).') + + +class KnowledgeUploadValidationForm(BaseModel): + count: int = Field(ge=0) + file_size: int | None = Field(default=None, ge=0) + removed_file_ids: list[str] = Field(default_factory=list) + + +@router.post('/{id}/files/validate') +async def validate_knowledge_upload( + id: str, + form_data: KnowledgeUploadValidationForm, + user=Depends(get_verified_user), + db: AsyncSession = Depends(get_async_session), +): + await _verify_knowledge_write_access(id, user, db) + # This is a preflight only. Actual additions recheck capacity without removal credits. + await check_knowledge_upload_limits( + id, + form_data.count, + batch=True, + file_size=form_data.file_size, + exclude_file_ids=form_data.removed_file_ids, + db=db, + ) + return {'status': True} + + class KnowledgeFileIdForm(BaseModel): file_id: str directory_id: Optional[str] = None @@ -1462,24 +1527,34 @@ async def add_file_to_knowledge_by_id( detail=ERROR_MESSAGES.ACCESS_PROHIBITED, ) - # Add content to the vector database + already_linked = await Knowledges.has_file(knowledge_id=id, file_id=form_data.file_id, db=db) + if not already_linked: + await check_knowledge_upload_limits( + id, 1, batch=True, file_size=(file.meta or {}).get('size', 0), exclude_file_ids=[form_data.file_id], db=db + ) + + linked = None try: + # Claim capacity atomically before indexing into the knowledge collection. + if not already_linked: + linked = await Knowledges.add_file_to_knowledge_by_id( + knowledge_id=id, + file_id=form_data.file_id, + user_id=user.id, + directory_id=form_data.directory_id, + db=db, + ) + if not linked: + raise ValueError(f'Failed to link file {form_data.file_id} to knowledge {id}') await process_file( request, ProcessFileForm(file_id=form_data.file_id, collection_name=id), user=user, db=db, ) - - # Add file to knowledge base - await Knowledges.add_file_to_knowledge_by_id( - knowledge_id=id, - file_id=form_data.file_id, - user_id=user.id, - directory_id=form_data.directory_id, - db=db, - ) except Exception as e: + if linked: + await Knowledges.remove_file_from_knowledge_by_id(id, form_data.file_id, db=db) log.debug(e) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -1943,6 +2018,18 @@ async def sync_knowledge_diff( if key not in manifest_keys: deleted.append({'file_id': file_info['file_id'], 'filename': key[1]}) + changed_keys = {(entry['path'], entry['filename']) for entry in added + modified} + await check_knowledge_upload_limits( + id, + len(added) + len(modified), + file_size=max( + (entry.size for entry in form_data.manifest if (entry.path, entry.filename) in changed_keys), default=0 + ), + batch=True, + exclude_file_ids=[entry['file_id'] for entry in deleted] + [entry['stale_file_id'] for entry in modified], + db=db, + ) + # ── Diff directories ── required_directory_paths: set[str] = set() for entry in form_data.manifest: @@ -2069,6 +2156,10 @@ async def add_files_to_knowledge_batch( detail=ERROR_MESSAGES.ACCESS_PROHIBITED, ) + batch_limit = await Config.get('rag.file.max_count') + if batch_limit is not None and len(form_data) > int(batch_limit): + raise HTTPException(400, f'You can only upload a maximum of {batch_limit} file(s) per batch.') + for directory_id in {form.directory_id for form in form_data if form.directory_id}: await _verify_directory_in_knowledge(id, directory_id, db, detail='Target directory not found.') @@ -2112,8 +2203,29 @@ async def add_files_to_knowledge_batch( new_file_ids = {form.file_id for form in new_entries} files = [f for f in files if f.id in new_file_ids] - # Process files + await check_knowledge_upload_limits( + id, + len(files), + file_size=max(((file.meta or {}).get('size', 0) or 0 for file in files), default=0), + exclude_file_ids=list(new_file_ids), + db=db, + ) + + # Claim slots before embedding; release them if processing fails. + reserved_ids = [] + dir_map = {form.file_id: form.directory_id for form in new_entries} try: + for file in files: + linked = await Knowledges.add_file_to_knowledge_by_id( + knowledge_id=id, + file_id=file.id, + user_id=user.id, + directory_id=dir_map.get(file.id), + db=db, + ) + if not linked: + raise ValueError(f'Failed to link file {file.id} to knowledge {id}') + reserved_ids.append(file.id) result = await process_files_batch( request=request, form_data=BatchProcessFilesForm(files=files, collection_name=id), @@ -2121,20 +2233,14 @@ async def add_files_to_knowledge_batch( db=db, ) except Exception as e: + for file_id in reserved_ids: + await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db) log.error(f'add_files_to_knowledge_batch: Exception occurred: {e}', exc_info=True) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) - # Only add files that were successfully processed successful_file_ids = [r.file_id for r in result.results if r.status == 'completed'] - dir_map = {form.file_id: form.directory_id for form in new_entries} - for file_id in successful_file_ids: - await Knowledges.add_file_to_knowledge_by_id( - knowledge_id=id, - file_id=file_id, - user_id=user.id, - directory_id=dir_map.get(file_id), - db=db, - ) + for file_id in set(reserved_ids) - set(successful_file_ids): + await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db) # If there were any errors, include them in the response if result.errors: diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index d2a6857c06..69005a3b22 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -305,6 +305,8 @@ RETRIEVAL_CONFIG_KEYS = { 'FILE_IMAGE_COMPRESSION_HEIGHT': 'file.image_compression_height', 'FILE_IMAGE_COMPRESSION_WIDTH': 'file.image_compression_width', 'FILE_MAX_COUNT': 'rag.file.max_count', + 'KNOWLEDGE_MAX_FILE_COUNT': 'knowledge.max_file_count', + 'KNOWLEDGE_MAX_FILE_SIZE': 'knowledge.max_file_size', 'FILE_MAX_SIZE': 'rag.file.max_size', 'FIRECRAWL_API_BASE_URL': 'web.loader.firecrawl_api_url', 'FIRECRAWL_API_KEY': 'web.loader.firecrawl_api_key', @@ -684,6 +686,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): # File upload settings 'FILE_MAX_SIZE': config['rag.file.max_size'], 'FILE_MAX_COUNT': config['rag.file.max_count'], + 'KNOWLEDGE_MAX_FILE_COUNT': config['knowledge.max_file_count'], + 'KNOWLEDGE_MAX_FILE_SIZE': config['knowledge.max_file_size'], 'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'], 'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'], 'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'], @@ -935,6 +939,8 @@ class ConfigForm(BaseModel): # File upload settings FILE_MAX_SIZE: Union[int, str | None] = None FILE_MAX_COUNT: Union[int, str | None] = None + KNOWLEDGE_MAX_FILE_COUNT: Union[int, str | None] = None + KNOWLEDGE_MAX_FILE_SIZE: Union[int, str | None] = None FILE_IMAGE_COMPRESSION_WIDTH: Union[int, str | None] = None FILE_IMAGE_COMPRESSION_HEIGHT: Union[int, str | None] = None ALLOWED_FILE_EXTENSIONS: list[str] | None = None @@ -998,10 +1004,33 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend } if form_data.RAG_TOKENIZER_MODEL is not None: updates['rag.tokenizer_model'] = form_data.RAG_TOKENIZER_MODEL.strip() - for field in ('FILE_MAX_SIZE', 'FILE_MAX_COUNT', 'FILE_IMAGE_COMPRESSION_WIDTH', 'FILE_IMAGE_COMPRESSION_HEIGHT'): + for field in ( + 'FILE_MAX_SIZE', + 'FILE_MAX_COUNT', + 'KNOWLEDGE_MAX_FILE_COUNT', + 'KNOWLEDGE_MAX_FILE_SIZE', + 'FILE_IMAGE_COMPRESSION_WIDTH', + 'FILE_IMAGE_COMPRESSION_HEIGHT', + ): if getattr(form_data, field) == '': updates[RETRIEVAL_CONFIG_KEYS[field]] = None + for field, label in ( + ('KNOWLEDGE_MAX_FILE_COUNT', 'Knowledge file count limit'), + ('KNOWLEDGE_MAX_FILE_SIZE', 'Knowledge file size limit'), + ): + value = getattr(form_data, field) + if field in form_data.model_fields_set and value is None: + updates[RETRIEVAL_CONFIG_KEYS[field]] = None + if value not in (None, ''): + try: + value = int(value) + if value <= 0: + raise ValueError + except (TypeError, ValueError): + raise HTTPException(400, f'{label} must be a positive integer.') + updates[RETRIEVAL_CONFIG_KEYS[field]] = value + # Unload the previous internal reranker before applying its new settings. if config['rag.reranking_engine'] == '': request.app.state.rf = None @@ -1116,6 +1145,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend # File upload settings 'FILE_MAX_SIZE': config['rag.file.max_size'], 'FILE_MAX_COUNT': config['rag.file.max_count'], + 'KNOWLEDGE_MAX_FILE_COUNT': config['knowledge.max_file_count'], + 'KNOWLEDGE_MAX_FILE_SIZE': config['knowledge.max_file_size'], 'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'], 'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'], 'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'], diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index d3a1780e9a..c6835ff208 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -1,10 +1,35 @@ import { WEBUI_API_BASE_URL } from '$lib/constants'; +type KnowledgeMeta = { + max_file_count: number | null; + max_file_size: number | null; +}; + +export const validateKnowledgeUpload = async ( + token: string, + id: string, + count: number, + removedFileIds: string[] = [], + fileSize?: number +) => { + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/files/validate`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify({ count, removed_file_ids: removedFileIds, file_size: fileSize }) + }); + if (!res.ok) throw (await res.json()).detail; + return res.json(); +}; + export const createNewKnowledge = async ( token: string, name: string, description: string, - accessGrants: object[] + accessGrants: object[], + meta?: KnowledgeMeta ) => { let error = null; @@ -18,7 +43,8 @@ export const createNewKnowledge = async ( body: JSON.stringify({ name: name, description: description, - access_grants: accessGrants + access_grants: accessGrants, + meta: meta }) }) .then(async (res) => { @@ -605,6 +631,7 @@ export const streamPendingKnowledgeFiles = async (token: string, id: string) => }; type KnowledgeUpdateForm = { + meta?: KnowledgeMeta; name?: string; description?: string; data?: object; @@ -623,7 +650,8 @@ export const updateKnowledgeById = async (token: string, id: string, form: Knowl body: JSON.stringify({ name: form?.name ? form.name : undefined, description: form?.description ? form.description : undefined, - data: form?.data ? form.data : undefined + data: form?.data ? form.data : undefined, + meta: form.meta }) }) .then(async (res) => { diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 9ac3656611..9d5678e1a3 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -282,6 +282,8 @@ // can distinguish "clear this field" from "don't change this field" FILE_MAX_SIZE: RAGConfig.FILE_MAX_SIZE ?? '', FILE_MAX_COUNT: RAGConfig.FILE_MAX_COUNT ?? '', + KNOWLEDGE_MAX_FILE_COUNT: RAGConfig.KNOWLEDGE_MAX_FILE_COUNT ?? '', + KNOWLEDGE_MAX_FILE_SIZE: RAGConfig.KNOWLEDGE_MAX_FILE_SIZE ?? '', FILE_IMAGE_COMPRESSION_WIDTH: RAGConfig.FILE_IMAGE_COMPRESSION_WIDTH ?? '', FILE_IMAGE_COMPRESSION_HEIGHT: RAGConfig.FILE_IMAGE_COMPRESSION_HEIGHT ?? '', ALLOWED_FILE_EXTENSIONS: RAGConfig.ALLOWED_FILE_EXTENSIONS.split(',') @@ -1591,6 +1593,37 @@ +
+ + + + + + +
+
- { closeCreateModal(); diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index c7a1edaf15..850ce952bb 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -3,6 +3,7 @@ import { v4 as uuidv4 } from 'uuid'; import { onMount, getContext, onDestroy } from 'svelte'; + import KnowledgeEditor from './KnowledgeEditor.svelte'; import KnowledgeBrowser from './KnowledgeBase/Browser.svelte'; import type { Writable } from 'svelte/store'; import type { i18n as i18nType } from 'i18next'; @@ -16,6 +17,7 @@ import { uploadFile } from '$lib/apis/files'; import { addFileToKnowledgeById, + validateKnowledgeUpload, getKnowledgeById, resetKnowledgeById, updateKnowledgeById, @@ -31,6 +33,11 @@ import { computeFileHash } from '$lib/utils/hash'; import Spinner from '$lib/components/common/Spinner.svelte'; + import Modal from '$lib/components/common/Modal.svelte'; + import Dropdown from '$lib/components/common/Dropdown.svelte'; + import DropdownMenu from '$lib/components/common/DropdownMenu.svelte'; + import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte'; + import Cog6 from '$lib/components/icons/Cog6.svelte'; import AddTextContentModal from './KnowledgeBase/AddTextContentModal.svelte'; @@ -51,12 +58,14 @@ let showAccessControlModal = false; let savedAccessGrants = []; let showResetConfirm = false; + let showSettings = false; type DirectoryFileEntry = { path: string; filename: string; file: File }; type DirectoryManifestEntry = DirectoryFileEntry & { checksum: string; size: number }; type Knowledge = { id: string; + user_id: string; name: string; description: string; data: { @@ -125,6 +134,35 @@ return file; }; + const validateUpload = async ( + count: number, + removedFileIds: string[] = [], + fileSize?: number + ) => { + if (!knowledge) return false; + if (count === 0) return true; + const maxCount = $config?.file?.max_count; + if (maxCount != null && count > maxCount) { + toast.error( + $i18n.t('You can only upload a maximum of {{maxCount}} file(s) per batch.', { maxCount }) + ); + return false; + } + try { + await validateKnowledgeUpload( + localStorage.token, + knowledge.id, + count, + removedFileIds, + fileSize + ); + return true; + } catch (error) { + toast.error(`${error}`); + return false; + } + }; + const uploadWeb = async (urls) => { const targetDirectoryId = currentDirectoryId; if (!knowledge) { @@ -135,6 +173,7 @@ if (!Array.isArray(urls)) { urls = [urls]; } + if (!(await validateUpload(urls.length))) return; const newFileItems = urls.map((url) => ({ type: 'file', @@ -331,6 +370,14 @@ // Rows go up front because each upload waits until its file is processed. const uploadFiles = async (files: File[]) => { + if ( + !(await validateUpload( + files.length, + [], + files.reduce((size, file) => Math.max(size, file.size), 0) + )) + ) + return; const uploads: { file: File; fileItem: { itemId: string; directory_id: string | null } }[] = []; for (const file of files) { const fileItem = createFileItem(file); @@ -529,6 +576,14 @@ const uploadDirectoryEntries = async (entries: DirectoryFileEntry[]) => { if (!knowledge) return; + if ( + !(await validateUpload( + entries.length, + [], + entries.reduce((size, entry) => Math.max(size, entry.file.size), 0) + )) + ) + return; const targetDirectoryId = currentDirectoryId; const targetPath = breadcrumbs.map((crumb) => crumb.name).join('/'); const getDirectoryUploadPath = (path: string) => @@ -603,6 +658,7 @@ ...diff.deleted.map((d: any) => d.file_id), ...diff.modified.map((m: any) => m.stale_file_id) ]; + if (!(await validateUpload(diff.added.length + diff.modified.length, staleFileIds))) return; if (staleFileIds.length > 0 || diff.rmdir.length > 0) { syncing = $i18n.t('Removing {{count}} stale files...', { count: staleFileIds.length }); @@ -779,6 +835,18 @@ } } + // A mixed drop is one batch, even though its upload paths differ. + if ( + !(await validateUpload( + looseFiles.length + directoryEntries.length, + [], + directoryEntries.reduce( + (size, entry) => Math.max(size, entry.file.size), + looseFiles.reduce((size, file) => Math.max(size, file.size), 0) + ) + )) + ) + return; await uploadFiles(looseFiles); if (directoryEntries.length > 0) { @@ -908,13 +976,45 @@ on:click={() => goto('/workspace/knowledge')} >{$i18n.t('Back')} - - {#if knowledge.write_access}{#if $user?.permissions?.sharing?.knowledge || $user?.role === 'admin'} { - savedAccessGrants = structuredClone(knowledge.access_grants ?? []); - showAccessControlModal = true; - }} - />{/if}{:else}{$i18n.t('Read Only')}{/if} +
+ {#if knowledge.write_access} + {#if $user?.permissions?.sharing?.knowledge || $user?.role === 'admin'} + { + savedAccessGrants = structuredClone(knowledge.access_grants ?? []); + showAccessControlModal = true; + }} + /> + {/if} + {:else} + {$i18n.t('Read Only')} + {/if} + {#if !isExternalKnowledge && (knowledge.user_id === $user?.id || $user?.role === 'admin')} + + +
+ + + +
+ + {/if} +
{/if}
+ + {#if knowledge} + { + showSettings = false; + }} + onSaved={(saved) => { + if (!knowledge) return; + knowledge = { + ...knowledge, + name: saved.name, + description: saved.description, + meta: saved.meta + }; + showSettings = false; + }} + /> + {/if} + + void | Promise} */ export let onBack = () => goto('/workspace/knowledge'); - /** @type {(knowledge: { id: string }) => void | Promise} */ - export let onCreated = (knowledge) => goto(`/workspace/knowledge/${knowledge.id}`); + /** @type {(knowledge: { id: string; name: string; description: string; meta?: any; access_grants?: any[] }) => void | Promise} */ + export let onSaved = (knowledge) => goto(`/workspace/knowledge/${knowledge.id}`); let loading = false; - let name = ''; - let description = ''; + let name = knowledge?.name ?? ''; + let description = knowledge?.description ?? ''; /** @type {{ id?: string; principal_type: 'user' | 'group'; principal_id: string; permission: 'read' | 'write' }[]} */ let accessGrants = []; + let showAdvanced = false; + /** @type {number | undefined} */ + let maxFileCount = knowledge?.meta?.max_file_count || undefined; + /** @type {number | undefined} */ + let maxFileSize = knowledge?.meta?.max_file_size || undefined; const submitHandler = async () => { - loading = true; - + if (loading) return; if (name.trim() === '' || description.trim() === '') { toast.error($i18n.t('Please fill in all fields.')); - name = ''; - description = ''; - loading = false; return; } - const res = await createNewKnowledge(localStorage.token, name, description, accessGrants).catch( - (e) => { - toast.error(`${e}`); + loading = true; + const meta = { + max_file_count: maxFileCount ?? null, + max_file_size: maxFileSize ?? null + }; + try { + const res = + edit && knowledge + ? await updateKnowledgeById(localStorage.token, knowledge.id, { + name, + description, + meta + }) + : await createNewKnowledge(localStorage.token, name, description, accessGrants, meta); + if (res) { + toast.success($i18n.t(edit ? 'Saved' : 'Knowledge created successfully.')); + await onSaved(res); } - ); - - if (res) { - toast.success($i18n.t('Knowledge created successfully.')); - await onCreated(res); + } catch (error) { + toast.error(`${error}`); + } finally { + loading = false; } - - loading = false; };
{#if modal}
-

{$i18n.t('Create a knowledge base')}

+

+ {$i18n.t(edit ? 'Settings' : 'Create a knowledge base')} +

-
- -
+ {#if !edit} +
+ +
+ {/if} + + + + {#if showAdvanced} +
+
+
+ + +

{$i18n.t('Maximum total files.')}

+
+
+ + +

{$i18n.t('Maximum size per file.')}

+
+
+

+ {$i18n.t('Global limits always apply.')} +

+
+ {/if}
{#if modal} @@ -177,7 +249,7 @@ type="submit" disabled={loading} > -
{$i18n.t('Create Knowledge')}
+
{$i18n.t(edit ? 'Save' : 'Create Knowledge')}
{#if loading}
diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json index 3e444dc8db..1147a9d29e 100644 --- a/src/lib/i18n/locales/en-US/translation.json +++ b/src/lib/i18n/locales/en-US/translation.json @@ -1,4 +1,11 @@ { + "Maximum total files.": "", + "Maximum size per file.": "", + "File count limit": "", + "File size limit (MB)": "", + "Use global setting": "", + "Global limits always apply.": "", + "You can only upload a maximum of {{maxCount}} file(s) per batch.": "", "- usage not recorded": "- usage not recorded", "- usage not recorded · * partial usage": "- usage not recorded · * partial usage", "-1 for no limit, or a positive integer for a specific limit": "", @@ -2870,8 +2877,12 @@ "settings.admin.documents.imageCompressionWidth.label": "Image Compression Width", "settings.admin.documents.markdownHeaderTextSplitter.description": "Split documents by markdown headers before character or token splitting.", "settings.admin.documents.markdownHeaderTextSplitter.label": "Markdown Header Text Splitter", - "settings.admin.documents.maxUploadCount.description": "Maximum number of files that can be used at once in chat.", + "settings.admin.documents.maxUploadCount.description": "Maximum number of files that can be used at once in chat or uploaded in one knowledge base batch.", "settings.admin.documents.maxUploadCount.label": "Max Upload Count", + "settings.admin.documents.knowledgeMaxSize.label": "Knowledge File Size Limit", + "settings.admin.documents.knowledgeMaxSize.description": "Maximum file size in MB per knowledge base. Leave empty for no limit.", + "settings.admin.documents.knowledgeMaxCount.label": "Knowledge File Count Limit", + "settings.admin.documents.knowledgeMaxCount.description": "Maximum total files per knowledge base. Leave empty for no limit.", "settings.admin.documents.maxUploadSize.description": "Maximum file size in MB. Leave empty for unlimited.", "settings.admin.documents.maxUploadSize.label": "Max Upload Size", "settings.admin.documents.mineruApiKey.description": "API key used for MinerU cloud mode.",