This commit is contained in:
Timothy Jaeryang Baek 2026-10-30 07:08:28 +09:00
parent 9f33c966c9
commit 7618df3f53
11 changed files with 588 additions and 79 deletions

View file

@ -996,6 +996,12 @@ ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = (
RAG_FULL_CONTEXT = os.getenv('RAG_FULL_CONTEXT', 'False').lower() == 'true'
RAG_FILE_MAX_COUNT = int(os.getenv('RAG_FILE_MAX_COUNT')) if os.getenv('RAG_FILE_MAX_COUNT') else None
KNOWLEDGE_MAX_FILE_COUNT = int(os.getenv('KNOWLEDGE_MAX_FILE_COUNT')) if os.getenv('KNOWLEDGE_MAX_FILE_COUNT') else None
KNOWLEDGE_MAX_FILE_SIZE = int(os.getenv('KNOWLEDGE_MAX_FILE_SIZE')) if os.getenv('KNOWLEDGE_MAX_FILE_SIZE') else None
if KNOWLEDGE_MAX_FILE_COUNT is not None and KNOWLEDGE_MAX_FILE_COUNT <= 0:
raise ValueError('KNOWLEDGE_MAX_FILE_COUNT must be a positive integer or unset')
if KNOWLEDGE_MAX_FILE_SIZE is not None and KNOWLEDGE_MAX_FILE_SIZE <= 0:
raise ValueError('KNOWLEDGE_MAX_FILE_SIZE must be a positive integer or unset')
RAG_FILE_MAX_SIZE = int(os.getenv('RAG_FILE_MAX_SIZE')) if os.getenv('RAG_FILE_MAX_SIZE') else None
@ -3013,6 +3019,8 @@ DEFAULT_CONFIG = {
'rag.enable_hybrid_search_enriched_texts': ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS,
'rag.full_context': RAG_FULL_CONTEXT,
'rag.file.max_count': RAG_FILE_MAX_COUNT,
'knowledge.max_file_count': KNOWLEDGE_MAX_FILE_COUNT,
'knowledge.max_file_size': KNOWLEDGE_MAX_FILE_SIZE,
'rag.file.max_size': RAG_FILE_MAX_SIZE,
'file.image_compression_width': FILE_IMAGE_COMPRESSION_WIDTH,
'file.image_compression_height': FILE_IMAGE_COMPRESSION_HEIGHT,

View file

@ -6,6 +6,7 @@ from typing import Optional
from open_webui.config import RAG_FILE_CONTENT_SEARCH_MAX_CHARS
from open_webui.internal.db import Base, JSONField, get_async_db_context
from open_webui.models.access_grants import AccessGrantModel, AccessGrants
from open_webui.models.config import Config
from open_webui.models.files import (
File,
FileMetadataResponse,
@ -162,10 +163,16 @@ class KnowledgeUserResponse(KnowledgeUserModel):
pass
class KnowledgeMeta(BaseModel):
max_file_count: int | None = Field(default=None, gt=0)
max_file_size: int | None = Field(default=None, gt=0)
class KnowledgeForm(BaseModel):
name: str
description: str
access_grants: Optional[list[dict]] = None
meta: KnowledgeMeta | None = None
class FileUserResponse(FileModelResponse):
@ -209,7 +216,8 @@ class KnowledgeTable:
async with get_async_db_context(db) as db:
knowledge = KnowledgeModel(
**{
**form_data.model_dump(exclude={'access_grants'}),
**form_data.model_dump(exclude={'access_grants', 'meta'}),
'meta': form_data.meta.model_dump() if form_data.meta else None,
'id': str(uuid.uuid4()),
'user_id': user_id,
'created_at': int(time.time()),
@ -671,6 +679,37 @@ class KnowledgeTable:
except Exception:
return []
async def get_file_limits(self, meta: dict | None) -> dict[str, int | None]:
local = meta or {}
counts = [
int(value) for value in (await Config.get('knowledge.max_file_count'), local.get('max_file_count')) if value
]
sizes = [
int(value) for value in (await Config.get('knowledge.max_file_size'), local.get('max_file_size')) if value
]
return {'max_file_count': min(counts, default=None), 'max_file_size': min(sizes, default=None)}
async def get_file_count_by_id(
self,
knowledge_id: str,
include_pending: bool = False,
exclude_file_ids: list[str] | None = None,
db: Optional[AsyncSession] = None,
) -> int:
async with get_async_db_context(db) as db:
linked = select(KnowledgeFile.file_id.label('file_id')).where(KnowledgeFile.knowledge_id == knowledge_id)
if include_pending:
pending = select(File.id.label('file_id')).where(
File.meta['data']['knowledge_id'].as_string() == knowledge_id,
File.data['status'].as_string().in_(['pending', 'processing']),
)
linked = linked.union(pending)
files = linked.subquery()
stmt = select(func.count()).select_from(files)
if exclude_file_ids:
stmt = stmt.where(files.c.file_id.notin_(exclude_file_ids))
return (await db.execute(stmt)).scalar_one()
async def add_file_to_knowledge_by_id(
self,
knowledge_id: str,
@ -679,7 +718,27 @@ class KnowledgeTable:
directory_id: Optional[str] = None,
db: Optional[AsyncSession] = None,
) -> Optional[KnowledgeFileModel]:
global_limit = await Config.get('knowledge.max_file_count')
async with get_async_db_context(db) as db:
# Read the local limit under the same write lock as count + insert.
await db.execute(
update(Knowledge).where(Knowledge.id == knowledge_id).values(updated_at=Knowledge.updated_at)
)
meta = await db.scalar(select(Knowledge.meta).where(Knowledge.id == knowledge_id))
local_limit = (meta or {}).get('max_file_count')
limit = min((int(value) for value in (global_limit, local_limit) if value), default=None)
if limit is not None:
existing = await db.scalar(select(KnowledgeFile).filter_by(knowledge_id=knowledge_id, file_id=file_id))
if existing:
await db.commit()
# Preserve duplicate-insert semantics; callers must not release another request's slot.
return None
count = await db.scalar(
select(func.count()).select_from(KnowledgeFile).where(KnowledgeFile.knowledge_id == knowledge_id)
)
if count >= int(limit):
await db.rollback()
raise ValueError(f'Knowledge base file limit exceeded ({limit} files maximum).')
knowledge_file = KnowledgeFileModel(
**{
'id': str(uuid.uuid4()),
@ -702,6 +761,7 @@ class KnowledgeTable:
else:
return None
except Exception:
await db.rollback()
return None
async def has_file(self, knowledge_id: str, file_id: str, db: Optional[AsyncSession] = None) -> bool:
@ -758,11 +818,15 @@ class KnowledgeTable:
) -> Optional[KnowledgeModel]:
try:
async with get_async_db_context(db) as db:
values = form_data.model_dump(exclude={'access_grants', 'meta'})
if form_data.meta is not None:
meta = await db.scalar(select(Knowledge.meta).where(Knowledge.id == id)) or {}
values['meta'] = {**meta, **form_data.meta.model_dump(exclude_unset=True)}
await db.execute(
update(Knowledge)
.filter_by(id=id)
.values(
**form_data.model_dump(exclude={'access_grants'}),
**values,
updated_at=int(time.time()),
)
)

View file

@ -41,6 +41,7 @@ from open_webui.models.knowledge import Knowledges
from open_webui.models.users import Users
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.routers.audio import transcribe
from open_webui.routers.knowledge import check_knowledge_upload_limits
from open_webui.routers.retrieval import ProcessFileForm, index_file, process_file
from open_webui.storage.provider import Storage
from open_webui.utils.auth import get_admin_user, get_verified_user
@ -136,6 +137,14 @@ async def process_uploaded_file(
):
async def _process_handler(db_session):
try:
if file_metadata.get('knowledge_id'):
await check_knowledge_upload_limits(
file_metadata['knowledge_id'],
1,
file_size=(file_item.meta or {}).get('size'),
exclude_file_ids=[file_item.id],
db=db_session,
)
await Files.update_file_data_by_id(file_item.id, {'status': 'processing'}, db=db_session)
content_type = file.content_type
@ -229,12 +238,6 @@ async def process_uploaded_file(
)
directory_id = None
await index_file(
request,
ProcessFileForm(file_id=file_item.id, collection_name=knowledge_id),
user=user,
db=db_session,
)
knowledge_file = await Knowledges.add_file_to_knowledge_by_id(
knowledge_id=knowledge_id,
file_id=file_item.id,
@ -244,6 +247,16 @@ async def process_uploaded_file(
)
if not knowledge_file:
raise Exception(f'Failed to link file {file_item.id} to knowledge {knowledge_id}')
try:
await index_file(
request,
ProcessFileForm(file_id=file_item.id, collection_name=knowledge_id),
user=user,
db=db_session,
)
except Exception:
await Knowledges.remove_file_from_knowledge_by_id(knowledge_id, file_item.id, db=db_session)
raise
log.info('Linked file %s to knowledge %s', file_item.id, knowledge_id)
except Exception as e:
log.warning(f'Failed to link file {file_item.id} to knowledge {knowledge_id}: {e}')
@ -357,6 +370,19 @@ async def upload_file_handler(
detail=ERROR_MESSAGES.DEFAULT(f'File type {file_extension} is not allowed'),
)
knowledge_id = file_metadata.get('knowledge_id')
if knowledge_id:
knowledge = await Knowledges.get_knowledge_by_id(knowledge_id, db=db)
if not knowledge or not (
knowledge.user_id == user.id
or user.role == 'admin'
or await AccessGrants.has_access(
user_id=user.id, resource_type='knowledge', resource_id=knowledge_id, permission='write', db=db
)
):
raise HTTPException(403, ERROR_MESSAGES.ACCESS_PROHIBITED)
await check_knowledge_upload_limits(knowledge_id, 1, batch=True, file_size=file.size, db=db)
# Prefer readable storage names for admins, but fall back if the filesystem rejects it.
id = str(uuid.uuid4())
name = filename
@ -395,6 +421,13 @@ async def upload_file_handler(
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB'),
)
if knowledge_id:
try:
await check_knowledge_upload_limits(knowledge_id, 0, file_size=len(contents), db=db)
except HTTPException:
await asyncio.to_thread(Storage.delete_file, file_path)
raise
# SHA-256 of raw uploaded bytes for incremental sync diffing.
# If the client pre-computed and sent file_hash, use that.
file_hash = file_metadata.get('file_hash') or await asyncio.to_thread(

View file

@ -46,7 +46,7 @@ from open_webui.utils.access_control import filter_allowed_access_grants, has_pe
from open_webui.utils.access_control.files import has_access_to_file
from open_webui.utils.auth import get_admin_user, get_verified_user
from open_webui.utils.json_codec import JSONCodec
from pydantic import BaseModel
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
log = logging.getLogger(__name__)
@ -1169,6 +1169,12 @@ async def update_knowledge_by_id(
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
if form_data.meta is not None:
if knowledge.user_id != user.id and user.role != 'admin':
raise HTTPException(403, ERROR_MESSAGES.ACCESS_PROHIBITED)
if is_external_knowledge(knowledge):
external_knowledge_error()
knowledge = await Knowledges.update_knowledge_by_id(id=id, form_data=form_data)
if knowledge:
# Re-embed knowledge base for semantic search
@ -1402,6 +1408,65 @@ async def get_knowledge_files_by_id(
############################
async def check_knowledge_upload_limits(
knowledge_id: str,
count: int,
*,
batch: bool = False,
file_size: int | None = None,
exclude_file_ids: list[str] | None = None,
db=None,
) -> None:
if batch:
batch_limit = await Config.get('rag.file.max_count')
if batch_limit is not None and count > int(batch_limit):
raise HTTPException(400, f'You can only upload a maximum of {batch_limit} file(s) per batch.')
knowledge = await Knowledges.get_knowledge_by_id(knowledge_id, db=db)
if not knowledge:
raise HTTPException(404, ERROR_MESSAGES.NOT_FOUND)
limits = await Knowledges.get_file_limits(knowledge.meta)
if file_size is not None:
for max_size in (await Config.get('rag.file.max_size'), limits['max_file_size']):
if max_size and file_size > int(max_size) * 1024 * 1024:
raise HTTPException(413, ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB'))
limit = limits['max_file_count']
if limit is not None and count > 0:
current_count = await Knowledges.get_file_count_by_id(
knowledge_id, include_pending=True, exclude_file_ids=exclude_file_ids, db=db
)
if current_count + count > int(limit):
raise HTTPException(400, f'Knowledge base file limit exceeded ({limit} files maximum).')
class KnowledgeUploadValidationForm(BaseModel):
count: int = Field(ge=0)
file_size: int | None = Field(default=None, ge=0)
removed_file_ids: list[str] = Field(default_factory=list)
@router.post('/{id}/files/validate')
async def validate_knowledge_upload(
id: str,
form_data: KnowledgeUploadValidationForm,
user=Depends(get_verified_user),
db: AsyncSession = Depends(get_async_session),
):
await _verify_knowledge_write_access(id, user, db)
# This is a preflight only. Actual additions recheck capacity without removal credits.
await check_knowledge_upload_limits(
id,
form_data.count,
batch=True,
file_size=form_data.file_size,
exclude_file_ids=form_data.removed_file_ids,
db=db,
)
return {'status': True}
class KnowledgeFileIdForm(BaseModel):
file_id: str
directory_id: Optional[str] = None
@ -1462,24 +1527,34 @@ async def add_file_to_knowledge_by_id(
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
# Add content to the vector database
already_linked = await Knowledges.has_file(knowledge_id=id, file_id=form_data.file_id, db=db)
if not already_linked:
await check_knowledge_upload_limits(
id, 1, batch=True, file_size=(file.meta or {}).get('size', 0), exclude_file_ids=[form_data.file_id], db=db
)
linked = None
try:
# Claim capacity atomically before indexing into the knowledge collection.
if not already_linked:
linked = await Knowledges.add_file_to_knowledge_by_id(
knowledge_id=id,
file_id=form_data.file_id,
user_id=user.id,
directory_id=form_data.directory_id,
db=db,
)
if not linked:
raise ValueError(f'Failed to link file {form_data.file_id} to knowledge {id}')
await process_file(
request,
ProcessFileForm(file_id=form_data.file_id, collection_name=id),
user=user,
db=db,
)
# Add file to knowledge base
await Knowledges.add_file_to_knowledge_by_id(
knowledge_id=id,
file_id=form_data.file_id,
user_id=user.id,
directory_id=form_data.directory_id,
db=db,
)
except Exception as e:
if linked:
await Knowledges.remove_file_from_knowledge_by_id(id, form_data.file_id, db=db)
log.debug(e)
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
@ -1943,6 +2018,18 @@ async def sync_knowledge_diff(
if key not in manifest_keys:
deleted.append({'file_id': file_info['file_id'], 'filename': key[1]})
changed_keys = {(entry['path'], entry['filename']) for entry in added + modified}
await check_knowledge_upload_limits(
id,
len(added) + len(modified),
file_size=max(
(entry.size for entry in form_data.manifest if (entry.path, entry.filename) in changed_keys), default=0
),
batch=True,
exclude_file_ids=[entry['file_id'] for entry in deleted] + [entry['stale_file_id'] for entry in modified],
db=db,
)
# ── Diff directories ──
required_directory_paths: set[str] = set()
for entry in form_data.manifest:
@ -2069,6 +2156,10 @@ async def add_files_to_knowledge_batch(
detail=ERROR_MESSAGES.ACCESS_PROHIBITED,
)
batch_limit = await Config.get('rag.file.max_count')
if batch_limit is not None and len(form_data) > int(batch_limit):
raise HTTPException(400, f'You can only upload a maximum of {batch_limit} file(s) per batch.')
for directory_id in {form.directory_id for form in form_data if form.directory_id}:
await _verify_directory_in_knowledge(id, directory_id, db, detail='Target directory not found.')
@ -2112,8 +2203,29 @@ async def add_files_to_knowledge_batch(
new_file_ids = {form.file_id for form in new_entries}
files = [f for f in files if f.id in new_file_ids]
# Process files
await check_knowledge_upload_limits(
id,
len(files),
file_size=max(((file.meta or {}).get('size', 0) or 0 for file in files), default=0),
exclude_file_ids=list(new_file_ids),
db=db,
)
# Claim slots before embedding; release them if processing fails.
reserved_ids = []
dir_map = {form.file_id: form.directory_id for form in new_entries}
try:
for file in files:
linked = await Knowledges.add_file_to_knowledge_by_id(
knowledge_id=id,
file_id=file.id,
user_id=user.id,
directory_id=dir_map.get(file.id),
db=db,
)
if not linked:
raise ValueError(f'Failed to link file {file.id} to knowledge {id}')
reserved_ids.append(file.id)
result = await process_files_batch(
request=request,
form_data=BatchProcessFilesForm(files=files, collection_name=id),
@ -2121,20 +2233,14 @@ async def add_files_to_knowledge_batch(
db=db,
)
except Exception as e:
for file_id in reserved_ids:
await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db)
log.error(f'add_files_to_knowledge_batch: Exception occurred: {e}', exc_info=True)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
# Only add files that were successfully processed
successful_file_ids = [r.file_id for r in result.results if r.status == 'completed']
dir_map = {form.file_id: form.directory_id for form in new_entries}
for file_id in successful_file_ids:
await Knowledges.add_file_to_knowledge_by_id(
knowledge_id=id,
file_id=file_id,
user_id=user.id,
directory_id=dir_map.get(file_id),
db=db,
)
for file_id in set(reserved_ids) - set(successful_file_ids):
await Knowledges.remove_file_from_knowledge_by_id(id, file_id, db=db)
# If there were any errors, include them in the response
if result.errors:

View file

@ -305,6 +305,8 @@ RETRIEVAL_CONFIG_KEYS = {
'FILE_IMAGE_COMPRESSION_HEIGHT': 'file.image_compression_height',
'FILE_IMAGE_COMPRESSION_WIDTH': 'file.image_compression_width',
'FILE_MAX_COUNT': 'rag.file.max_count',
'KNOWLEDGE_MAX_FILE_COUNT': 'knowledge.max_file_count',
'KNOWLEDGE_MAX_FILE_SIZE': 'knowledge.max_file_size',
'FILE_MAX_SIZE': 'rag.file.max_size',
'FIRECRAWL_API_BASE_URL': 'web.loader.firecrawl_api_url',
'FIRECRAWL_API_KEY': 'web.loader.firecrawl_api_key',
@ -684,6 +686,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
# File upload settings
'FILE_MAX_SIZE': config['rag.file.max_size'],
'FILE_MAX_COUNT': config['rag.file.max_count'],
'KNOWLEDGE_MAX_FILE_COUNT': config['knowledge.max_file_count'],
'KNOWLEDGE_MAX_FILE_SIZE': config['knowledge.max_file_size'],
'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'],
'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'],
'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'],
@ -935,6 +939,8 @@ class ConfigForm(BaseModel):
# File upload settings
FILE_MAX_SIZE: Union[int, str | None] = None
FILE_MAX_COUNT: Union[int, str | None] = None
KNOWLEDGE_MAX_FILE_COUNT: Union[int, str | None] = None
KNOWLEDGE_MAX_FILE_SIZE: Union[int, str | None] = None
FILE_IMAGE_COMPRESSION_WIDTH: Union[int, str | None] = None
FILE_IMAGE_COMPRESSION_HEIGHT: Union[int, str | None] = None
ALLOWED_FILE_EXTENSIONS: list[str] | None = None
@ -998,10 +1004,33 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
}
if form_data.RAG_TOKENIZER_MODEL is not None:
updates['rag.tokenizer_model'] = form_data.RAG_TOKENIZER_MODEL.strip()
for field in ('FILE_MAX_SIZE', 'FILE_MAX_COUNT', 'FILE_IMAGE_COMPRESSION_WIDTH', 'FILE_IMAGE_COMPRESSION_HEIGHT'):
for field in (
'FILE_MAX_SIZE',
'FILE_MAX_COUNT',
'KNOWLEDGE_MAX_FILE_COUNT',
'KNOWLEDGE_MAX_FILE_SIZE',
'FILE_IMAGE_COMPRESSION_WIDTH',
'FILE_IMAGE_COMPRESSION_HEIGHT',
):
if getattr(form_data, field) == '':
updates[RETRIEVAL_CONFIG_KEYS[field]] = None
for field, label in (
('KNOWLEDGE_MAX_FILE_COUNT', 'Knowledge file count limit'),
('KNOWLEDGE_MAX_FILE_SIZE', 'Knowledge file size limit'),
):
value = getattr(form_data, field)
if field in form_data.model_fields_set and value is None:
updates[RETRIEVAL_CONFIG_KEYS[field]] = None
if value not in (None, ''):
try:
value = int(value)
if value <= 0:
raise ValueError
except (TypeError, ValueError):
raise HTTPException(400, f'{label} must be a positive integer.')
updates[RETRIEVAL_CONFIG_KEYS[field]] = value
# Unload the previous internal reranker before applying its new settings.
if config['rag.reranking_engine'] == '':
request.app.state.rf = None
@ -1116,6 +1145,8 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
# File upload settings
'FILE_MAX_SIZE': config['rag.file.max_size'],
'FILE_MAX_COUNT': config['rag.file.max_count'],
'KNOWLEDGE_MAX_FILE_COUNT': config['knowledge.max_file_count'],
'KNOWLEDGE_MAX_FILE_SIZE': config['knowledge.max_file_size'],
'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'],
'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'],
'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'],

View file

@ -1,10 +1,35 @@
import { WEBUI_API_BASE_URL } from '$lib/constants';
type KnowledgeMeta = {
max_file_count: number | null;
max_file_size: number | null;
};
export const validateKnowledgeUpload = async (
token: string,
id: string,
count: number,
removedFileIds: string[] = [],
fileSize?: number
) => {
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/files/validate`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify({ count, removed_file_ids: removedFileIds, file_size: fileSize })
});
if (!res.ok) throw (await res.json()).detail;
return res.json();
};
export const createNewKnowledge = async (
token: string,
name: string,
description: string,
accessGrants: object[]
accessGrants: object[],
meta?: KnowledgeMeta
) => {
let error = null;
@ -18,7 +43,8 @@ export const createNewKnowledge = async (
body: JSON.stringify({
name: name,
description: description,
access_grants: accessGrants
access_grants: accessGrants,
meta: meta
})
})
.then(async (res) => {
@ -605,6 +631,7 @@ export const streamPendingKnowledgeFiles = async (token: string, id: string) =>
};
type KnowledgeUpdateForm = {
meta?: KnowledgeMeta;
name?: string;
description?: string;
data?: object;
@ -623,7 +650,8 @@ export const updateKnowledgeById = async (token: string, id: string, form: Knowl
body: JSON.stringify({
name: form?.name ? form.name : undefined,
description: form?.description ? form.description : undefined,
data: form?.data ? form.data : undefined
data: form?.data ? form.data : undefined,
meta: form.meta
})
})
.then(async (res) => {

View file

@ -282,6 +282,8 @@
// can distinguish "clear this field" from "don't change this field"
FILE_MAX_SIZE: RAGConfig.FILE_MAX_SIZE ?? '',
FILE_MAX_COUNT: RAGConfig.FILE_MAX_COUNT ?? '',
KNOWLEDGE_MAX_FILE_COUNT: RAGConfig.KNOWLEDGE_MAX_FILE_COUNT ?? '',
KNOWLEDGE_MAX_FILE_SIZE: RAGConfig.KNOWLEDGE_MAX_FILE_SIZE ?? '',
FILE_IMAGE_COMPRESSION_WIDTH: RAGConfig.FILE_IMAGE_COMPRESSION_WIDTH ?? '',
FILE_IMAGE_COMPRESSION_HEIGHT: RAGConfig.FILE_IMAGE_COMPRESSION_HEIGHT ?? '',
ALLOWED_FILE_EXTENSIONS: RAGConfig.ALLOWED_FILE_EXTENSIONS.split(',')
@ -1591,6 +1593,37 @@
</AdminSettingField>
</div>
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
<AdminSettingField
label={$i18n.t('settings.admin.documents.knowledgeMaxSize.label')}
description={$i18n.t('settings.admin.documents.knowledgeMaxSize.description')}
>
<input
class={inputClass}
type="number"
placeholder={$i18n.t('Leave empty for unlimited')}
bind:value={RAGConfig.KNOWLEDGE_MAX_FILE_SIZE}
autocomplete="off"
min="1"
step="1"
/>
</AdminSettingField>
<AdminSettingField
label={$i18n.t('settings.admin.documents.knowledgeMaxCount.label')}
description={$i18n.t('settings.admin.documents.knowledgeMaxCount.description')}
>
<input
class={inputClass}
type="number"
placeholder={$i18n.t('Leave empty for unlimited')}
bind:value={RAGConfig.KNOWLEDGE_MAX_FILE_COUNT}
autocomplete="off"
min="1"
step="1"
/>
</AdminSettingField>
</div>
<div class="grid grid-cols-1 gap-x-3 gap-y-2.5 sm:grid-cols-2">
<AdminSettingField
label={$i18n.t('settings.admin.documents.imageCompressionWidth.label')}

View file

@ -24,7 +24,7 @@
import DeleteConfirmDialog from '../common/ConfirmDialog.svelte';
import ItemMenu from './Knowledge/ItemMenu.svelte';
import CreateKnowledgeBase from './Knowledge/CreateKnowledgeBase.svelte';
import KnowledgeEditor from './Knowledge/KnowledgeEditor.svelte';
import Badge from '../common/Badge.svelte';
import ChevronDown from '../icons/ChevronDown.svelte';
import ChevronUp from '../icons/ChevronUp.svelte';
@ -322,7 +322,7 @@
/>
<Modal bind:show={showCreateModal} size="md" ariaLabel={$i18n.t('Create a knowledge base')}>
<CreateKnowledgeBase
<KnowledgeEditor
modal={true}
onBack={() => {
closeCreateModal();

View file

@ -3,6 +3,7 @@
import { v4 as uuidv4 } from 'uuid';
import { onMount, getContext, onDestroy } from 'svelte';
import KnowledgeEditor from './KnowledgeEditor.svelte';
import KnowledgeBrowser from './KnowledgeBase/Browser.svelte';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
@ -16,6 +17,7 @@
import { uploadFile } from '$lib/apis/files';
import {
addFileToKnowledgeById,
validateKnowledgeUpload,
getKnowledgeById,
resetKnowledgeById,
updateKnowledgeById,
@ -31,6 +33,11 @@
import { computeFileHash } from '$lib/utils/hash';
import Spinner from '$lib/components/common/Spinner.svelte';
import Modal from '$lib/components/common/Modal.svelte';
import Dropdown from '$lib/components/common/Dropdown.svelte';
import DropdownMenu from '$lib/components/common/DropdownMenu.svelte';
import EllipsisHorizontal from '$lib/components/icons/EllipsisHorizontal.svelte';
import Cog6 from '$lib/components/icons/Cog6.svelte';
import AddTextContentModal from './KnowledgeBase/AddTextContentModal.svelte';
@ -51,12 +58,14 @@
let showAccessControlModal = false;
let savedAccessGrants = [];
let showResetConfirm = false;
let showSettings = false;
type DirectoryFileEntry = { path: string; filename: string; file: File };
type DirectoryManifestEntry = DirectoryFileEntry & { checksum: string; size: number };
type Knowledge = {
id: string;
user_id: string;
name: string;
description: string;
data: {
@ -125,6 +134,35 @@
return file;
};
const validateUpload = async (
count: number,
removedFileIds: string[] = [],
fileSize?: number
) => {
if (!knowledge) return false;
if (count === 0) return true;
const maxCount = $config?.file?.max_count;
if (maxCount != null && count > maxCount) {
toast.error(
$i18n.t('You can only upload a maximum of {{maxCount}} file(s) per batch.', { maxCount })
);
return false;
}
try {
await validateKnowledgeUpload(
localStorage.token,
knowledge.id,
count,
removedFileIds,
fileSize
);
return true;
} catch (error) {
toast.error(`${error}`);
return false;
}
};
const uploadWeb = async (urls) => {
const targetDirectoryId = currentDirectoryId;
if (!knowledge) {
@ -135,6 +173,7 @@
if (!Array.isArray(urls)) {
urls = [urls];
}
if (!(await validateUpload(urls.length))) return;
const newFileItems = urls.map((url) => ({
type: 'file',
@ -331,6 +370,14 @@
// Rows go up front because each upload waits until its file is processed.
const uploadFiles = async (files: File[]) => {
if (
!(await validateUpload(
files.length,
[],
files.reduce((size, file) => Math.max(size, file.size), 0)
))
)
return;
const uploads: { file: File; fileItem: { itemId: string; directory_id: string | null } }[] = [];
for (const file of files) {
const fileItem = createFileItem(file);
@ -529,6 +576,14 @@
const uploadDirectoryEntries = async (entries: DirectoryFileEntry[]) => {
if (!knowledge) return;
if (
!(await validateUpload(
entries.length,
[],
entries.reduce((size, entry) => Math.max(size, entry.file.size), 0)
))
)
return;
const targetDirectoryId = currentDirectoryId;
const targetPath = breadcrumbs.map((crumb) => crumb.name).join('/');
const getDirectoryUploadPath = (path: string) =>
@ -603,6 +658,7 @@
...diff.deleted.map((d: any) => d.file_id),
...diff.modified.map((m: any) => m.stale_file_id)
];
if (!(await validateUpload(diff.added.length + diff.modified.length, staleFileIds))) return;
if (staleFileIds.length > 0 || diff.rmdir.length > 0) {
syncing = $i18n.t('Removing {{count}} stale files...', { count: staleFileIds.length });
@ -779,6 +835,18 @@
}
}
// A mixed drop is one batch, even though its upload paths differ.
if (
!(await validateUpload(
looseFiles.length + directoryEntries.length,
[],
directoryEntries.reduce(
(size, entry) => Math.max(size, entry.file.size),
looseFiles.reduce((size, file) => Math.max(size, file.size), 0)
)
))
)
return;
await uploadFiles(looseFiles);
if (directoryEntries.length > 0) {
@ -908,13 +976,45 @@
on:click={() => goto('/workspace/knowledge')}
><ChevronLeft className="size-3" strokeWidth="2" />{$i18n.t('Back')}</button
>
<!-- Previous file count label: {$i18n.t('{{COUNT}} files')} -->
{#if knowledge.write_access}{#if $user?.permissions?.sharing?.knowledge || $user?.role === 'admin'}<AccessButton
on:click={() => {
savedAccessGrants = structuredClone(knowledge.access_grants ?? []);
showAccessControlModal = true;
}}
/>{/if}{:else}<span class="text-xs text-gray-500">{$i18n.t('Read Only')}</span>{/if}
<div class="flex items-center gap-2">
{#if knowledge.write_access}
{#if $user?.permissions?.sharing?.knowledge || $user?.role === 'admin'}
<AccessButton
on:click={() => {
savedAccessGrants = structuredClone(knowledge.access_grants ?? []);
showAccessControlModal = true;
}}
/>
{/if}
{:else}
<span class="text-xs text-gray-500">{$i18n.t('Read Only')}</span>
{/if}
{#if !isExternalKnowledge && (knowledge.user_id === $user?.id || $user?.role === 'admin')}
<Dropdown closeOnSelect align="end">
<button
type="button"
aria-label={$i18n.t('More Options')}
class="flex size-6 items-center justify-center rounded-lg text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800"
>
<EllipsisHorizontal className="size-4" />
</button>
<div slot="content">
<DropdownMenu className="min-w-36">
<button
type="button"
class="flex h-[1.6875rem] w-full items-center gap-2 rounded-xl px-2 text-xs hover:text-gray-900 dark:hover:text-gray-100"
on:click={() => {
showSettings = true;
}}
>
<Cog6 className="size-3.5" />
{$i18n.t('Settings')}
</button>
</DropdownMenu>
</div>
</Dropdown>
{/if}
</div>
</div>
<div class="shrink-0 px-1 pb-2.5">
<input
@ -1038,6 +1138,29 @@
{:else}<Spinner className="size-5" />{/if}
</div>
<Modal bind:show={showSettings} size="md" ariaLabel={$i18n.t('Settings')}>
{#if knowledge}
<KnowledgeEditor
modal
edit
{knowledge}
onBack={() => {
showSettings = false;
}}
onSaved={(saved) => {
if (!knowledge) return;
knowledge = {
...knowledge,
name: saved.name,
description: saved.description,
meta: saved.meta
};
showSettings = false;
}}
/>
{/if}
</Modal>
<ConfirmDialog
bind:show={showResetConfirm}
title={$i18n.t('Reset knowledge base?')}

View file

@ -6,55 +6,74 @@
const i18n = getContext('i18n');
import { user } from '$lib/stores';
import { createNewKnowledge } from '$lib/apis/knowledge';
import { createNewKnowledge, updateKnowledgeById } from '$lib/apis/knowledge';
import AccessControl from '../common/AccessControl.svelte';
import Spinner from '$lib/components/common/Spinner.svelte';
import XMark from '$lib/components/icons/XMark.svelte';
import ChevronRight from '$lib/components/icons/ChevronRight.svelte';
export let modal = false;
export let edit = false;
/** @type {{ id: string; name: string; description: string; access_grants?: any[]; meta?: any } | null} */
export let knowledge = null;
/** @type {() => void | Promise<void>} */
export let onBack = () => goto('/workspace/knowledge');
/** @type {(knowledge: { id: string }) => void | Promise<void>} */
export let onCreated = (knowledge) => goto(`/workspace/knowledge/${knowledge.id}`);
/** @type {(knowledge: { id: string; name: string; description: string; meta?: any; access_grants?: any[] }) => void | Promise<void>} */
export let onSaved = (knowledge) => goto(`/workspace/knowledge/${knowledge.id}`);
let loading = false;
let name = '';
let description = '';
let name = knowledge?.name ?? '';
let description = knowledge?.description ?? '';
/** @type {{ id?: string; principal_type: 'user' | 'group'; principal_id: string; permission: 'read' | 'write' }[]} */
let accessGrants = [];
let showAdvanced = false;
/** @type {number | undefined} */
let maxFileCount = knowledge?.meta?.max_file_count || undefined;
/** @type {number | undefined} */
let maxFileSize = knowledge?.meta?.max_file_size || undefined;
const submitHandler = async () => {
loading = true;
if (loading) return;
if (name.trim() === '' || description.trim() === '') {
toast.error($i18n.t('Please fill in all fields.'));
name = '';
description = '';
loading = false;
return;
}
const res = await createNewKnowledge(localStorage.token, name, description, accessGrants).catch(
(e) => {
toast.error(`${e}`);
loading = true;
const meta = {
max_file_count: maxFileCount ?? null,
max_file_size: maxFileSize ?? null
};
try {
const res =
edit && knowledge
? await updateKnowledgeById(localStorage.token, knowledge.id, {
name,
description,
meta
})
: await createNewKnowledge(localStorage.token, name, description, accessGrants, meta);
if (res) {
toast.success($i18n.t(edit ? 'Saved' : 'Knowledge created successfully.'));
await onSaved(res);
}
);
if (res) {
toast.success($i18n.t('Knowledge created successfully.'));
await onCreated(res);
} catch (error) {
toast.error(`${error}`);
} finally {
loading = false;
}
loading = false;
};
</script>
<div class="w-full max-h-full">
{#if modal}
<div class="flex justify-between items-center dark:text-gray-100 px-5 pt-4 pb-2">
<h3 class="text-base font-normal">{$i18n.t('Create a knowledge base')}</h3>
<h3 class="text-base font-normal">
{$i18n.t(edit ? 'Settings' : 'Create a knowledge base')}
</h3>
<button
class="self-center shrink-0 ml-2"
aria-label={$i18n.t('Close')}
@ -100,7 +119,7 @@
<div class="w-full flex flex-col {modal ? '' : 'justify-center'}">
{#if !modal}
<div class=" text-2xl font-normal mb-2.5">
{$i18n.t('Create a knowledge base')}
{$i18n.t(edit ? 'Settings' : 'Create a knowledge base')}
</div>
{/if}
@ -143,18 +162,71 @@
</div>
</div>
<div class="mt-2">
<AccessControl
bind:accessGrants
accessRoles={['read', 'write']}
share={$user?.permissions?.sharing?.knowledge || $user?.role === 'admin'}
sharePublic={$user?.permissions?.sharing?.public_knowledge || $user?.role === 'admin'}
shareUsers={($user?.permissions?.access_grants?.allow_users ?? true) ||
$user?.role === 'admin'}
allowGroups={($user?.permissions?.access_grants?.allow_groups ?? true) ||
$user?.role === 'admin'}
/>
</div>
{#if !edit}
<div class="mt-2">
<AccessControl
bind:accessGrants
accessRoles={['read', 'write']}
share={$user?.permissions?.sharing?.knowledge || $user?.role === 'admin'}
sharePublic={$user?.permissions?.sharing?.public_knowledge || $user?.role === 'admin'}
shareUsers={($user?.permissions?.access_grants?.allow_users ?? true) ||
$user?.role === 'admin'}
allowGroups={($user?.permissions?.access_grants?.allow_groups ?? true) ||
$user?.role === 'admin'}
/>
</div>
{/if}
<button
type="button"
class="flex items-center gap-1 text-xs text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 transition mt-3 self-start"
aria-expanded={showAdvanced}
aria-controls="knowledge-advanced"
on:click={() => (showAdvanced = !showAdvanced)}
>
<ChevronRight className={`size-3 transition-transform ${showAdvanced ? 'rotate-90' : ''}`} />
{$i18n.t('Advanced')}
</button>
{#if showAdvanced}
<div id="knowledge-advanced" class="mt-3 space-y-3">
<div class="grid grid-cols-1 sm:grid-cols-2 gap-3">
<div>
<label for="knowledge-max-file-count" class="block text-xs text-gray-500 mb-2">
{$i18n.t('File count limit')}
</label>
<input
id="knowledge-max-file-count"
type="number"
min="1"
step="1"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
placeholder={$i18n.t('Use global setting')}
bind:value={maxFileCount}
/>
<p class="mt-1 text-xs text-gray-500">{$i18n.t('Maximum total files.')}</p>
</div>
<div>
<label for="knowledge-max-file-size" class="block text-xs text-gray-500 mb-2"
>{$i18n.t('File size limit (MB)')}</label
>
<input
id="knowledge-max-file-size"
type="number"
min="1"
step="1"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
placeholder={$i18n.t('Use global setting')}
bind:value={maxFileSize}
/>
<p class="mt-1 text-xs text-gray-500">{$i18n.t('Maximum size per file.')}</p>
</div>
</div>
<p class="text-xs text-gray-500">
{$i18n.t('Global limits always apply.')}
</p>
</div>
{/if}
<div class="flex justify-end {modal ? 'pt-3 gap-2' : 'mt-2'}">
{#if modal}
@ -177,7 +249,7 @@
type="submit"
disabled={loading}
>
<div class=" self-center font-normal">{$i18n.t('Create Knowledge')}</div>
<div class=" self-center font-normal">{$i18n.t(edit ? 'Save' : 'Create Knowledge')}</div>
{#if loading}
<div class="shrink-0">

View file

@ -1,4 +1,11 @@
{
"Maximum total files.": "",
"Maximum size per file.": "",
"File count limit": "",
"File size limit (MB)": "",
"Use global setting": "",
"Global limits always apply.": "",
"You can only upload a maximum of {{maxCount}} file(s) per batch.": "",
"- usage not recorded": "- usage not recorded",
"- usage not recorded · * partial usage": "- usage not recorded · * partial usage",
"-1 for no limit, or a positive integer for a specific limit": "",
@ -2870,8 +2877,12 @@
"settings.admin.documents.imageCompressionWidth.label": "Image Compression Width",
"settings.admin.documents.markdownHeaderTextSplitter.description": "Split documents by markdown headers before character or token splitting.",
"settings.admin.documents.markdownHeaderTextSplitter.label": "Markdown Header Text Splitter",
"settings.admin.documents.maxUploadCount.description": "Maximum number of files that can be used at once in chat.",
"settings.admin.documents.maxUploadCount.description": "Maximum number of files that can be used at once in chat or uploaded in one knowledge base batch.",
"settings.admin.documents.maxUploadCount.label": "Max Upload Count",
"settings.admin.documents.knowledgeMaxSize.label": "Knowledge File Size Limit",
"settings.admin.documents.knowledgeMaxSize.description": "Maximum file size in MB per knowledge base. Leave empty for no limit.",
"settings.admin.documents.knowledgeMaxCount.label": "Knowledge File Count Limit",
"settings.admin.documents.knowledgeMaxCount.description": "Maximum total files per knowledge base. Leave empty for no limit.",
"settings.admin.documents.maxUploadSize.description": "Maximum file size in MB. Leave empty for unlimited.",
"settings.admin.documents.maxUploadSize.label": "Max Upload Size",
"settings.admin.documents.mineruApiKey.description": "API key used for MinerU cloud mode.",