scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-08-28 05:27:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refac

Browse source
This commit is contained in:
Timothy Jaeryang Baek 2026-06-23 23:13:32 +02:00
parent b1c2536ed2
commit 15c7e37438
15 changed files with 2929 additions and 394 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
backend/open_webui/config.py
View file

@ -857,6 +857,16 @@ EXTERNAL_DOCUMENT_LOADER_URL = os.getenv('EXTERNAL_DOCUMENT_LOADER_URL', '')
EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '')
external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '')
try:
external_document_loader_headers = json.loads(external_document_loader_headers)
except json.JSONDecodeError:
external_document_loader_headers = {}
if not isinstance(external_document_loader_headers, dict):
external_document_loader_headers = {}
EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers
TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998')
DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001')
@ -2652,6 +2662,7 @@ DEFAULT_CONFIG = {
'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS,
'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL,
'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY,
'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS,
'rag.tika_server_url': TIKA_SERVER_URL,
'rag.docling_server_url': DOCLING_SERVER_URL,
'rag.docling_api_key': DOCLING_API_KEY,

30
backend/open_webui/models/knowledge.py
View file

@ -286,6 +286,17 @@ class KnowledgeTable:
elif view_option == 'shared':
stmt = stmt.filter(Knowledge.user_id != user_id)
source = filter.get('source')
if source == 'external':
stmt = stmt.filter(Knowledge.meta['source'].as_string() == 'external')
elif source == 'local':
stmt = stmt.filter(
or_(
Knowledge.meta.is_(None),
Knowledge.meta['source'].as_string() != 'external',
)
)
stmt = AccessGrants.has_permission_filter(
db=db,
query=stmt,
@ -765,6 +776,25 @@ class KnowledgeTable:
log.exception(e)
return None
async def update_knowledge_meta_by_id(
self, id: str, meta: dict, db: Optional[AsyncSession] = None
) -> Optional[KnowledgeModel]:
try:
async with get_async_db_context(db) as db:
await db.execute(
update(Knowledge)
.filter_by(id=id)
.values(
meta=meta,
updated_at=int(time.time()),
)
)
await db.commit()
return await self.get_knowledge_by_id(id=id, db=db)
except Exception as e:
log.exception(e)
return None
async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
try:
async with get_async_db_context(db) as db:

378
backend/open_webui/retrieval/external.py Normal file
View file

@ -0,0 +1,378 @@
import asyncio
import logging
import re
import time
from typing import Any, Optional
from open_webui.models.config import Config
from open_webui.models.knowledge import KnowledgeModel
log = logging.getLogger(__name__)
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
IDENTIFIER_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
async def _get_external_connection(connection_id: str) -> Optional[dict]:
connections = await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
return next((connection for connection in connections if connection.get('id') == connection_id), None)
def _get_path(data: Any, path: Optional[str], default=None):
if not path:
return default
value = data
for part in path.split('.'):
if isinstance(value, dict):
value = value.get(part, default)
else:
return default
return value
def _normalize_result(result: dict, mapping: dict, knowledge: KnowledgeModel, distance: Optional[float] = None) -> dict:
content = _get_path(result, mapping.get('content_field', 'content'), '')
title = _get_path(result, mapping.get('title_field', 'title'), None)
source = _get_path(result, mapping.get('source_field', 'source'), None)
url = _get_path(result, mapping.get('url_field', 'url'), None)
document_id = _get_path(result, mapping.get('document_id_field', 'document_id'), None)
page = _get_path(result, mapping.get('page_field', 'page'), None)
metadata = _get_path(result, mapping.get('metadata_field', 'metadata'), {}) or {}
score = _get_path(result, mapping.get('score_field', 'score'), distance)
if not isinstance(metadata, dict):
metadata = {'external_metadata': metadata}
source_name = source or title or metadata.get('source') or metadata.get('name') or knowledge.name
metadata.update(
{
'name': title or source_name,
'source': source_name,
'url': url,
'file_id': document_id or f'external-{knowledge.id}',
'knowledge_id': knowledge.id,
'knowledge_name': knowledge.name,
'external': True,
}
)
if page is not None:
metadata['page'] = page
if document_id is not None:
metadata['document_id'] = document_id
return {
'content': content,
'metadata': metadata,
'distance': score,
}
def _source_config(knowledge: KnowledgeModel) -> dict:
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
return source.get('config') or {}
def _root_field(path: Optional[str]) -> Optional[str]:
if not path:
return None
return path.split('.')[0]
def _safe_identifier(value: str, label: str) -> str:
if not value or not IDENTIFIER_RE.match(value):
raise RuntimeError(f'Invalid {label}')
return value
async def _retrieve_qdrant(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
from qdrant_client import QdrantClient
except ImportError as exc:
raise RuntimeError('qdrant-client is not installed') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('External source collection is not configured')
source_config = _source_config(knowledge)
vector_field = source_config.get('vector_field') or None
vector = await embedding_function(query)
def _search():
client = QdrantClient(
url=connection.get('endpoint'),
api_key=(auth_config or {}).get('api_key'),
timeout=config.get('timeout') or 30,
)
return client.query_points(
collection_name=collection_name,
query=vector,
using=vector_field,
limit=count,
)
response = await asyncio.to_thread(_search)
mapping = {
'content_field': source_config.get('content_field') or 'payload.text',
'metadata_field': source_config.get('metadata_field') or 'payload.metadata',
'document_id_field': source_config.get('document_id_field') or 'id',
'score_field': 'score',
}
normalized = []
for point in response.points:
normalized.append(_normalize_result(point.model_dump(), mapping, knowledge, distance=point.score))
return normalized
async def _retrieve_milvus(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
from pymilvus import MilvusClient
except ImportError as exc:
raise RuntimeError('pymilvus is not installed') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('Milvus collection is not configured')
source_config = _source_config(knowledge)
vector_field = source_config.get('vector_field') or 'vector'
content_field = source_config.get('content_field') or 'data.text'
metadata_field = source_config.get('metadata_field') or 'metadata'
vector = await embedding_function(query)
def _search():
client_kwargs = {
'uri': connection.get('endpoint'),
}
token = (auth_config or {}).get('api_key') or (auth_config or {}).get('token')
if token:
client_kwargs['token'] = token
if config.get('db_name'):
client_kwargs['db_name'] = config.get('db_name')
client = MilvusClient(**client_kwargs)
output_fields = {
field
for field in (
_root_field(content_field),
_root_field(metadata_field),
_root_field(source_config.get('document_id_field')),
)
if field and field != vector_field
}
kwargs = {
'collection_name': collection_name,
'data': [vector],
'anns_field': vector_field,
'limit': count,
'output_fields': list(output_fields),
}
return client.search(**kwargs)
response = await asyncio.to_thread(_search)
mapping = {
'content_field': content_field,
'metadata_field': metadata_field,
'document_id_field': source_config.get('document_id_field') or 'id',
'score_field': 'distance',
}
normalized = []
for hit in (response[0] if response else []):
item = dict(hit)
entity = item.get('entity') or {}
result = {
**entity,
'id': item.get('id') or entity.get('id'),
'distance': item.get('distance'),
}
normalized.append(_normalize_result(result, mapping, knowledge, distance=item.get('distance')))
return normalized
async def _retrieve_pgvector(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]:
try:
import psycopg
from pgvector.psycopg import register_vector
from psycopg.rows import dict_row
except ImportError as exc:
raise RuntimeError('psycopg and pgvector are required for pgvector retrieval') from exc
if not embedding_function:
raise RuntimeError('Embedding function is not configured')
config = connection.get('config') or {}
external = (knowledge.meta or {}).get('external', {})
source = external.get('source') or {}
collection_name = source.get('name')
if not collection_name:
raise RuntimeError('pgvector collection is not configured')
source_config = _source_config(knowledge)
table_name = source_config.get('table_name') or 'document_chunk'
collection_field = source_config.get('collection_field') or 'collection_name'
content_field = source_config.get('content_field') or 'text'
vector_field = source_config.get('vector_field') or 'vector'
metadata_field = source_config.get('metadata_field') or 'vmetadata'
document_id_field = source_config.get('document_id_field') or 'id'
vector = await embedding_function(query)
def _search():
from psycopg import sql
table_identifier = sql.SQL('.').join(
sql.Identifier(_safe_identifier(part, 'table name')) for part in table_name.split('.')
)
collection_identifier = sql.Identifier(_safe_identifier(collection_field, 'collection field'))
content_identifier = sql.Identifier(_safe_identifier(content_field, 'content field'))
vector_identifier = sql.Identifier(_safe_identifier(vector_field, 'vector field'))
document_id_identifier = sql.Identifier(_safe_identifier(document_id_field, 'document id field'))
metadata_sql = (
sql.Identifier(_safe_identifier(metadata_field, 'metadata field'))
if metadata_field
else sql.SQL("'{}'::jsonb")
)
with psycopg.connect(
connection.get('endpoint'),
row_factory=dict_row,
connect_timeout=config.get('timeout') or 30,
) as conn:
register_vector(conn)
with conn.cursor() as cur:
cur.execute(
sql.SQL(
"""
SELECT {document_id} AS id,
{content} AS content,
{metadata} AS metadata,
{vector_column} <=> %s AS distance
FROM {table_name}
WHERE {collection} = %s
ORDER BY distance ASC
LIMIT %s
"""
).format(
document_id=document_id_identifier,
content=content_identifier,
metadata=metadata_sql,
vector_column=vector_identifier,
table_name=table_identifier,
collection=collection_identifier,
),
(vector, collection_name, count),
)
return cur.fetchall()
rows = await asyncio.to_thread(_search)
mapping = {
'content_field': 'content',
'metadata_field': 'metadata',
'document_id_field': 'id',
'score_field': 'distance',
}
return [_normalize_result(row, mapping, knowledge, distance=row.get('distance')) for row in rows]
async def retrieve_external_knowledge(
request,
knowledge: KnowledgeModel,
queries: list[str],
count: int,
user=None,
) -> dict:
external = (knowledge.meta or {}).get('external', {})
connection_id = external.get('connection_id')
if not connection_id:
raise RuntimeError('External knowledge connection is not configured')
connection = await _get_external_connection(connection_id)
if not connection:
raise RuntimeError('External knowledge connection not found')
return await retrieve_external_knowledge_for_connection(request, knowledge, connection, queries, count, user=user)
async def retrieve_external_knowledge_for_connection(
request,
knowledge: KnowledgeModel,
connection: dict,
queries: list[str],
count: int,
user=None,
) -> dict:
auth_config = connection.get('auth_config') or {}
if not connection.get('enabled', True):
raise RuntimeError('External knowledge connection is disabled')
started_at = time.monotonic()
chunks = []
provider = (connection.get('provider') or '').lower()
for query in queries:
if provider == 'qdrant':
chunks.extend(
await _retrieve_qdrant(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
elif provider == 'milvus':
chunks.extend(
await _retrieve_milvus(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
elif provider == 'pgvector':
chunks.extend(
await _retrieve_pgvector(
connection,
auth_config,
knowledge,
query,
count,
getattr(request.app.state, 'EMBEDDING_FUNCTION', None),
)
)
else:
raise RuntimeError(f'Unsupported external knowledge provider: {connection.get("provider")}')
chunks = chunks[:count]
log.info(
'external_knowledge_retrieval knowledge_id=%s connection_id=%s provider=%s user_id=%s latency_ms=%s result_count=%s',
knowledge.id,
connection.get('id'),
connection.get('provider'),
getattr(user, 'id', None),
round((time.monotonic() - started_at) * 1000),
len(chunks),
)
return {
'documents': [[chunk['content'] for chunk in chunks]],
'metadatas': [[chunk['metadata'] for chunk in chunks]],
'distances': [[chunk['distance'] for chunk in chunks]],
}

109
backend/open_webui/retrieval/utils.py
View file

@ -43,6 +43,7 @@ from open_webui.models.config import Config
from open_webui.models.users import UserModel
from open_webui.retrieval.loaders.youtube import YoutubeLoader
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.retrieval.external import retrieve_external_knowledge
from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT
from open_webui.retrieval.vector.main import GetResult, SearchResult
from open_webui.retrieval.web.utils import get_web_loader
@ -84,6 +85,7 @@ LOADER_CONFIG_KEYS = {
'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'TIKA_SERVER_URL': 'rag.tika_server_url',
'DOCLING_SERVER_URL': 'rag.docling_server_url',
'DOCLING_API_KEY': 'rag.docling_api_key',
@ -130,18 +132,16 @@ def build_loader_from_config(request, config: dict):
"""Build a Loader instance with the admin's configured extraction engine settings."""
from open_webui.retrieval.loaders.main import Loader
loader_config = {
key: config.get(key)
for key in LOADER_CONFIG_KEYS
if key.isupper()
}
loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
return Loader(
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
)
def _extract_text_from_binary_response(request, response: requests.Response, url: str, loader_config: dict) -> tuple[str, list]:
def _extract_text_from_binary_response(
request, response: requests.Response, url: str, loader_config: dict
) -> tuple[str, list]:
"""Download response body to a temp file and extract text using the Loader pipeline."""
import mimetypes
import tempfile
@ -774,11 +774,7 @@ async def query_collection_with_hybrid_search(
return result
native_task_results = await asyncio.gather(
*[
process_native_query(collection_name, query)
for collection_name in collection_names
for query in queries
]
*[process_native_query(collection_name, query) for collection_name in collection_names for query in queries]
)
if native_task_results and all(result is not None for result in native_task_results):
return merge_and_sort_query_results(native_task_results, k=k)
@ -1485,50 +1481,61 @@ async def get_sources_from_items(
permission='read',
)
):
if item.get('context') == 'full' or bypass_embedding_and_retrieval:
if knowledge_base and (
user.role == 'admin'
or knowledge_base.user_id == user.id
or await AccessGrants.has_access(
user_id=user.id,
resource_type='knowledge',
resource_id=knowledge_base.id,
permission='read',
)
):
files = await Knowledges.get_files_by_id(knowledge_base.id)
if (knowledge_base.meta or {}).get('source') == 'external':
query_result = await retrieve_external_knowledge(
request,
knowledge_base,
queries=queries,
count=k,
user=user,
)
extracted_collections.append(knowledge_base.id)
documents = []
metadatas = []
for file in files:
documents.append(file.data.get('content', ''))
metadatas.append(
{
'file_id': file.id,
'name': file.filename,
'source': file.filename,
}
)
query_result = {
'documents': [documents],
'metadatas': [metadatas],
}
else:
if item.get('legacy'):
if BYPASS_RETRIEVAL_ACCESS_CONTROL:
collection_names = item.get('collection_names', [])
else:
# Legacy KB: item.collection_names is client-supplied.
# Validate against the KB's actual files to prevent
# cross-tenant collection name substitution.
if item.get('context') == 'full' or bypass_embedding_and_retrieval:
if knowledge_base and (
user.role == 'admin'
or knowledge_base.user_id == user.id
or await AccessGrants.has_access(
user_id=user.id,
resource_type='knowledge',
resource_id=knowledge_base.id,
permission='read',
)
):
files = await Knowledges.get_files_by_id(knowledge_base.id)
owned_names = {f'file-{f.id}' for f in files}
owned_names.add(knowledge_base.id)
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
collection_names = valid_names if valid_names else [knowledge_base.id]
documents = []
metadatas = []
for file in files:
documents.append(file.data.get('content', ''))
metadatas.append(
{
'file_id': file.id,
'name': file.filename,
'source': file.filename,
}
)
query_result = {
'documents': [documents],
'metadatas': [metadatas],
}
else:
collection_names.append(item['id'])
if item.get('legacy'):
if BYPASS_RETRIEVAL_ACCESS_CONTROL:
collection_names = item.get('collection_names', [])
else:
# Legacy KB: item.collection_names is client-supplied.
# Validate against the KB's actual files to prevent
# cross-tenant collection name substitution.
files = await Knowledges.get_files_by_id(knowledge_base.id)
owned_names = {f'file-{f.id}' for f in files}
owned_names.add(knowledge_base.id)
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
collection_names = valid_names if valid_names else [knowledge_base.id]
else:
collection_names.append(item['id'])
elif item.get('docs'):
# BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL

619
backend/open_webui/routers/knowledge.py
View file

@ -3,6 +3,8 @@ from __future__ import annotations
import asyncio
import io
import logging
import time
import uuid
import zipfile
from typing import List, Optional
from urllib.parse import quote
@ -27,6 +29,7 @@ from open_webui.models.knowledge import (
)
from open_webui.models.models import ModelForm, Models
from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT
from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection
from open_webui.routers.retrieval import (
BatchProcessFilesForm,
ProcessFileForm,
@ -110,6 +113,17 @@ class KnowledgeAccessListResponse(BaseModel):
total: int
def is_external_knowledge(knowledge) -> bool:
return (knowledge.meta or {}).get('source') == 'external'
def external_knowledge_error():
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='External knowledge bases are read-only.',
)
@router.get('/', response_model=KnowledgeAccessListResponse)
async def get_knowledge_bases(
page: int | None = 1,
@ -163,6 +177,7 @@ async def get_knowledge_bases(
async def search_knowledge_bases(
query: str | None = None,
view_option: str | None = None,
source: str | None = None,
page: int | None = 1,
user=Depends(get_verified_user),
db: AsyncSession = Depends(get_async_session),
@ -176,6 +191,8 @@ async def search_knowledge_bases(
filter['query'] = query
if view_option:
filter['view_option'] = view_option
if source in {'local', 'external'}:
filter['source'] = source
groups = await Groups.get_groups_by_member_id(user.id, db=db)
user_group_ids = {group.id for group in groups}
@ -379,6 +396,574 @@ async def reindex_knowledge_base_metadata_embeddings(
return {'total': len(knowledge_bases), 'success': success_count}
############################
# External Knowledge Sources
############################
class ExternalKnowledgeSourceForm(BaseModel):
type: str = 'collection'
name: str
config: Optional[dict] = None
class ExternalKnowledgeCreateForm(BaseModel):
name: str
description: str = ''
connection_id: str
source: ExternalKnowledgeSourceForm
access_grants: Optional[list[dict]] = None
class ExternalKnowledgeSourceCreateForm(BaseModel):
name: str
description: str = ''
connection: ExternalKnowledgeConnectionForm
source: ExternalKnowledgeSourceForm
access_grants: Optional[list[dict]] = None
test_query: str
test_count: int = 5
class ExternalKnowledgeSourceUpdateForm(ExternalKnowledgeSourceCreateForm):
pass
class ExternalKnowledgeSourceTestForm(BaseModel):
connection_id: Optional[str] = None
connection: ExternalKnowledgeConnectionForm
source: ExternalKnowledgeSourceForm
query: str
count: int = 5
class ExternalKnowledgeRetrieveTestForm(BaseModel):
query: str
source: Optional[ExternalKnowledgeSourceForm] = None
count: int = 5
class ExternalKnowledgeConnectionForm(BaseModel):
name: str
provider: str
endpoint: str
auth_config: Optional[dict] = None
config: Optional[dict] = None
capabilities: Optional[dict] = None
enabled: bool = True
class ExternalKnowledgeConnectionListResponse(BaseModel):
items: list[dict]
total: int
EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections'
EXTERNAL_KNOWLEDGE_PROVIDERS = {'qdrant', 'milvus', 'pgvector'}
def _validate_external_connection_form(form_data: ExternalKnowledgeConnectionForm) -> tuple[str, dict]:
provider = form_data.provider.lower().strip()
if provider not in EXTERNAL_KNOWLEDGE_PROVIDERS:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='Unsupported external knowledge provider.',
)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source name is required.')
if not form_data.endpoint.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source endpoint is required.')
config = form_data.config or {}
allowed_config_keys = {'timeout'}
if provider == 'milvus':
allowed_config_keys.add('db_name')
return provider, {key: value for key, value in config.items() if key in allowed_config_keys}
def _external_auth_config(provider: str, incoming: Optional[dict], existing: Optional[dict] = None) -> dict:
if provider == 'pgvector':
return {}
return existing if incoming is None else incoming or {}
def _normalize_external_source(source: ExternalKnowledgeSourceForm, provider: str) -> ExternalKnowledgeSourceForm:
source.type = (source.type or 'collection').strip()
source.name = source.name.strip()
if source.type != 'collection':
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Only collection sources are supported.')
if not source.name:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Collection name is required.')
config = source.config or {}
allowed_keys = {'content_field', 'metadata_field', 'document_id_field'}
if provider in {'qdrant', 'milvus'}:
allowed_keys.add('vector_field')
if provider == 'pgvector':
allowed_keys.update({'table_name', 'collection_field', 'vector_field'})
normalized_config = {
key: value.strip() if isinstance(value, str) else value
for key, value in config.items()
if key in allowed_keys and value is not None and (not isinstance(value, str) or value.strip())
}
if not normalized_config.get('content_field'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Content field is required.')
if provider in {'milvus', 'pgvector'} and not normalized_config.get('vector_field'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Vector field is required.')
source.config = normalized_config
return source
def _sanitize_external_connection(connection: dict) -> dict:
sanitized = {**connection}
sanitized.pop('auth_config', None)
sanitized['auth_configured'] = bool(connection.get('auth_config'))
return sanitized
async def _get_external_connections() -> list[dict]:
return await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or []
async def _set_external_connections(connections: list[dict]) -> None:
await Config.upsert({EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY: connections})
def _external_connection_dict(form_data: ExternalKnowledgeConnectionForm, user_id: str, id: Optional[str] = None) -> dict:
provider, config = _validate_external_connection_form(form_data)
now = int(time.time())
return {
'id': id or str(uuid.uuid4()),
'name': form_data.name.strip(),
'provider': provider,
'endpoint': form_data.endpoint.strip(),
'auth_config': _external_auth_config(provider, form_data.auth_config),
'config': config,
'capabilities': form_data.capabilities or {'retrieve': True},
'health': None,
'enabled': form_data.enabled,
'created_by': user_id,
'created_at': now,
'updated_at': now,
}
def _external_connection_update_dict(
form_data: ExternalKnowledgeConnectionForm,
existing: dict,
) -> dict:
provider, config = _validate_external_connection_form(form_data)
return {
**existing,
'name': form_data.name.strip(),
'provider': provider,
'endpoint': form_data.endpoint.strip(),
'auth_config': _external_auth_config(provider, form_data.auth_config, existing.get('auth_config')) or {},
'config': config,
'capabilities': form_data.capabilities or {'retrieve': True},
'enabled': form_data.enabled,
'updated_at': int(time.time()),
}
async def _get_external_connection(id: str) -> Optional[dict]:
connections = await _get_external_connections()
return next((connection for connection in connections if connection.get('id') == id), None)
async def _count_external_connection_mappings(connection_id: str, db: Optional[AsyncSession] = None) -> int:
count = 0
for knowledge in await Knowledges.get_knowledge_bases(db=db):
if (knowledge.meta or {}).get('external', {}).get('connection_id') == connection_id:
count += 1
return count
@router.get('/external/connections', response_model=ExternalKnowledgeConnectionListResponse)
async def get_external_knowledge_connections(
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connections = [_sanitize_external_connection(connection) for connection in await _get_external_connections()]
return ExternalKnowledgeConnectionListResponse(items=connections, total=len(connections))
@router.post('/external/connections', response_model=dict)
async def create_external_knowledge_connection(
form_data: ExternalKnowledgeConnectionForm,
user=Depends(get_admin_user),
):
connections = await _get_external_connections()
connection = _external_connection_dict(form_data, user.id)
connections.append(connection)
await _set_external_connections(connections)
return _sanitize_external_connection(connection)
@router.get('/external/connections/{id}', response_model=dict)
async def get_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
return _sanitize_external_connection(connection)
@router.patch('/external/connections/{id}', response_model=dict)
async def update_external_knowledge_connection(
id: str,
form_data: ExternalKnowledgeConnectionForm,
user=Depends(get_admin_user),
):
connections = await _get_external_connections()
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == id), None)
if idx is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
connection = _external_connection_update_dict(form_data, connections[idx])
connections[idx] = connection
await _set_external_connections(connections)
return _sanitize_external_connection(connection)
@router.delete('/external/connections/{id}', response_model=bool)
async def delete_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if await _count_external_connection_mappings(id, db=db) > 0:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail='External connection is still used by knowledge bases.',
)
connections = [connection for connection in await _get_external_connections() if connection.get('id') != id]
await _set_external_connections(connections)
return True
@router.post('/external/connections/{id}/test', response_model=dict)
async def test_external_knowledge_connection(
id: str,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
health = {
'ok': bool(connection.get('enabled') and connection.get('endpoint')),
'provider': connection.get('provider'),
'checked_at': int(time.time()),
}
connections = await _get_external_connections()
for item in connections:
if item.get('id') == id:
item['health'] = health
item['updated_at'] = int(time.time())
break
await _set_external_connections(connections)
return health
async def _test_external_source_definition(
request: Request,
connection: dict,
source: ExternalKnowledgeSourceForm,
query: str,
count: int,
user,
) -> dict:
if not query.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query is required.')
source = _normalize_external_source(source, connection.get('provider'))
test_knowledge = KnowledgeResponse(
id='external-test',
user_id=user.id,
name=connection.get('name'),
description='',
meta={
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
},
access_grants=[],
created_at=int(time.time()),
updated_at=int(time.time()),
)
result = await retrieve_external_knowledge_for_connection(
request,
test_knowledge,
connection,
[query.strip()],
count,
user=user,
)
return {
'documents': result.get('documents', [[]])[0],
'metadatas': result.get('metadatas', [[]])[0],
'distances': result.get('distances', [[]])[0],
}
@router.post('/external/source/test', response_model=dict)
async def test_external_knowledge_source(
request: Request,
form_data: ExternalKnowledgeSourceTestForm,
user=Depends(get_admin_user),
):
if form_data.connection_id:
existing_connection = await _get_external_connection(form_data.connection_id)
if not existing_connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
connection = _external_connection_update_dict(form_data.connection, existing_connection)
else:
connection = _external_connection_dict(form_data.connection, user.id, id='external-test')
return await _test_external_source_definition(
request,
connection,
form_data.source,
form_data.query,
form_data.count,
user,
)
@router.post('/external/connections/{id}/retrieve-test', response_model=dict)
async def test_external_knowledge_retrieval(
request: Request,
id: str,
form_data: ExternalKnowledgeRetrieveTestForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
source = form_data.source or ExternalKnowledgeSourceForm(name='test', config={'content_field': 'payload.text'})
return await _test_external_source_definition(request, connection, source, form_data.query, form_data.count, user)
@router.post('/external/knowledge/create', response_model=KnowledgeResponse | None)
async def create_external_knowledge(
request: Request,
form_data: ExternalKnowledgeCreateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
connection = await _get_external_connection(form_data.connection_id)
if not connection:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
source = _normalize_external_source(form_data.source, connection.get('provider'))
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
knowledge = await Knowledges.insert_new_knowledge(
user.id,
KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not knowledge:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': form_data.connection_id,
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
return knowledge
@router.post('/external/source/create', response_model=KnowledgeResponse | None)
async def create_external_knowledge_source(
request: Request,
form_data: ExternalKnowledgeSourceCreateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
connection = _external_connection_dict(form_data.connection, user.id)
source = _normalize_external_source(form_data.source, connection.get('provider'))
test_result = await _test_external_source_definition(
request,
connection,
source,
form_data.test_query,
form_data.test_count,
user,
)
if not test_result.get('documents'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
connections = await _get_external_connections()
connections.append(connection)
await _set_external_connections(connections)
knowledge = await Knowledges.insert_new_knowledge(
user.id,
KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not knowledge:
connections = [item for item in await _get_external_connections() if item.get('id') != connection.get('id')]
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS)
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db)
await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description)
return knowledge
@router.patch('/external/source/{id}', response_model=KnowledgeResponse | None)
async def update_external_knowledge_source(
request: Request,
id: str,
form_data: ExternalKnowledgeSourceUpdateForm,
user=Depends(get_admin_user),
db: AsyncSession = Depends(get_async_session),
):
knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db)
if not knowledge or not is_external_knowledge(knowledge):
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND)
if not form_data.name.strip():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.')
connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
connections = await _get_external_connections()
idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == connection_id), None)
if idx is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.')
existing_connection = connections[idx]
connection = _external_connection_update_dict(form_data.connection, existing_connection)
source = _normalize_external_source(form_data.source, connection.get('provider'))
test_result = await _test_external_source_definition(
request,
connection,
source,
form_data.test_query,
form_data.test_count,
user,
)
if not test_result.get('documents'):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.')
form_data.access_grants = await filter_allowed_access_grants(
await Config.get('user.permissions'),
user.id,
user.role,
form_data.access_grants,
'sharing.public_knowledge',
)
connections[idx] = connection
await _set_external_connections(connections)
updated = await Knowledges.update_knowledge_by_id(
id=id,
form_data=KnowledgeForm(
name=form_data.name.strip(),
description=form_data.description,
access_grants=form_data.access_grants,
),
db=db,
)
if not updated:
connections[idx] = existing_connection
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
meta = {
'source': 'external',
'read_only': True,
'external': {
'connection_id': connection.get('id'),
'source': source.model_dump(),
'provider': connection.get('provider'),
'auth_mode': 'service_account',
'capabilities': {'retrieve': True},
},
}
updated = await Knowledges.update_knowledge_meta_by_id(id, meta, db=db)
if not updated:
connections[idx] = existing_connection
await _set_external_connections(connections)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT())
await embed_knowledge_base_metadata(request, id, updated.name, updated.description)
return updated
############################
# GetKnowledgeById
############################
@ -711,6 +1296,8 @@ async def add_file_to_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
@ -798,6 +1385,8 @@ async def update_file_from_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
@ -877,6 +1466,8 @@ async def remove_file_from_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
@ -1004,11 +1595,21 @@ async def delete_knowledge_by_id(
await Models.update_model_by_id(model.id, model_form, db=db)
# Clean up vector DB
try:
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
except Exception as e:
log.debug(e)
pass
if is_external_knowledge(knowledge):
connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id')
if connection_id:
connections = [
connection
for connection in await _get_external_connections()
if connection.get('id') != connection_id
]
await _set_external_connections(connections)
else:
try:
await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id)
except Exception as e:
log.debug(e)
pass
# Remove knowledge base embedding
await remove_knowledge_base_metadata_embedding(id)
@ -1035,6 +1636,8 @@ async def reset_knowledge_by_id(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
@ -1263,6 +1866,8 @@ async def add_files_to_knowledge_batch(
status_code=status.HTTP_400_BAD_REQUEST,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
@ -1380,6 +1985,8 @@ async def export_knowledge_by_id(id: str, user=Depends(get_admin_user), db: Asyn
status_code=status.HTTP_404_NOT_FOUND,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
files = await Knowledges.get_files_by_id(id, db=db)
@ -1441,6 +2048,8 @@ async def _verify_knowledge_write_access(id: str, user, db: AsyncSession):
status_code=status.HTTP_404_NOT_FOUND,
detail=ERROR_MESSAGES.NOT_FOUND,
)
if is_external_knowledge(knowledge):
external_knowledge_error()
if (
knowledge.user_id != user.id
and not await AccessGrants.has_access(

14
backend/open_webui/routers/retrieval.py
View file

@ -288,6 +288,7 @@ RETRIEVAL_CONFIG_KEYS = {
'ENABLE_WEB_SEARCH': 'rag.web.search.enable',
'EXA_API_KEY': 'rag.web.search.exa_api_key',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key',
'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url',
@ -632,6 +633,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY,
@ -846,6 +848,7 @@ class ConfigForm(BaseModel):
EXTERNAL_DOCUMENT_LOADER_URL: str | None = None
EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None
EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None
TIKA_SERVER_URL: str | None = None
DOCLING_SERVER_URL: str | None = None
@ -1021,6 +1024,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None
else config.EXTERNAL_DOCUMENT_LOADER_API_KEY
)
config.EXTERNAL_DOCUMENT_LOADER_HEADERS = (
form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS
if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None
else config.EXTERNAL_DOCUMENT_LOADER_HEADERS
)
config.TIKA_SERVER_URL = (
form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL
)
@ -1336,6 +1344,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT,
'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL,
'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY,
'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS,
'TIKA_SERVER_URL': config.TIKA_SERVER_URL,
'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL,
'DOCLING_API_KEY': config.DOCLING_API_KEY,
@ -1843,6 +1852,11 @@ async def process_file(
loader_config = await get_loader_config()
loader = build_loader_from_config(request, loader_config)
loader.user = user
loader.metadata = {
'file_id': file.id,
'file_name': file.filename,
'file_content_type': file.meta.get('content_type'),
}
docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path)
docs = [

56
backend/open_webui/tools/builtin.py
View file

@ -514,21 +514,14 @@ async def execute_code(
elif engine == 'jupyter':
from open_webui.utils.code_interpreter import execute_code_jupyter
jupyter_auth = await Config.get('code_interpreter.jupyter.auth')
output = await execute_code_jupyter(
await Config.get('code_interpreter.jupyter.url'),
code,
(
await Config.get('code_interpreter.jupyter.auth_token')
if jupyter_auth == 'token'
else None
),
(
await Config.get('code_interpreter.jupyter.auth_password')
if jupyter_auth == 'password'
else None
),
(await Config.get('code_interpreter.jupyter.auth_token') if jupyter_auth == 'token' else None),
(await Config.get('code_interpreter.jupyter.auth_password') if jupyter_auth == 'password' else None),
await Config.get('code_interpreter.jupyter.timeout'),
)
@ -2385,6 +2378,7 @@ async def query_knowledge_files(
from open_webui.models.files import Files
from open_webui.models.knowledge import Knowledges
from open_webui.models.notes import Notes
from open_webui.retrieval.external import retrieve_external_knowledge
from open_webui.retrieval.utils import query_collection
user_id = __user__.get('id')
@ -2396,6 +2390,7 @@ async def query_knowledge_files(
return json.dumps({'error': 'Embedding function not configured'})
collection_names = []
external_knowledges = []
note_results = [] # Notes aren't vectorized, handle separately
# If model has attached knowledge, use those
@ -2418,7 +2413,10 @@ async def query_knowledge_files(
user_group_ids=set(user_group_ids),
)
):
collection_names.append(item_id)
if (knowledge.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge)
else:
collection_names.append(item_id)
elif item_type == 'file':
# Individual file - use file-{id} as collection name
@ -2464,7 +2462,10 @@ async def query_knowledge_files(
user_group_ids=set(user_group_ids),
)
):
collection_names.append(knowledge_id)
if (knowledge.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge)
else:
collection_names.append(knowledge_id)
else:
# No model knowledge and no specific IDs - search all accessible KBs
result = await Knowledges.search_knowledge_bases(
@ -2477,7 +2478,11 @@ async def query_knowledge_files(
skip=0,
limit=50,
)
collection_names = [knowledge_base.id for knowledge_base in result.items]
for knowledge_base in result.items:
if (knowledge_base.meta or {}).get('source') == 'external':
external_knowledges.append(knowledge_base)
else:
collection_names.append(knowledge_base.id)
chunks = []
@ -2509,6 +2514,31 @@ async def query_knowledge_files(
chunk_info['distance'] = distances[idx]
chunks.append(chunk_info)
for knowledge in external_knowledges:
query_results = await retrieve_external_knowledge(
__request__,
knowledge,
queries=[query],
count=count,
user=type('UserContext', (), {'id': user_id, 'role': user_role})(),
)
documents = query_results.get('documents', [[]])[0]
metadatas = query_results.get('metadatas', [[]])[0]
distances = query_results.get('distances', [[]])[0]
for idx, doc in enumerate(documents):
metadata = metadatas[idx] if idx < len(metadatas) else {}
chunk_info = {
'content': doc,
'source': metadata.get('source', metadata.get('name', knowledge.name)),
'file_id': metadata.get('file_id', f'external-{knowledge.id}'),
'type': 'external',
'knowledge_id': knowledge.id,
}
if idx < len(distances):
chunk_info['distance'] = distances[idx]
chunks.append(chunk_info)
# Limit to requested count
chunks = chunks[:count]

302
src/lib/apis/knowledge/index.ts
View file

@ -38,6 +38,304 @@ export const createNewKnowledge = async (
return res;
};
export const getExternalKnowledgeConnections = async (token: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
method: 'GET',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledgeConnection = async (token: string, connection: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(connection)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const updateExternalKnowledgeConnection = async (
token: string,
id: string,
connection: object
) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
method: 'PATCH',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(connection)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const deleteExternalKnowledgeConnection = async (token: string, id: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeConnection = async (token: string, id: string) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/test`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
}
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeRetrieval = async (
token: string,
id: string,
payload: object
) => {
let error = null;
const res = await fetch(
`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/retrieve-test`,
{
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
}
)
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const testExternalKnowledgeSource = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/test`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledgeSource = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const updateExternalKnowledgeSource = async (token: string, id: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/${id}`, {
method: 'PATCH',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const createExternalKnowledge = async (token: string, payload: object) => {
let error = null;
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/knowledge/create`, {
method: 'POST',
headers: {
Accept: 'application/json',
'Content-Type': 'application/json',
authorization: `Bearer ${token}`
},
body: JSON.stringify(payload)
})
.then(async (res) => {
if (!res.ok) throw await res.json();
return res.json();
})
.catch((err) => {
error = err.detail;
console.error(err);
return null;
});
if (error) {
throw error;
}
return res;
};
export const getKnowledgeBases = async (token: string = '', page: number | null = null) => {
let error = null;
@ -76,13 +374,15 @@ export const searchKnowledgeBases = async (
token: string = '',
query: string | null = null,
viewOption: string | null = null,
page: number | null = null
page: number | null = null,
source: string | null = null
) => {
let error = null;
const searchParams = new URLSearchParams();
if (query) searchParams.append('query', query);
if (viewOption) searchParams.append('view_option', viewOption);
if (source) searchParams.append('source', source);
if (page) searchParams.append('page', page.toString());
const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, {

1
src/lib/apis/retrieval/index.ts
View file

@ -54,6 +54,7 @@ type RAGConfigForm = {
PDF_EXTRACT_IMAGES?: boolean;
ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean;
ENABLE_ONEDRIVE_INTEGRATION?: boolean;
EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record<string, string>;
chunk?: ChunkConfigForm;
content_extraction?: ContentExtractConfigForm;
web_loader_ssl_verification?: boolean;

16
src/lib/components/admin/Settings.svelte
View file

@ -155,7 +155,21 @@
id: 'integrations',
title: 'Integrations',
route: '/admin/settings/integrations',
keywords: ['tools', 'integrations', 'plugins', 'extensions', 'functions', 'openapi', 'server']
keywords: [
'tools',
'integrations',
'plugins',
'extensions',
'functions',
'openapi',
'server',
'knowledge',
'vector db',
'qdrant',
'rag',
'retrieval',
'sources'
]
},
{
id: 'documents',

111
src/lib/components/admin/Settings/Documents.svelte
View file

@ -37,6 +37,7 @@
let showResetConfirm = false;
let showResetUploadDirConfirm = false;
let showReindexConfirm = false;
let showExternalDocumentLoaderHeadersHint = false;
let RAG_EMBEDDING_ENGINE = '';
let RAG_EMBEDDING_MODEL = '';
@ -149,6 +150,21 @@
toast.error($i18n.t('External Document Loader URL required.'));
return;
}
if (
RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external' &&
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS
) {
try {
const headers = JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS);
if (headers === null || typeof headers !== 'object' || Array.isArray(headers)) {
throw new Error('Headers must be a valid JSON object');
}
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS = JSON.stringify(headers, null, 2);
} catch (error) {
toast.error($i18n.t('Headers must be a valid JSON object'));
return;
}
}
if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') {
toast.error($i18n.t('Tika Server URL required.'));
return;
@ -241,6 +257,11 @@
typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== ''
? JSON.parse(RAGConfig.DOCLING_PARAMS)
: {},
EXTERNAL_DOCUMENT_LOADER_HEADERS:
typeof RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'string' &&
RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS.trim() !== ''
? JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS)
: {},
MINERU_PARAMS:
typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
? JSON.parse(RAGConfig.MINERU_PARAMS)
@ -289,6 +310,13 @@
? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
: config.MINERU_PARAMS;
config.EXTERNAL_DOCUMENT_LOADER_HEADERS =
typeof config.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'object'
? Object.keys(config.EXTERNAL_DOCUMENT_LOADER_HEADERS ?? {}).length > 0
? JSON.stringify(config.EXTERNAL_DOCUMENT_LOADER_HEADERS, null, 2)
: ''
: config.EXTERNAL_DOCUMENT_LOADER_HEADERS;
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
RAGConfig = config;
@ -581,17 +609,78 @@
</div>
</div>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'}
<div class="my-0.5 flex gap-2 pr-2">
<input
class="flex-1 w-full text-sm bg-transparent outline-hidden"
placeholder={$i18n.t('Enter External Document Loader URL')}
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL}
/>
<SensitiveInput
placeholder={$i18n.t('Enter External Document Loader API Key')}
required={false}
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY}
/>
<div class="my-0.5 flex flex-col gap-2 pr-2">
<div class="flex gap-2">
<input
class="flex-1 w-full text-sm bg-transparent outline-hidden"
placeholder={$i18n.t('Enter External Document Loader URL')}
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_URL}
/>
<SensitiveInput
placeholder={$i18n.t('Enter External Document Loader API Key')}
required={false}
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_API_KEY}
/>
</div>
<div class="flex flex-col">
<div class="mb-0.5 text-xs text-gray-500">{$i18n.t('Headers')}</div>
<Tooltip
content={$i18n.t(
'Enter additional headers in JSON format (e.g. {"X-Custom-Header": "value"}'
)}
>
<Textarea
className="w-full text-sm outline-hidden"
bind:value={RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS}
placeholder={$i18n.t('Enter additional headers in JSON format')}
required={false}
/>
</Tooltip>
<button
type="button"
class="mt-1 flex items-center gap-1 text-xs text-gray-500 dark:text-gray-400 hover:text-gray-700 dark:hover:text-gray-200 transition w-fit"
on:click={() =>
(showExternalDocumentLoaderHeadersHint =
!showExternalDocumentLoaderHeadersHint)}
>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-3 h-3 transition-transform {showExternalDocumentLoaderHeadersHint
? 'rotate-90'
: ''}"
>
<path
fill-rule="evenodd"
d="M7.21 14.77a.75.75 0 01.02-1.06L11.168 10 7.23 6.29a.75.75 0 111.04-1.08l4.5 4.25a.75.75 0 010 1.08l-4.5 4.25a.75.75 0 01-1.06-.02z"
clip-rule="evenodd"
/>
</svg>
{$i18n.t('Header variables')}
</button>
{#if showExternalDocumentLoaderHeadersHint}
<div class="mt-1 text-xs text-gray-500 dark:text-gray-400 leading-5">
<div>
{$i18n.t('No additional headers are sent unless configured.')}
</div>
<div>
{$i18n.t('Example')}:
<code class="text-gray-700 dark:text-gray-300"
>{'{"X-OpenWebUI-File-Id": "{{FILE_ID}}"}'}</code
>
</div>
<div>
{$i18n.t('Available variables')}:
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_ID}}'}</code>,
<code class="text-gray-700 dark:text-gray-300">{'{{FILE_NAME}}'}</code>,
<code class="text-gray-700 dark:text-gray-300"
>{'{{FILE_CONTENT_TYPE}}'}</code
>
</div>
</div>
{/if}
</div>
</div>
{:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika'}
<div class="flex w-full mt-1">

883
src/lib/components/admin/Settings/ExternalKnowledge.svelte Normal file
View file

@ -0,0 +1,883 @@
<script lang="ts">
import { getContext, onMount } from 'svelte';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
import { toast } from 'svelte-sonner';
import AccessControl from '$lib/components/workspace/common/AccessControl.svelte';
import Modal from '$lib/components/common/Modal.svelte';
import Spinner from '$lib/components/common/Spinner.svelte';
import Switch from '$lib/components/common/Switch.svelte';
import Tooltip from '$lib/components/common/Tooltip.svelte';
import Cog6 from '$lib/components/icons/Cog6.svelte';
import DatabaseSettings from '$lib/components/icons/DatabaseSettings.svelte';
import Plus from '$lib/components/icons/Plus.svelte';
import XMark from '$lib/components/icons/XMark.svelte';
import {
createExternalKnowledgeSource,
getExternalKnowledgeConnections,
searchKnowledgeBases,
testExternalKnowledgeSource,
updateExternalKnowledgeConnection,
updateExternalKnowledgeSource
} from '$lib/apis/knowledge';
const i18n = getContext<Writable<i18nType>>('i18n');
type ExternalKnowledgeConnection = {
id: string;
name: string;
provider: string;
endpoint: string;
config?: Record<string, any>;
capabilities?: Record<string, any>;
enabled?: boolean;
auth_configured?: boolean;
};
type ExternalKnowledgeItem = {
id: string;
name: string;
description?: string;
access_grants?: any[];
meta?: {
external?: {
connection_id?: string;
provider?: string;
source?: {
name?: string;
config?: Record<string, any>;
};
};
};
};
let loading = false;
let testing = false;
let creating = false;
let showSourceModal = false;
let connections: ExternalKnowledgeConnection[] = [];
let items: ExternalKnowledgeItem[] = [];
let editingItem: ExternalKnowledgeItem | null = null;
let editingConnection: ExternalKnowledgeConnection | null = null;
let accessGrants: any[] = [];
let testResult: {
documents?: string[];
metadatas?: Record<string, any>[];
distances?: number[];
} | null = null;
let sourceForm = {
name: '',
description: '',
provider: 'qdrant',
endpoint: '',
apiKey: '',
timeout: 30,
dbName: '',
sourceName: '',
contentField: 'payload.text',
vectorField: '',
metadataField: 'payload.metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name',
testQuery: ''
};
const schemaDefaults = (provider = sourceForm.provider) => {
if (provider === 'milvus') {
return {
contentField: 'data.text',
vectorField: 'vector',
metadataField: 'metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
}
if (provider === 'pgvector') {
return {
contentField: 'text',
vectorField: 'vector',
metadataField: 'vmetadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
}
return {
contentField: 'payload.text',
vectorField: '',
metadataField: 'payload.metadata',
documentIdField: 'id',
tableName: 'document_chunk',
collectionField: 'collection_name'
};
};
const resetForm = () => {
sourceForm = {
name: '',
description: '',
provider: 'qdrant',
endpoint: '',
apiKey: '',
timeout: 30,
dbName: '',
sourceName: '',
testQuery: '',
...schemaDefaults('qdrant')
};
accessGrants = [];
testResult = null;
editingItem = null;
editingConnection = null;
};
const openCreateSource = () => {
resetForm();
showSourceModal = true;
};
const openEditSource = (item: ExternalKnowledgeItem) => {
const connection = connectionForItem(item);
if (!connection) {
toast.error($i18n.t('External connection not found.'));
return;
}
resetForm();
editingItem = item;
editingConnection = connection;
const source = item.meta?.external?.source;
const sourceConfig = source?.config ?? {};
const defaults = schemaDefaults(connection.provider);
sourceForm = {
name: item.name ?? '',
description: item.description ?? '',
provider: connection.provider ?? 'qdrant',
endpoint: connection.endpoint ?? '',
apiKey: '',
timeout: connection.config?.timeout ?? 30,
dbName: connection.config?.db_name ?? '',
sourceName: source?.name ?? '',
testQuery: '',
contentField: sourceConfig.content_field ?? defaults.contentField,
vectorField: sourceConfig.vector_field ?? defaults.vectorField,
metadataField: sourceConfig.metadata_field ?? defaults.metadataField,
documentIdField: sourceConfig.document_id_field ?? defaults.documentIdField,
tableName: sourceConfig.table_name ?? defaults.tableName,
collectionField: sourceConfig.collection_field ?? defaults.collectionField
};
accessGrants = item.access_grants ?? [];
showSourceModal = true;
};
const markUntested = () => {
testResult = null;
};
const refresh = async () => {
loading = true;
const [connectionRes, knowledgeRes] = await Promise.all([
getExternalKnowledgeConnections(localStorage.token).catch((error) => {
toast.error(`${error}`);
return null;
}),
searchKnowledgeBases(localStorage.token, null, null, 1, 'external').catch((error) => {
toast.error(`${error}`);
return null;
})
]);
connections = connectionRes?.items ?? [];
items = knowledgeRes?.items ?? [];
loading = false;
};
const endpointPlaceholder = () => {
if (sourceForm.provider === 'pgvector') {
return 'postgresql://user:password@host:5432/db';
}
if (sourceForm.provider === 'milvus') {
return 'http://milvus.example.com:19530';
}
return 'https://qdrant.example.com';
};
const connectionForItem = (item: ExternalKnowledgeItem) =>
connections.find((connection) => connection.id === item?.meta?.external?.connection_id);
const connectionPayload = () => ({
name:
sourceForm.name.trim() ||
sourceForm.sourceName.trim() ||
$i18n.t('External Knowledge Source'),
provider: sourceForm.provider,
endpoint: sourceForm.endpoint,
auth_config:
sourceForm.provider === 'pgvector'
? {}
: sourceForm.apiKey
? { type: 'bearer', api_key: sourceForm.apiKey }
: editingConnection
? null
: {},
config: {
timeout: Number(sourceForm.timeout) || 30,
...(sourceForm.provider === 'milvus' && sourceForm.dbName
? { db_name: sourceForm.dbName }
: {})
},
capabilities: { retrieve: true },
enabled: editingConnection?.enabled !== false
});
const sourcePayload = () => {
const config: Record<string, string> = {
content_field: sourceForm.contentField.trim(),
...(sourceForm.vectorField.trim() ? { vector_field: sourceForm.vectorField.trim() } : {}),
...(sourceForm.metadataField.trim()
? { metadata_field: sourceForm.metadataField.trim() }
: {}),
...(sourceForm.documentIdField.trim()
? { document_id_field: sourceForm.documentIdField.trim() }
: {})
};
if (sourceForm.provider === 'pgvector') {
config.table_name = sourceForm.tableName.trim();
config.collection_field = sourceForm.collectionField.trim();
}
return {
type: 'collection',
name: sourceForm.sourceName.trim(),
config
};
};
const testFormIsValid = () =>
!!sourceForm.endpoint.trim() &&
!!sourceForm.sourceName.trim() &&
!!sourceForm.contentField.trim() &&
(sourceForm.provider === 'qdrant' || !!sourceForm.vectorField.trim()) &&
(sourceForm.provider !== 'pgvector' ||
(!!sourceForm.tableName.trim() && !!sourceForm.collectionField.trim())) &&
!!sourceForm.testQuery.trim();
const sourceFormIsValid = () => !!sourceForm.name.trim() && testFormIsValid();
const testSource = async () => {
if (!testFormIsValid()) {
toast.error($i18n.t('Fill the source fields and test query first.'));
return;
}
testing = true;
testResult = null;
const res = await testExternalKnowledgeSource(localStorage.token, {
...(editingConnection?.id ? { connection_id: editingConnection.id } : {}),
connection: connectionPayload(),
source: sourcePayload(),
query: sourceForm.testQuery,
count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res?.documents?.length) {
testResult = res;
toast.success($i18n.t('Test succeeded.'));
} else if (res) {
toast.error($i18n.t('Test returned no results.'));
}
testing = false;
};
const createSource = async () => {
if (!sourceFormIsValid()) {
toast.error($i18n.t('Fill the required fields first.'));
return;
}
if (!testResult?.documents?.length) {
toast.error($i18n.t('Test the source before creating it.'));
return;
}
creating = true;
const res = await createExternalKnowledgeSource(localStorage.token, {
name: sourceForm.name,
description: sourceForm.description,
connection: connectionPayload(),
source: sourcePayload(),
access_grants: accessGrants,
test_query: sourceForm.testQuery,
test_count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
toast.success($i18n.t('Knowledge source created.'));
showSourceModal = false;
resetForm();
await refresh();
}
creating = false;
};
const updateSource = async () => {
if (!editingItem) return;
if (!sourceFormIsValid()) {
toast.error($i18n.t('Fill the required fields first.'));
return;
}
if (!testResult?.documents?.length) {
toast.error($i18n.t('Test the source before saving it.'));
return;
}
creating = true;
const res = await updateExternalKnowledgeSource(localStorage.token, editingItem.id, {
name: sourceForm.name,
description: sourceForm.description,
connection: connectionPayload(),
source: sourcePayload(),
access_grants: accessGrants,
test_query: sourceForm.testQuery,
test_count: 5
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
toast.success($i18n.t('Knowledge source updated.'));
showSourceModal = false;
resetForm();
await refresh();
}
creating = false;
};
const submitSource = async () => {
if (editingItem) {
await updateSource();
} else {
await createSource();
}
};
const toggleSource = async (item: ExternalKnowledgeItem) => {
const connection = connectionForItem(item);
if (!connection) return;
const res = await updateExternalKnowledgeConnection(localStorage.token, connection.id, {
name: connection.name,
provider: connection.provider,
endpoint: connection.endpoint,
auth_config: null,
config: connection.config ?? {},
capabilities: connection.capabilities ?? { retrieve: true },
enabled: !connection.enabled
}).catch((error) => {
toast.error(`${error}`);
return null;
});
if (res) {
await refresh();
}
};
const updateProvider = () => {
sourceForm = {
...sourceForm,
...schemaDefaults(sourceForm.provider)
};
markUntested();
};
onMount(refresh);
</script>
<Modal bind:show={showSourceModal} size="sm">
<div>
<div class="flex justify-between dark:text-gray-100 px-5 pt-4 pb-2">
<h1 class="text-lg font-medium self-center font-primary">
{editingItem ? $i18n.t('Edit Knowledge Connection') : $i18n.t('Add Knowledge Connection')}
</h1>
<button
class="self-center"
aria-label={$i18n.t('Close')}
type="button"
on:click={() => {
showSourceModal = false;
resetForm();
}}
>
<XMark className="size-5" />
</button>
</div>
<div class="flex flex-col md:flex-row w-full px-4 pb-4 md:space-x-4 dark:text-gray-200">
<div class="flex flex-col w-full sm:flex-row sm:justify-center sm:space-x-6">
<form class="flex flex-col w-full" on:submit|preventDefault={submitSource}>
<div class="px-1">
<div class="flex gap-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-name"
>{$i18n.t('Name')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-name"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.name}
on:input={markUntested}
placeholder={$i18n.t('Research Knowledge')}
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-provider"
>{$i18n.t('Provider')}</label
>
</div>
<div class="flex flex-1 items-center">
<select
id="external-source-provider"
class="w-full text-sm bg-transparent outline-hidden"
bind:value={sourceForm.provider}
on:change={updateProvider}
>
<option value="qdrant">Qdrant</option>
<option value="milvus">Milvus</option>
<option value="pgvector">pgvector</option>
</select>
</div>
</div>
</div>
<div class="flex flex-col w-full mt-2">
<label class="text-xs text-gray-500 mb-0.5" for="external-source-description"
>{$i18n.t('Description')}</label
>
<textarea
id="external-source-description"
class="w-full text-sm bg-transparent outline-hidden resize-none placeholder:text-gray-300 dark:placeholder:text-gray-700"
rows="2"
bind:value={sourceForm.description}
></textarea>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-endpoint"
>{$i18n.t('Endpoint')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-endpoint"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.endpoint}
on:input={markUntested}
placeholder={endpointPlaceholder()}
required
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-timeout"
>{$i18n.t('Timeout')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-timeout"
class="w-full text-sm bg-transparent outline-hidden"
type="number"
bind:value={sourceForm.timeout}
on:input={markUntested}
/>
</div>
</div>
{#if sourceForm.provider !== 'pgvector'}
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-api-key"
>{$i18n.t('API Key / Token')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-api-key"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
type="password"
bind:value={sourceForm.apiKey}
on:input={markUntested}
autocomplete="off"
/>
</div>
</div>
{/if}
</div>
{#if sourceForm.provider === 'milvus'}
<div class="flex gap-2 mt-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-db-name"
>{$i18n.t('Database')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-db-name"
class="w-full text-sm bg-transparent outline-hidden"
bind:value={sourceForm.dbName}
on:input={markUntested}
placeholder={$i18n.t('Default')}
/>
</div>
</div>
</div>
{/if}
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
<div class="flex gap-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-collection"
>{$i18n.t('Collection')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-collection"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.sourceName}
on:input={markUntested}
placeholder="research-docs"
required
/>
</div>
</div>
</div>
{#if sourceForm.provider === 'pgvector'}
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-table"
>{$i18n.t('Table')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-table"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.tableName}
on:input={markUntested}
placeholder="document_chunk"
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-collection-field"
>{$i18n.t('Collection Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-collection-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.collectionField}
on:input={markUntested}
placeholder="collection_name"
required
/>
</div>
</div>
</div>
{/if}
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-content-field"
>{$i18n.t('Content Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-content-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.contentField}
on:input={markUntested}
placeholder={sourceForm.provider === 'pgvector' ? 'text' : 'payload.text'}
required
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-vector-field">
{$i18n.t('Vector Field')}
</label>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-vector-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.vectorField}
on:input={markUntested}
placeholder={sourceForm.provider === 'qdrant' ? $i18n.t('Default') : 'vector'}
required={sourceForm.provider !== 'qdrant'}
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-metadata-field"
>{$i18n.t('Metadata Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-metadata-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.metadataField}
on:input={markUntested}
placeholder={sourceForm.provider === 'pgvector'
? 'vmetadata'
: 'payload.metadata'}
/>
</div>
</div>
<div class="flex flex-col flex-1">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-document-id-field"
>{$i18n.t('Document ID Field')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-document-id-field"
class="w-full text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.documentIdField}
on:input={markUntested}
placeholder="id"
/>
</div>
</div>
</div>
<div class="flex gap-2 mt-2">
<div class="flex flex-col w-full">
<div class="flex justify-between mb-0.5">
<label class="text-xs text-gray-500" for="external-source-test-query"
>{$i18n.t('Test Query')}</label
>
</div>
<div class="flex flex-1 items-center">
<input
id="external-source-test-query"
class="w-full flex-1 text-sm bg-transparent outline-hidden placeholder:text-gray-300 dark:placeholder:text-gray-700"
bind:value={sourceForm.testQuery}
on:input={markUntested}
placeholder={$i18n.t('Ask a test question')}
required
/>
<Tooltip
content={$i18n.t('Verify Connection')}
className="shrink-0 flex items-center mr-1"
>
<button
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition"
type="button"
on:click={testSource}
disabled={testing}
aria-label={$i18n.t('Verify Connection')}
>
{#if testing}
<Spinner />
{:else}
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 20 20"
fill="currentColor"
class="w-4 h-4"
aria-hidden="true"
>
<path
fill-rule="evenodd"
d="M15.312 11.424a5.5 5.5 0 01-9.201 2.466l-.312-.311h2.433a.75.75 0 000-1.5H3.989a.75.75 0 00-.75.75v4.242a.75.75 0 001.5 0v-2.43l.31.31a7 7 0 0011.712-3.138.75.75 0 00-1.449-.39zm1.23-3.723a.75.75 0 00.219-.53V2.929a.75.75 0 00-1.5 0V5.36l-.31-.31A7 7 0 003.239 8.188a.75.75 0 101.448.389A5.5 5.5 0 0113.89 6.11l.311.31h-2.432a.75.75 0 000 1.5h4.243a.75.75 0 00.53-.219z"
clip-rule="evenodd"
/>
</svg>
{/if}
</button>
</Tooltip>
</div>
<div class="text-xs text-gray-500 mt-1">
{$i18n.t(
'External vectors must be generated with the same embedding model configured in Open WebUI.'
)}
</div>
</div>
</div>
<hr class="border-gray-100/50 dark:border-gray-700/10 my-2.5 w-full" />
<AccessControl
bind:accessGrants
accessRoles={['read']}
share={true}
sharePublic={true}
shareUsers={true}
/>
<div class="flex justify-between items-center pt-3 text-sm font-medium">
<div></div>
<button
class="px-3.5 py-1.5 text-sm font-medium bg-black hover:bg-gray-900 text-white dark:bg-white dark:text-black dark:hover:bg-gray-100 transition rounded-full flex items-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
type="submit"
disabled={creating || !sourceFormIsValid() || !testResult?.documents?.length}
>
{editingItem ? $i18n.t('Save') : $i18n.t('Create')}
{#if creating}<Spinner />{/if}
</button>
</div>
</div>
</form>
</div>
</div>
</div>
</Modal>
<div class="mb-2.5 flex flex-col w-full justify-between text-sm">
<div class="flex justify-between items-center mb-1">
<div class="flex items-center gap-2">
<div class="font-medium">{$i18n.t('External Knowledge Sources')}</div>
<span
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
>{$i18n.t('Experimental')}</span
>
</div>
<Tooltip content={$i18n.t('Add Connection')}>
<button class="px-1" on:click={openCreateSource} type="button">
<Plus />
</button>
</Tooltip>
</div>
<div class="flex flex-col gap-1.5">
{#each items as item}
{@const connection = connectionForItem(item)}
<div class="flex w-full gap-2 items-center">
<Tooltip className="w-full relative" content={''} placement="top-start">
<div class="flex w-full">
<div
class="flex-1 relative flex gap-1.5 items-center min-w-0 {connection?.enabled ===
false
? 'opacity-50'
: ''}"
>
<Tooltip content={$i18n.t('Knowledge')}>
<DatabaseSettings className="size-4" strokeWidth="1.5" />
</Tooltip>
<div class="outline-hidden w-full bg-transparent text-sm min-w-0 line-clamp-1">
<span>{item.name}</span>
{' '}
<span class="text-gray-500">
{item?.meta?.external?.provider ?? connection?.provider}
{item?.meta?.external?.source?.name ? `· ${item.meta.external.source.name}` : ''}
</span>
</div>
</div>
</div>
</Tooltip>
<div class="flex gap-1 items-center">
<Tooltip content={$i18n.t('Configure')}>
<button
class="self-center p-1 bg-transparent hover:bg-gray-100 dark:hover:bg-gray-850 rounded-lg transition disabled:opacity-50 disabled:cursor-not-allowed"
type="button"
disabled={!connection}
aria-label={$i18n.t('Configure')}
on:click={() => {
openEditSource(item);
}}
>
<Cog6 />
</button>
</Tooltip>
<Tooltip
content={connection?.enabled !== false ? $i18n.t('Enabled') : $i18n.t('Disabled')}
>
<Switch
state={connection?.enabled !== false}
on:change={() => {
toggleSource(item);
}}
/>
</Tooltip>
</div>
</div>
{/each}
</div>
{#if loading}
<div class="py-2">
<Spinner />
</div>
{:else if items.length === 0}
<div class="text-xs text-gray-400 dark:text-gray-500">
{$i18n.t('No external knowledge sources configured.')}
</div>
{/if}
<div class="my-1.5">
<div class="text-xs text-gray-500">
{$i18n.t(
'Create one read-only Knowledge source per external collection. Test must pass before the source is created.'
)}
</div>
</div>
</div>

73
src/lib/components/admin/Settings/Integrations.svelte
View file

@ -1,11 +1,12 @@
<script lang="ts">
import { toast } from 'svelte-sonner';
import { createEventDispatcher, onMount, getContext, tick } from 'svelte';
import { v4 as uuidv4 } from 'uuid';
import { getModels as _getModels } from '$lib/apis';
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const dispatch = createEventDispatcher();
const i18n = getContext('i18n');
const i18n = getContext<Writable<i18nType>>('i18n');
import { models, settings, user, terminalServers } from '$lib/stores';
import { getTerminalServers } from '$lib/apis/terminal';
@ -22,6 +23,7 @@
import AddToolServerModal from '$lib/components/AddToolServerModal.svelte';
import AddTerminalServerModal from '$lib/components/AddTerminalServerModal.svelte';
import ExternalKnowledge from './ExternalKnowledge.svelte';
import {
getToolServerConnections,
@ -32,16 +34,26 @@
export let saveSettings: Function;
let servers = null;
type ToolServerConnection = any;
type TerminalConnection = {
id?: string;
url?: string;
name?: string;
key?: string;
enabled?: boolean;
[key: string]: any;
};
let servers: ToolServerConnection[] | null = null;
let showConnectionModal = false;
// Terminal server admin connections
let terminalConnections = [];
let terminalConnections: TerminalConnection[] = [];
let showAddTerminalModal = false;
let editTerminalIdx: number | null = null;
const addConnectionHandler = async (server) => {
servers = [...servers, server];
const addConnectionHandler = async (server: ToolServerConnection) => {
servers = [...(servers ?? []), server];
await updateHandler();
};
@ -71,7 +83,9 @@
// Refresh the terminalServers store so changes are reflected immediately
// Preserve user direct terminals, refresh system terminals from backend
const existingDirectTerminals = ($terminalServers ?? []).filter((t) => !t.id);
const existingDirectTerminals = (($terminalServers ?? []) as TerminalConnection[]).filter(
(t) => !t.id
);
const systemTerminals = await getTerminalServers(localStorage.token);
const systemEntries = systemTerminals.map((t) => ({
id: t.id,
@ -79,16 +93,19 @@
name: t.name,
key: localStorage.token
}));
terminalServers.set([...existingDirectTerminals, ...systemEntries]);
terminalServers.set([...existingDirectTerminals, ...systemEntries] as any);
}
};
const addTerminalConnection = (server) => {
terminalConnections = [...terminalConnections, { ...server, id: server.id ?? uuidv4() }];
const addTerminalConnection = (server: TerminalConnection) => {
terminalConnections = [
...terminalConnections,
{ ...server, id: server.id ?? crypto.randomUUID() }
];
saveTerminalServers();
};
const updateTerminalConnection = (idx: number, updated) => {
const updateTerminalConnection = (idx: number, updated: TerminalConnection) => {
terminalConnections = terminalConnections.map((c, i) =>
i === idx ? { ...c, ...updated, id: updated.id ?? c.id } : c
);
@ -102,13 +119,13 @@
onMount(async () => {
const res = await getToolServerConnections(localStorage.token);
servers = res.TOOL_SERVER_CONNECTIONS;
servers = res.TOOL_SERVER_CONNECTIONS as ToolServerConnection[];
// Load terminal server connections
try {
const terminalRes = await getTerminalServerConnections(localStorage.token);
if (terminalRes?.TERMINAL_SERVER_CONNECTIONS) {
terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS;
terminalConnections = terminalRes.TERMINAL_SERVER_CONNECTIONS as TerminalConnection[];
}
} catch {
// Not configured yet
@ -122,7 +139,7 @@
bind:show={showAddTerminalModal}
edit={editTerminalIdx !== null}
connection={editTerminalIdx !== null ? terminalConnections[editTerminalIdx] : null}
onSubmit={(c) => {
onSubmit={(c: TerminalConnection) => {
if (editTerminalIdx !== null) {
updateTerminalConnection(editTerminalIdx, c);
editTerminalIdx = null;
@ -148,13 +165,13 @@
{#if servers !== null}
<div class="">
<div class="mb-3">
<div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('General')}</div>
<div class=" mt-0.5 mb-2.5 text-base font-medium">{$i18n.t('Tools')}</div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
<div class="mb-2.5 flex flex-col w-full justify-between">
<div class="flex justify-between items-center mb-0.5">
<div class="font-medium">{$i18n.t('Manage Tool Servers')}</div>
<div class="font-medium">{$i18n.t('External Tool Servers')}</div>
<Tooltip content={$i18n.t(`Add Connection`)}>
<button
@ -170,21 +187,21 @@
</div>
<div class="flex flex-col gap-1">
{#each servers as server, idx}
{#each servers ?? [] as server, idx}
<Connection
bind:connection={server}
onSubmit={() => {
updateHandler();
}}
onDelete={() => {
servers = servers.filter((_, i) => i !== idx);
servers = (servers ?? []).filter((_, i) => i !== idx);
updateHandler();
}}
/>
{/each}
</div>
{#if servers.length === 0}
{#if (servers ?? []).length === 0}
<div class="text-xs text-gray-400 dark:text-gray-500">
{$i18n.t('No tool server connections configured.')}
</div>
@ -197,17 +214,9 @@
</div>
</div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-4" />
<div class="mb-2.5 flex flex-col w-full">
<div class="mt-4 mb-2.5 flex flex-col w-full">
<div class="flex justify-between items-center mb-1">
<div class="flex items-center gap-2">
<div class="font-medium">{$i18n.t('Open Terminal')}</div>
<span
class="text-[0.65rem] font-medium uppercase px-1.5 py-0.5 rounded-full bg-gray-100 dark:bg-gray-800 text-gray-500 dark:text-gray-400"
>{$i18n.t('Experimental')}</span
>
</div>
<div class="font-medium">{$i18n.t('Open Terminal')}</div>
<Tooltip content={$i18n.t('Add Connection')}>
<button
@ -300,6 +309,12 @@
</div>
</div>
</div>
<div class="mt-8 mb-2.5 text-base font-medium">{$i18n.t('Knowledge')}</div>
<hr class=" border-gray-100/30 dark:border-gray-850/30 my-2" />
<ExternalKnowledge />
</div>
</div>
{:else}

85
src/lib/components/workspace/Knowledge.svelte
View file

@ -5,7 +5,10 @@
import { toast } from 'svelte-sonner';
import { onMount, getContext, tick, onDestroy } from 'svelte';
const i18n = getContext('i18n');
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const i18n = getContext<Writable<i18nType>>('i18n');
import { WEBUI_NAME, knowledge, user } from '$lib/stores';
import {
@ -28,19 +31,33 @@
import ViewSelector from './common/ViewSelector.svelte';
import Loader from '../common/Loader.svelte';
type KnowledgeListItem = {
id: string;
name: string;
description?: string;
updated_at: number;
write_access?: boolean;
meta?: any;
user?: {
name?: string;
email?: string;
};
};
let loaded = false;
let showDeleteConfirm = false;
let tagsContainerElement: HTMLDivElement;
let selectedItem = null;
let selectedItem: KnowledgeListItem | null = null;
let page = 1;
let query = '';
let searchDebounceTimer: ReturnType<typeof setTimeout>;
let viewOption = '';
let sourceOption = '';
let items = null;
let total = null;
let items: KnowledgeListItem[] | null = null;
let total: number | null = null;
let allItemsLoaded = false;
let itemsLoading = false;
@ -56,7 +73,7 @@
clearTimeout(searchDebounceTimer);
});
$: if (loaded && viewOption !== undefined) {
$: if (loaded && viewOption !== undefined && sourceOption !== undefined) {
init();
}
@ -83,16 +100,20 @@
const getItemsPage = async () => {
itemsLoading = true;
const res = await searchKnowledgeBases(localStorage.token, query, viewOption, page).catch(
() => {
return [];
}
);
const res = await searchKnowledgeBases(
localStorage.token,
query,
viewOption,
page,
sourceOption
).catch(() => {
return [];
});
if (res) {
console.log(res);
total = res.total;
const pageItems = res.items;
const pageItems: KnowledgeListItem[] = res.items ?? [];
if ((pageItems ?? []).length === 0) {
allItemsLoaded = true;
@ -113,7 +134,9 @@
return res;
};
const deleteHandler = async (item) => {
const deleteHandler = async (item: KnowledgeListItem | null) => {
if (!item) return;
const res = await deleteKnowledgeById(localStorage.token, item.id).catch((e) => {
toast.error(`${e}`);
});
@ -124,7 +147,7 @@
}
};
const exportHandler = async (item) => {
const exportHandler = async (item: KnowledgeListItem) => {
try {
const blob = await exportKnowledgeById(localStorage.token, item.id);
if (blob) {
@ -145,6 +168,7 @@
onMount(async () => {
viewOption = localStorage?.workspaceViewOption || '';
sourceOption = localStorage?.workspaceKnowledgeSourceOption || '';
loaded = true;
});
</script>
@ -241,6 +265,19 @@
await tick();
}}
/>
<select
class="relative w-full flex items-center gap-0.5 px-2.5 py-1.5 bg-gray-50 dark:bg-gray-850 rounded-xl outline-hidden"
bind:value={sourceOption}
on:change={async () => {
localStorage.workspaceKnowledgeSourceOption = sourceOption;
await tick();
}}
>
<option value="">{$i18n.t('All Sources')}</option>
<option value="local">{$i18n.t('Local')}</option>
<option value="external">{$i18n.t('Connected')}</option>
</select>
</div>
</div>
@ -267,11 +304,23 @@
<div class=" self-center flex-1 justify-between">
<div class="flex items-center justify-between -my-1 h-8">
<div class=" flex gap-2 items-center justify-between w-full">
<div>
<Badge type="success" content={$i18n.t('Collection')} />
</div>
{#if item?.meta?.source === 'external'}
<div>
<Badge
type="muted"
content={item?.meta?.external?.provider ?? $i18n.t('Connected')}
/>
</div>
<div>
<Badge type="muted" content={$i18n.t('Read Only')} />
</div>
{:else}
<div>
<Badge type="success" content={$i18n.t('Collection')} />
</div>
{/if}
{#if !item?.write_access}
{#if !item?.write_access && item?.meta?.source !== 'external'}
<div>
<Badge type="muted" content={$i18n.t('Read Only')} />
</div>
@ -282,7 +331,7 @@
<div class="flex items-center gap-2">
<div class=" flex self-center">
<ItemMenu
onExport={$user.role === 'admin'
onExport={$user?.role === 'admin'
? () => {
exportHandler(item);
}

635
src/lib/components/workspace/Knowledge/KnowledgeBase.svelte
View file

@ -5,7 +5,10 @@
import { PaneGroup, Pane, PaneResizer } from 'paneforge';
import { onMount, getContext, onDestroy, tick } from 'svelte';
const i18n = getContext('i18n');
import type { Writable } from 'svelte/store';
import type { i18n as i18nType } from 'i18next';
const i18n = getContext<Writable<i18nType>>('i18n');
import { goto } from '$app/navigation';
import { page } from '$app/stores';
@ -40,7 +43,8 @@
deleteKnowledgeDirectory,
moveFileInKnowledge,
syncKnowledgeDiff,
syncKnowledgeCleanup
syncKnowledgeCleanup,
testExternalKnowledgeRetrieval
} from '$lib/apis/knowledge';
import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';
@ -98,11 +102,13 @@
files: any[];
access_grants?: any[];
write_access?: boolean;
meta?: any;
};
let id = null;
let knowledge: Knowledge | null = null;
let knowledgeId = null;
let isExternalKnowledge = false;
let selectedFileId = null;
let selectedFile = null;
@ -132,6 +138,14 @@
let deleteDirectoryContents = true;
let pendingPollTimer: ReturnType<typeof setInterval> | null = null;
let externalTestQuery = '';
let externalTestResult: {
documents?: string[];
metadatas?: Record<string, any>[];
distances?: number[];
} | null = null;
$: isExternalKnowledge = knowledge?.meta?.source === 'external';
const reset = () => {
currentPage = 1;
@ -243,6 +257,24 @@
}
};
const externalTestHandler = async () => {
if (!isExternalKnowledge || !externalTestQuery.trim()) return;
const external = knowledge?.meta?.external ?? {};
const res = await testExternalKnowledgeRetrieval(localStorage.token, external.connection_id, {
query: externalTestQuery,
source: external.source,
count: 5
}).catch((e) => {
toast.error(`${e}`);
return null;
});
if (res) {
externalTestResult = res;
}
};
const createFileFromText = (name, content) => {
const blob = new Blob([content], { type: 'text/plain' });
const file = blobToFile(blob, `${name}.txt`);
@ -1299,284 +1331,357 @@
<div
class="mt-2 mb-2.5 py-2 -mx-0 bg-white dark:bg-gray-900 rounded-3xl border border-gray-100/30 dark:border-gray-850/30 flex-1"
>
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
<div class="flex flex-1 items-center">
<div class=" self-center ml-1 mr-3">
<Search className="size-3.5" />
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
bind:value={query}
on:input={handleSearchInput}
aria-label={$i18n.t('Search Collection')}
placeholder={$i18n.t('Search Collection')}
on:focus={() => {
selectedFileId = null;
}}
/>
<Dropdown align="end">
<button
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
type="button"
>
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
</button>
<div slot="content">
<div
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
>
<button
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
type="button"
on:click={() => {
includeContent = !includeContent;
}}
>
<Checkbox
state={includeContent ? 'checked' : 'unchecked'}
on:change={(e) => {
includeContent = e.detail === 'checked';
}}
/>
{$i18n.t('File content')}
</button>
</div>
{#if isExternalKnowledge}
<div class="p-5 flex flex-col gap-4">
<div class="flex flex-wrap gap-2 text-xs">
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{$i18n.t('Connected')}
</div>
</Dropdown>
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{$i18n.t('Read Only')}
</div>
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{knowledge?.meta?.external?.provider ?? $i18n.t('Provider')}
</div>
<div class="px-2 py-1 rounded-lg bg-gray-50 dark:bg-gray-850">
{$i18n.t('Service Account')}
</div>
</div>
{#if knowledge?.write_access}
<div class="grid grid-cols-1 md:grid-cols-2 gap-3 text-sm">
<div>
<AddContentMenu
onUpload={(data) => {
if (data.type === 'directory') {
uploadDirectoryHandler();
} else if (data.type === 'new_directory') {
showNewDirectoryModal = true;
} else if (data.type === 'web') {
showAddWebpageModal = true;
} else if (data.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
onSync={async () => {
pendingSyncFiles = await collectDirectoryFiles();
if (pendingSyncFiles?.length) {
showSyncConfirmModal = true;
}
}}
onReset={() => {
showResetConfirm = true;
}}
/>
<div class="text-xs text-gray-500 mb-1">{$i18n.t('Mapped Source')}</div>
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
{knowledge?.meta?.external?.source?.name ?? $i18n.t('Not configured')}
</div>
</div>
{/if}
</div>
</div>
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
on:wheel={(e) => {
if (e.deltaY !== 0) {
e.preventDefault();
e.currentTarget.scrollLeft += e.deltaY;
}
}}
>
<div
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
>
<DropdownOptions
align="start"
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
bind:value={viewOption}
items={[
{ value: null, label: $i18n.t('All') },
{ value: 'created', label: $i18n.t('Created by you') },
{ value: 'shared', label: $i18n.t('Shared with you') }
]}
onChange={(value) => {
if (value) {
localStorage.workspaceViewOption = value;
} else {
delete localStorage.workspaceViewOption;
}
}}
/>
<DropdownOptions
align="start"
bind:value={sortKey}
placeholder={$i18n.t('Sort')}
items={[
{ value: 'name', label: $i18n.t('Name') },
{ value: 'created_at', label: $i18n.t('Created At') },
{ value: 'updated_at', label: $i18n.t('Updated At') }
]}
/>
{#if sortKey}
<DropdownOptions
align="start"
bind:value={direction}
items={[
{ value: 'asc', label: $i18n.t('Asc') },
{ value: null, label: $i18n.t('Desc') }
]}
/>
{/if}
</div>
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 mt-2">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
{#if syncing}
<div class="mx-2.5 mt-2.5 -mb-0.5">
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
<Spinner className="size-3.5 shrink-0" />
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
{syncing}
</div>
</div>
</div>
{/if}
{#if fileItems !== null && fileItemsTotal !== null}
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
<div class="flex-1 flex">
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
<div class="w-full h-full flex flex-col min-h-full">
{#if fileItems.length > 0 || directoryItems.length > 0}
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
<Files
files={fileItems}
directories={directoryItems}
{knowledge}
{selectedFileId}
onClick={(fileId) => {
selectedFileId = fileId;
if (fileItems) {
const file = fileItems.find((file) => file.id === selectedFileId);
if (file) {
fileSelectHandler(file);
} else {
selectedFile = null;
}
}
}}
onDelete={(fileId) => {
selectedFileId = null;
selectedFile = null;
deleteFileHandler(fileId);
}}
onRename={(fileId, name) => renameFileHandler(fileId, name)}
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
onMoveFileToDirectory={(fileId, dirId) =>
moveFileToDirectoryHandler(fileId, dirId)}
onMoveDirectoryToDirectory={(dirId, targetId) =>
moveDirectoryHandler(dirId, targetId)}
/>
</div>
{#if fileItemsTotal > 30}
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
{/if}
{:else}
<div class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs">
<div>
{$i18n.t('No content found')}
</div>
</div>
{/if}
<div>
<div class="text-xs text-gray-500 mb-1">{$i18n.t('Auth Mode')}</div>
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2">
{$i18n.t('Admin-managed service account')}
</div>
</div>
</div>
{#if selectedFileId !== null}
<Drawer
className="h-full"
show={selectedFileId !== null}
onClose={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<div class="flex flex-col justify-start h-full max-h-full">
<div class=" flex flex-col w-full h-full max-h-full">
<div class="shrink-0 flex items-center p-2">
<div class="mr-2">
<button
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
aria-label={$i18n.t('Close')}
on:click={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<ChevronLeft strokeWidth="2.5" />
</button>
</div>
<div class=" flex-1 text-lg line-clamp-1">
{selectedFile?.meta?.name}
</div>
<div class="text-xs text-gray-500">
{$i18n.t(
'This knowledge base retrieves from a connected source. Open WebUI can query it, but cannot upload, sync, edit, delete, reset, or reindex its source data.'
)}
</div>
{#if knowledge?.write_access}
<div>
<button
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isSaving}
on:click={() => {
updateFileContentHandler();
}}
>
{$i18n.t('Save')}
{#if isSaving}
<div class="ml-2 self-center">
<Spinner />
</div>
{/if}
</button>
</div>
{/if}
<div class="flex flex-col gap-2">
<div class="font-medium text-sm">{$i18n.t('Test Query')}</div>
<div class="flex gap-2">
<input
class="w-full text-sm rounded-xl bg-gray-50 dark:bg-gray-850 px-3 py-2 outline-hidden"
bind:value={externalTestQuery}
placeholder={$i18n.t('Ask this knowledge source a test question')}
/>
<button
class="px-3 py-2 rounded-xl bg-black text-white dark:bg-white dark:text-black text-sm"
on:click={externalTestHandler}
>
{$i18n.t('Test')}
</button>
</div>
</div>
{#if externalTestResult}
<div class="rounded-xl bg-gray-50 dark:bg-gray-850 p-3 text-xs">
<div class="font-medium mb-2">{$i18n.t('Preview')}</div>
{#each externalTestResult.documents ?? [] as document, idx}
<div class="border-t border-gray-100 dark:border-gray-800 py-2">
<div class="line-clamp-4">{document}</div>
<div class="text-gray-500 mt-1">
{externalTestResult.metadatas?.[idx]?.source ?? ''}
</div>
{#key selectedFile.id}
<textarea
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
bind:value={selectedFileContent}
disabled={!knowledge?.write_access}
aria-label={$i18n.t('File content')}
placeholder={$i18n.t('Add content here')}
/>
{/key}
</div>
</div>
</Drawer>
{/each}
</div>
{/if}
</div>
{:else}
<div class="my-10">
<Spinner className="size-4" />
<div class="px-3.5 flex flex-1 items-center w-full space-x-2 py-0.5 pb-2">
<div class="flex flex-1 items-center">
<div class=" self-center ml-1 mr-3">
<Search className="size-3.5" />
</div>
<input
class=" w-full text-sm pr-4 py-1 rounded-r-xl outline-hidden bg-transparent"
bind:value={query}
on:input={handleSearchInput}
aria-label={$i18n.t('Search Collection')}
placeholder={$i18n.t('Search Collection')}
on:focus={() => {
selectedFileId = null;
}}
/>
<Dropdown align="end">
<button
class="p-1.5 mr-1 rounded-xl text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-800 transition"
type="button"
>
<AdjustmentsHorizontal className="size-3.5" strokeWidth="2" />
</button>
<div slot="content">
<div
class="min-w-[180px] rounded-2xl px-1 py-1 border border-gray-100 dark:border-gray-800 z-50 bg-white dark:bg-gray-850 dark:text-white shadow-lg"
>
<button
class="select-none flex gap-2 items-center px-3 py-1.5 text-sm cursor-pointer hover:bg-gray-50 dark:hover:bg-gray-800 rounded-xl w-full"
type="button"
on:click={() => {
includeContent = !includeContent;
}}
>
<Checkbox
state={includeContent ? 'checked' : 'unchecked'}
on:change={(e) => {
includeContent = e.detail === 'checked';
}}
/>
{$i18n.t('File content')}
</button>
</div>
</div>
</Dropdown>
{#if knowledge?.write_access}
<div>
<AddContentMenu
onUpload={(data) => {
if (data.type === 'directory') {
uploadDirectoryHandler();
} else if (data.type === 'new_directory') {
showNewDirectoryModal = true;
} else if (data.type === 'web') {
showAddWebpageModal = true;
} else if (data.type === 'text') {
showAddTextContentModal = true;
} else {
document.getElementById('files-input').click();
}
}}
onSync={async () => {
pendingSyncFiles = await collectDirectoryFiles();
if (pendingSyncFiles?.length) {
showSyncConfirmModal = true;
}
}}
onReset={() => {
showResetConfirm = true;
}}
/>
</div>
{/if}
</div>
</div>
<div class="px-3 flex justify-between">
<div
class="flex w-full bg-transparent overflow-x-auto scrollbar-none"
on:wheel={(e) => {
if (e.deltaY !== 0) {
e.preventDefault();
e.currentTarget.scrollLeft += e.deltaY;
}
}}
>
<div
class="flex gap-3 w-fit text-center text-sm rounded-full bg-transparent px-0.5 whitespace-nowrap"
>
<DropdownOptions
align="start"
className="flex shrink-0 items-center gap-2 px-3 py-1.5 text-sm bg-gray-50 dark:bg-gray-850 rounded-xl placeholder-gray-400 outline-hidden focus:outline-hidden"
bind:value={viewOption}
items={[
{ value: null, label: $i18n.t('All') },
{ value: 'created', label: $i18n.t('Created by you') },
{ value: 'shared', label: $i18n.t('Shared with you') }
]}
onChange={(value) => {
if (value) {
localStorage.workspaceViewOption = value;
} else {
delete localStorage.workspaceViewOption;
}
}}
/>
<DropdownOptions
align="start"
bind:value={sortKey}
placeholder={$i18n.t('Sort')}
items={[
{ value: 'name', label: $i18n.t('Name') },
{ value: 'created_at', label: $i18n.t('Created At') },
{ value: 'updated_at', label: $i18n.t('Updated At') }
]}
/>
{#if sortKey}
<DropdownOptions
align="start"
bind:value={direction}
items={[
{ value: 'asc', label: $i18n.t('Asc') },
{ value: null, label: $i18n.t('Desc') }
]}
/>
{/if}
</div>
</div>
</div>
{#if currentDirectoryId !== null}
<div class="px-5 mt-2">
<KnowledgeBreadcrumbs
rootLabel={knowledge.name}
{breadcrumbs}
onNavigate={(dirId) => navigateToDirectory(dirId)}
onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)}
onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)}
/>
</div>
{/if}
{#if syncing}
<div class="mx-2.5 mt-2.5 -mb-0.5">
<div class="flex items-center gap-2.5 rounded-xl py-2 px-3 bg-gray-50 dark:bg-gray-850">
<Spinner className="size-3.5 shrink-0" />
<div class="text-xs text-gray-500 dark:text-gray-400 truncate">
{syncing}
</div>
</div>
</div>
{/if}
{#if fileItems !== null && fileItemsTotal !== null}
<div class="flex flex-row flex-1 gap-3 px-2.5 mt-2">
<div class="flex-1 flex">
<div class=" flex flex-col w-full space-x-2 rounded-lg h-full">
<div class="w-full h-full flex flex-col min-h-full">
{#if fileItems.length > 0 || directoryItems.length > 0}
<div class=" flex overflow-y-auto h-full w-full scrollbar-hidden text-xs">
<Files
files={fileItems}
directories={directoryItems}
{knowledge}
{selectedFileId}
onClick={(fileId) => {
selectedFileId = fileId;
if (fileItems) {
const file = fileItems.find((file) => file.id === selectedFileId);
if (file) {
fileSelectHandler(file);
} else {
selectedFile = null;
}
}
}}
onDelete={(fileId) => {
selectedFileId = null;
selectedFile = null;
deleteFileHandler(fileId);
}}
onRename={(fileId, name) => renameFileHandler(fileId, name)}
onNavigateDirectory={(dirId) => navigateToDirectory(dirId)}
onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)}
onDeleteDirectory={(id) => confirmDeleteDirectory(id)}
onMoveFileToDirectory={(fileId, dirId) =>
moveFileToDirectoryHandler(fileId, dirId)}
onMoveDirectoryToDirectory={(dirId, targetId) =>
moveDirectoryHandler(dirId, targetId)}
/>
</div>
{#if fileItemsTotal > 30}
<Pagination bind:page={currentPage} count={fileItemsTotal} perPage={30} />
{/if}
{:else}
<div
class="my-3 flex flex-col justify-center text-center text-gray-500 text-xs"
>
<div>
{$i18n.t('No content found')}
</div>
</div>
{/if}
</div>
</div>
</div>
{#if selectedFileId !== null}
<Drawer
className="h-full"
show={selectedFileId !== null}
onClose={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<div class="flex flex-col justify-start h-full max-h-full">
<div class=" flex flex-col w-full h-full max-h-full">
<div class="shrink-0 flex items-center p-2">
<div class="mr-2">
<button
class="w-full text-left text-sm p-1.5 rounded-lg dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-gray-850"
aria-label={$i18n.t('Close')}
on:click={() => {
selectedFileId = null;
selectedFile = null;
}}
>
<ChevronLeft strokeWidth="2.5" />
</button>
</div>
<div class=" flex-1 text-lg line-clamp-1">
{selectedFile?.meta?.name}
</div>
{#if knowledge?.write_access}
<div>
<button
class="flex self-center w-fit text-sm py-1 px-2.5 dark:text-gray-300 dark:hover:text-white hover:bg-black/5 dark:hover:bg-white/5 rounded-lg disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isSaving}
on:click={() => {
updateFileContentHandler();
}}
>
{$i18n.t('Save')}
{#if isSaving}
<div class="ml-2 self-center">
<Spinner />
</div>
{/if}
</button>
</div>
{/if}
</div>
{#key selectedFile.id}
<textarea
class="w-full h-full text-sm outline-none resize-none px-3 py-2"
bind:value={selectedFileContent}
disabled={!knowledge?.write_access}
aria-label={$i18n.t('File content')}
placeholder={$i18n.t('Add content here')}
/>
{/key}
</div>
</div>
</Drawer>
{/if}
</div>
{:else}
<div class="my-10">
<Spinner className="size-4" />
</div>
{/if}
{/if}
</div>
{:else}