From 15c7e374384488effc3d6059d09b3a8aa79c618d Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 23 Jun 2026 23:13:32 +0200 Subject: [PATCH] refac --- backend/open_webui/config.py | 11 + backend/open_webui/models/knowledge.py | 30 + backend/open_webui/retrieval/external.py | 378 ++++++++ backend/open_webui/retrieval/utils.py | 109 ++- backend/open_webui/routers/knowledge.py | 619 +++++++++++- backend/open_webui/routers/retrieval.py | 14 + backend/open_webui/tools/builtin.py | 56 +- src/lib/apis/knowledge/index.ts | 302 +++++- src/lib/apis/retrieval/index.ts | 1 + src/lib/components/admin/Settings.svelte | 16 +- .../admin/Settings/Documents.svelte | 111 ++- .../admin/Settings/ExternalKnowledge.svelte | 883 ++++++++++++++++++ .../admin/Settings/Integrations.svelte | 73 +- src/lib/components/workspace/Knowledge.svelte | 85 +- .../workspace/Knowledge/KnowledgeBase.svelte | 635 +++++++------ 15 files changed, 2929 insertions(+), 394 deletions(-) create mode 100644 backend/open_webui/retrieval/external.py create mode 100644 src/lib/components/admin/Settings/ExternalKnowledge.svelte diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index ba4a378548..4f6ca7af51 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -857,6 +857,16 @@ EXTERNAL_DOCUMENT_LOADER_URL = os.getenv('EXTERNAL_DOCUMENT_LOADER_URL', '') EXTERNAL_DOCUMENT_LOADER_API_KEY = os.getenv('EXTERNAL_DOCUMENT_LOADER_API_KEY', '') +external_document_loader_headers = os.getenv('EXTERNAL_DOCUMENT_LOADER_HEADERS', '') +try: + external_document_loader_headers = json.loads(external_document_loader_headers) +except json.JSONDecodeError: + external_document_loader_headers = {} +if not isinstance(external_document_loader_headers, dict): + external_document_loader_headers = {} + +EXTERNAL_DOCUMENT_LOADER_HEADERS = external_document_loader_headers + TIKA_SERVER_URL = os.getenv('TIKA_SERVER_URL', 'http://tika:9998') DOCLING_SERVER_URL = os.getenv('DOCLING_SERVER_URL', 'http://docling:5001') @@ -2652,6 +2662,7 @@ DEFAULT_CONFIG = { 'rag.mineru_file_extensions': MINERU_FILE_EXTENSIONS, 'rag.external_document_loader_url': EXTERNAL_DOCUMENT_LOADER_URL, 'rag.external_document_loader_api_key': EXTERNAL_DOCUMENT_LOADER_API_KEY, + 'rag.external_document_loader_headers': EXTERNAL_DOCUMENT_LOADER_HEADERS, 'rag.tika_server_url': TIKA_SERVER_URL, 'rag.docling_server_url': DOCLING_SERVER_URL, 'rag.docling_api_key': DOCLING_API_KEY, diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index 84cf4b7ae8..f0ca77a6e7 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -286,6 +286,17 @@ class KnowledgeTable: elif view_option == 'shared': stmt = stmt.filter(Knowledge.user_id != user_id) + source = filter.get('source') + if source == 'external': + stmt = stmt.filter(Knowledge.meta['source'].as_string() == 'external') + elif source == 'local': + stmt = stmt.filter( + or_( + Knowledge.meta.is_(None), + Knowledge.meta['source'].as_string() != 'external', + ) + ) + stmt = AccessGrants.has_permission_filter( db=db, query=stmt, @@ -765,6 +776,25 @@ class KnowledgeTable: log.exception(e) return None + async def update_knowledge_meta_by_id( + self, id: str, meta: dict, db: Optional[AsyncSession] = None + ) -> Optional[KnowledgeModel]: + try: + async with get_async_db_context(db) as db: + await db.execute( + update(Knowledge) + .filter_by(id=id) + .values( + meta=meta, + updated_at=int(time.time()), + ) + ) + await db.commit() + return await self.get_knowledge_by_id(id=id, db=db) + except Exception as e: + log.exception(e) + return None + async def delete_knowledge_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool: try: async with get_async_db_context(db) as db: diff --git a/backend/open_webui/retrieval/external.py b/backend/open_webui/retrieval/external.py new file mode 100644 index 0000000000..802132b0ba --- /dev/null +++ b/backend/open_webui/retrieval/external.py @@ -0,0 +1,378 @@ +import asyncio +import logging +import re +import time +from typing import Any, Optional + +from open_webui.models.config import Config +from open_webui.models.knowledge import KnowledgeModel + +log = logging.getLogger(__name__) + +EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections' +IDENTIFIER_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + +async def _get_external_connection(connection_id: str) -> Optional[dict]: + connections = await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or [] + return next((connection for connection in connections if connection.get('id') == connection_id), None) + + +def _get_path(data: Any, path: Optional[str], default=None): + if not path: + return default + value = data + for part in path.split('.'): + if isinstance(value, dict): + value = value.get(part, default) + else: + return default + return value + + +def _normalize_result(result: dict, mapping: dict, knowledge: KnowledgeModel, distance: Optional[float] = None) -> dict: + content = _get_path(result, mapping.get('content_field', 'content'), '') + title = _get_path(result, mapping.get('title_field', 'title'), None) + source = _get_path(result, mapping.get('source_field', 'source'), None) + url = _get_path(result, mapping.get('url_field', 'url'), None) + document_id = _get_path(result, mapping.get('document_id_field', 'document_id'), None) + page = _get_path(result, mapping.get('page_field', 'page'), None) + metadata = _get_path(result, mapping.get('metadata_field', 'metadata'), {}) or {} + score = _get_path(result, mapping.get('score_field', 'score'), distance) + + if not isinstance(metadata, dict): + metadata = {'external_metadata': metadata} + + source_name = source or title or metadata.get('source') or metadata.get('name') or knowledge.name + metadata.update( + { + 'name': title or source_name, + 'source': source_name, + 'url': url, + 'file_id': document_id or f'external-{knowledge.id}', + 'knowledge_id': knowledge.id, + 'knowledge_name': knowledge.name, + 'external': True, + } + ) + if page is not None: + metadata['page'] = page + if document_id is not None: + metadata['document_id'] = document_id + + return { + 'content': content, + 'metadata': metadata, + 'distance': score, + } + + +def _source_config(knowledge: KnowledgeModel) -> dict: + external = (knowledge.meta or {}).get('external', {}) + source = external.get('source') or {} + return source.get('config') or {} + + +def _root_field(path: Optional[str]) -> Optional[str]: + if not path: + return None + return path.split('.')[0] + + +def _safe_identifier(value: str, label: str) -> str: + if not value or not IDENTIFIER_RE.match(value): + raise RuntimeError(f'Invalid {label}') + return value + + +async def _retrieve_qdrant(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]: + try: + from qdrant_client import QdrantClient + except ImportError as exc: + raise RuntimeError('qdrant-client is not installed') from exc + + if not embedding_function: + raise RuntimeError('Embedding function is not configured') + + config = connection.get('config') or {} + external = (knowledge.meta or {}).get('external', {}) + source = external.get('source') or {} + collection_name = source.get('name') + if not collection_name: + raise RuntimeError('External source collection is not configured') + source_config = _source_config(knowledge) + vector_field = source_config.get('vector_field') or None + + vector = await embedding_function(query) + + def _search(): + client = QdrantClient( + url=connection.get('endpoint'), + api_key=(auth_config or {}).get('api_key'), + timeout=config.get('timeout') or 30, + ) + return client.query_points( + collection_name=collection_name, + query=vector, + using=vector_field, + limit=count, + ) + + response = await asyncio.to_thread(_search) + mapping = { + 'content_field': source_config.get('content_field') or 'payload.text', + 'metadata_field': source_config.get('metadata_field') or 'payload.metadata', + 'document_id_field': source_config.get('document_id_field') or 'id', + 'score_field': 'score', + } + + normalized = [] + for point in response.points: + normalized.append(_normalize_result(point.model_dump(), mapping, knowledge, distance=point.score)) + return normalized + + +async def _retrieve_milvus(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]: + try: + from pymilvus import MilvusClient + except ImportError as exc: + raise RuntimeError('pymilvus is not installed') from exc + + if not embedding_function: + raise RuntimeError('Embedding function is not configured') + + config = connection.get('config') or {} + external = (knowledge.meta or {}).get('external', {}) + source = external.get('source') or {} + collection_name = source.get('name') + if not collection_name: + raise RuntimeError('Milvus collection is not configured') + source_config = _source_config(knowledge) + vector_field = source_config.get('vector_field') or 'vector' + content_field = source_config.get('content_field') or 'data.text' + metadata_field = source_config.get('metadata_field') or 'metadata' + + vector = await embedding_function(query) + + def _search(): + client_kwargs = { + 'uri': connection.get('endpoint'), + } + token = (auth_config or {}).get('api_key') or (auth_config or {}).get('token') + if token: + client_kwargs['token'] = token + if config.get('db_name'): + client_kwargs['db_name'] = config.get('db_name') + + client = MilvusClient(**client_kwargs) + output_fields = { + field + for field in ( + _root_field(content_field), + _root_field(metadata_field), + _root_field(source_config.get('document_id_field')), + ) + if field and field != vector_field + } + kwargs = { + 'collection_name': collection_name, + 'data': [vector], + 'anns_field': vector_field, + 'limit': count, + 'output_fields': list(output_fields), + } + return client.search(**kwargs) + + response = await asyncio.to_thread(_search) + mapping = { + 'content_field': content_field, + 'metadata_field': metadata_field, + 'document_id_field': source_config.get('document_id_field') or 'id', + 'score_field': 'distance', + } + + normalized = [] + for hit in (response[0] if response else []): + item = dict(hit) + entity = item.get('entity') or {} + result = { + **entity, + 'id': item.get('id') or entity.get('id'), + 'distance': item.get('distance'), + } + normalized.append(_normalize_result(result, mapping, knowledge, distance=item.get('distance'))) + return normalized + + +async def _retrieve_pgvector(connection, auth_config, knowledge, query, count, embedding_function) -> list[dict]: + try: + import psycopg + from pgvector.psycopg import register_vector + from psycopg.rows import dict_row + except ImportError as exc: + raise RuntimeError('psycopg and pgvector are required for pgvector retrieval') from exc + + if not embedding_function: + raise RuntimeError('Embedding function is not configured') + + config = connection.get('config') or {} + external = (knowledge.meta or {}).get('external', {}) + source = external.get('source') or {} + collection_name = source.get('name') + if not collection_name: + raise RuntimeError('pgvector collection is not configured') + source_config = _source_config(knowledge) + table_name = source_config.get('table_name') or 'document_chunk' + collection_field = source_config.get('collection_field') or 'collection_name' + content_field = source_config.get('content_field') or 'text' + vector_field = source_config.get('vector_field') or 'vector' + metadata_field = source_config.get('metadata_field') or 'vmetadata' + document_id_field = source_config.get('document_id_field') or 'id' + + vector = await embedding_function(query) + + def _search(): + from psycopg import sql + + table_identifier = sql.SQL('.').join( + sql.Identifier(_safe_identifier(part, 'table name')) for part in table_name.split('.') + ) + collection_identifier = sql.Identifier(_safe_identifier(collection_field, 'collection field')) + content_identifier = sql.Identifier(_safe_identifier(content_field, 'content field')) + vector_identifier = sql.Identifier(_safe_identifier(vector_field, 'vector field')) + document_id_identifier = sql.Identifier(_safe_identifier(document_id_field, 'document id field')) + metadata_sql = ( + sql.Identifier(_safe_identifier(metadata_field, 'metadata field')) + if metadata_field + else sql.SQL("'{}'::jsonb") + ) + + with psycopg.connect( + connection.get('endpoint'), + row_factory=dict_row, + connect_timeout=config.get('timeout') or 30, + ) as conn: + register_vector(conn) + with conn.cursor() as cur: + cur.execute( + sql.SQL( + """ + SELECT {document_id} AS id, + {content} AS content, + {metadata} AS metadata, + {vector_column} <=> %s AS distance + FROM {table_name} + WHERE {collection} = %s + ORDER BY distance ASC + LIMIT %s + """ + ).format( + document_id=document_id_identifier, + content=content_identifier, + metadata=metadata_sql, + vector_column=vector_identifier, + table_name=table_identifier, + collection=collection_identifier, + ), + (vector, collection_name, count), + ) + return cur.fetchall() + + rows = await asyncio.to_thread(_search) + mapping = { + 'content_field': 'content', + 'metadata_field': 'metadata', + 'document_id_field': 'id', + 'score_field': 'distance', + } + return [_normalize_result(row, mapping, knowledge, distance=row.get('distance')) for row in rows] + + +async def retrieve_external_knowledge( + request, + knowledge: KnowledgeModel, + queries: list[str], + count: int, + user=None, +) -> dict: + external = (knowledge.meta or {}).get('external', {}) + connection_id = external.get('connection_id') + if not connection_id: + raise RuntimeError('External knowledge connection is not configured') + + connection = await _get_external_connection(connection_id) + if not connection: + raise RuntimeError('External knowledge connection not found') + + return await retrieve_external_knowledge_for_connection(request, knowledge, connection, queries, count, user=user) + + +async def retrieve_external_knowledge_for_connection( + request, + knowledge: KnowledgeModel, + connection: dict, + queries: list[str], + count: int, + user=None, +) -> dict: + auth_config = connection.get('auth_config') or {} + if not connection.get('enabled', True): + raise RuntimeError('External knowledge connection is disabled') + + started_at = time.monotonic() + chunks = [] + provider = (connection.get('provider') or '').lower() + + for query in queries: + if provider == 'qdrant': + chunks.extend( + await _retrieve_qdrant( + connection, + auth_config, + knowledge, + query, + count, + getattr(request.app.state, 'EMBEDDING_FUNCTION', None), + ) + ) + elif provider == 'milvus': + chunks.extend( + await _retrieve_milvus( + connection, + auth_config, + knowledge, + query, + count, + getattr(request.app.state, 'EMBEDDING_FUNCTION', None), + ) + ) + elif provider == 'pgvector': + chunks.extend( + await _retrieve_pgvector( + connection, + auth_config, + knowledge, + query, + count, + getattr(request.app.state, 'EMBEDDING_FUNCTION', None), + ) + ) + else: + raise RuntimeError(f'Unsupported external knowledge provider: {connection.get("provider")}') + + chunks = chunks[:count] + log.info( + 'external_knowledge_retrieval knowledge_id=%s connection_id=%s provider=%s user_id=%s latency_ms=%s result_count=%s', + knowledge.id, + connection.get('id'), + connection.get('provider'), + getattr(user, 'id', None), + round((time.monotonic() - started_at) * 1000), + len(chunks), + ) + + return { + 'documents': [[chunk['content'] for chunk in chunks]], + 'metadatas': [[chunk['metadata'] for chunk in chunks]], + 'distances': [[chunk['distance'] for chunk in chunks]], + } diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index c8c603f3d8..f805fb4090 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -43,6 +43,7 @@ from open_webui.models.config import Config from open_webui.models.users import UserModel from open_webui.retrieval.loaders.youtube import YoutubeLoader from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT +from open_webui.retrieval.external import retrieve_external_knowledge from open_webui.retrieval.vector.factory import VECTOR_DB_CLIENT from open_webui.retrieval.vector.main import GetResult, SearchResult from open_webui.retrieval.web.utils import get_web_loader @@ -84,6 +85,7 @@ LOADER_CONFIG_KEYS = { 'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format', 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers', 'TIKA_SERVER_URL': 'rag.tika_server_url', 'DOCLING_SERVER_URL': 'rag.docling_server_url', 'DOCLING_API_KEY': 'rag.docling_api_key', @@ -130,18 +132,16 @@ def build_loader_from_config(request, config: dict): """Build a Loader instance with the admin's configured extraction engine settings.""" from open_webui.retrieval.loaders.main import Loader - loader_config = { - key: config.get(key) - for key in LOADER_CONFIG_KEYS - if key.isupper() - } + loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()} return Loader( engine=loader_config['CONTENT_EXTRACTION_ENGINE'], **{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'}, ) -def _extract_text_from_binary_response(request, response: requests.Response, url: str, loader_config: dict) -> tuple[str, list]: +def _extract_text_from_binary_response( + request, response: requests.Response, url: str, loader_config: dict +) -> tuple[str, list]: """Download response body to a temp file and extract text using the Loader pipeline.""" import mimetypes import tempfile @@ -774,11 +774,7 @@ async def query_collection_with_hybrid_search( return result native_task_results = await asyncio.gather( - *[ - process_native_query(collection_name, query) - for collection_name in collection_names - for query in queries - ] + *[process_native_query(collection_name, query) for collection_name in collection_names for query in queries] ) if native_task_results and all(result is not None for result in native_task_results): return merge_and_sort_query_results(native_task_results, k=k) @@ -1485,50 +1481,61 @@ async def get_sources_from_items( permission='read', ) ): - if item.get('context') == 'full' or bypass_embedding_and_retrieval: - if knowledge_base and ( - user.role == 'admin' - or knowledge_base.user_id == user.id - or await AccessGrants.has_access( - user_id=user.id, - resource_type='knowledge', - resource_id=knowledge_base.id, - permission='read', - ) - ): - files = await Knowledges.get_files_by_id(knowledge_base.id) + if (knowledge_base.meta or {}).get('source') == 'external': + query_result = await retrieve_external_knowledge( + request, + knowledge_base, + queries=queries, + count=k, + user=user, + ) + extracted_collections.append(knowledge_base.id) - documents = [] - metadatas = [] - for file in files: - documents.append(file.data.get('content', '')) - metadatas.append( - { - 'file_id': file.id, - 'name': file.filename, - 'source': file.filename, - } - ) - - query_result = { - 'documents': [documents], - 'metadatas': [metadatas], - } else: - if item.get('legacy'): - if BYPASS_RETRIEVAL_ACCESS_CONTROL: - collection_names = item.get('collection_names', []) - else: - # Legacy KB: item.collection_names is client-supplied. - # Validate against the KB's actual files to prevent - # cross-tenant collection name substitution. + if item.get('context') == 'full' or bypass_embedding_and_retrieval: + if knowledge_base and ( + user.role == 'admin' + or knowledge_base.user_id == user.id + or await AccessGrants.has_access( + user_id=user.id, + resource_type='knowledge', + resource_id=knowledge_base.id, + permission='read', + ) + ): files = await Knowledges.get_files_by_id(knowledge_base.id) - owned_names = {f'file-{f.id}' for f in files} - owned_names.add(knowledge_base.id) - valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names] - collection_names = valid_names if valid_names else [knowledge_base.id] + + documents = [] + metadatas = [] + for file in files: + documents.append(file.data.get('content', '')) + metadatas.append( + { + 'file_id': file.id, + 'name': file.filename, + 'source': file.filename, + } + ) + + query_result = { + 'documents': [documents], + 'metadatas': [metadatas], + } else: - collection_names.append(item['id']) + if item.get('legacy'): + if BYPASS_RETRIEVAL_ACCESS_CONTROL: + collection_names = item.get('collection_names', []) + else: + # Legacy KB: item.collection_names is client-supplied. + # Validate against the KB's actual files to prevent + # cross-tenant collection name substitution. + files = await Knowledges.get_files_by_id(knowledge_base.id) + owned_names = {f'file-{f.id}' for f in files} + owned_names.add(knowledge_base.id) + valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names] + collection_names = valid_names if valid_names else [knowledge_base.id] + else: + collection_names.append(item['id']) elif item.get('docs'): # BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL diff --git a/backend/open_webui/routers/knowledge.py b/backend/open_webui/routers/knowledge.py index 3b68671c28..54c6177245 100644 --- a/backend/open_webui/routers/knowledge.py +++ b/backend/open_webui/routers/knowledge.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio import io import logging +import time +import uuid import zipfile from typing import List, Optional from urllib.parse import quote @@ -27,6 +29,7 @@ from open_webui.models.knowledge import ( ) from open_webui.models.models import ModelForm, Models from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT +from open_webui.retrieval.external import retrieve_external_knowledge, retrieve_external_knowledge_for_connection from open_webui.routers.retrieval import ( BatchProcessFilesForm, ProcessFileForm, @@ -110,6 +113,17 @@ class KnowledgeAccessListResponse(BaseModel): total: int +def is_external_knowledge(knowledge) -> bool: + return (knowledge.meta or {}).get('source') == 'external' + + +def external_knowledge_error(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='External knowledge bases are read-only.', + ) + + @router.get('/', response_model=KnowledgeAccessListResponse) async def get_knowledge_bases( page: int | None = 1, @@ -163,6 +177,7 @@ async def get_knowledge_bases( async def search_knowledge_bases( query: str | None = None, view_option: str | None = None, + source: str | None = None, page: int | None = 1, user=Depends(get_verified_user), db: AsyncSession = Depends(get_async_session), @@ -176,6 +191,8 @@ async def search_knowledge_bases( filter['query'] = query if view_option: filter['view_option'] = view_option + if source in {'local', 'external'}: + filter['source'] = source groups = await Groups.get_groups_by_member_id(user.id, db=db) user_group_ids = {group.id for group in groups} @@ -379,6 +396,574 @@ async def reindex_knowledge_base_metadata_embeddings( return {'total': len(knowledge_bases), 'success': success_count} +############################ +# External Knowledge Sources +############################ + + +class ExternalKnowledgeSourceForm(BaseModel): + type: str = 'collection' + name: str + config: Optional[dict] = None + + +class ExternalKnowledgeCreateForm(BaseModel): + name: str + description: str = '' + connection_id: str + source: ExternalKnowledgeSourceForm + access_grants: Optional[list[dict]] = None + + +class ExternalKnowledgeSourceCreateForm(BaseModel): + name: str + description: str = '' + connection: ExternalKnowledgeConnectionForm + source: ExternalKnowledgeSourceForm + access_grants: Optional[list[dict]] = None + test_query: str + test_count: int = 5 + + +class ExternalKnowledgeSourceUpdateForm(ExternalKnowledgeSourceCreateForm): + pass + + +class ExternalKnowledgeSourceTestForm(BaseModel): + connection_id: Optional[str] = None + connection: ExternalKnowledgeConnectionForm + source: ExternalKnowledgeSourceForm + query: str + count: int = 5 + + +class ExternalKnowledgeRetrieveTestForm(BaseModel): + query: str + source: Optional[ExternalKnowledgeSourceForm] = None + count: int = 5 + + +class ExternalKnowledgeConnectionForm(BaseModel): + name: str + provider: str + endpoint: str + auth_config: Optional[dict] = None + config: Optional[dict] = None + capabilities: Optional[dict] = None + enabled: bool = True + + +class ExternalKnowledgeConnectionListResponse(BaseModel): + items: list[dict] + total: int + + +EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY = 'external_knowledge.connections' +EXTERNAL_KNOWLEDGE_PROVIDERS = {'qdrant', 'milvus', 'pgvector'} + + +def _validate_external_connection_form(form_data: ExternalKnowledgeConnectionForm) -> tuple[str, dict]: + provider = form_data.provider.lower().strip() + if provider not in EXTERNAL_KNOWLEDGE_PROVIDERS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='Unsupported external knowledge provider.', + ) + + if not form_data.name.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source name is required.') + + if not form_data.endpoint.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge source endpoint is required.') + + config = form_data.config or {} + allowed_config_keys = {'timeout'} + if provider == 'milvus': + allowed_config_keys.add('db_name') + + return provider, {key: value for key, value in config.items() if key in allowed_config_keys} + + +def _external_auth_config(provider: str, incoming: Optional[dict], existing: Optional[dict] = None) -> dict: + if provider == 'pgvector': + return {} + return existing if incoming is None else incoming or {} + + +def _normalize_external_source(source: ExternalKnowledgeSourceForm, provider: str) -> ExternalKnowledgeSourceForm: + source.type = (source.type or 'collection').strip() + source.name = source.name.strip() + + if source.type != 'collection': + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Only collection sources are supported.') + if not source.name: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Collection name is required.') + + config = source.config or {} + allowed_keys = {'content_field', 'metadata_field', 'document_id_field'} + if provider in {'qdrant', 'milvus'}: + allowed_keys.add('vector_field') + if provider == 'pgvector': + allowed_keys.update({'table_name', 'collection_field', 'vector_field'}) + + normalized_config = { + key: value.strip() if isinstance(value, str) else value + for key, value in config.items() + if key in allowed_keys and value is not None and (not isinstance(value, str) or value.strip()) + } + + if not normalized_config.get('content_field'): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Content field is required.') + if provider in {'milvus', 'pgvector'} and not normalized_config.get('vector_field'): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Vector field is required.') + + source.config = normalized_config + return source + + +def _sanitize_external_connection(connection: dict) -> dict: + sanitized = {**connection} + sanitized.pop('auth_config', None) + sanitized['auth_configured'] = bool(connection.get('auth_config')) + return sanitized + + +async def _get_external_connections() -> list[dict]: + return await Config.get(EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY, []) or [] + + +async def _set_external_connections(connections: list[dict]) -> None: + await Config.upsert({EXTERNAL_KNOWLEDGE_CONNECTIONS_CONFIG_KEY: connections}) + + +def _external_connection_dict(form_data: ExternalKnowledgeConnectionForm, user_id: str, id: Optional[str] = None) -> dict: + provider, config = _validate_external_connection_form(form_data) + now = int(time.time()) + return { + 'id': id or str(uuid.uuid4()), + 'name': form_data.name.strip(), + 'provider': provider, + 'endpoint': form_data.endpoint.strip(), + 'auth_config': _external_auth_config(provider, form_data.auth_config), + 'config': config, + 'capabilities': form_data.capabilities or {'retrieve': True}, + 'health': None, + 'enabled': form_data.enabled, + 'created_by': user_id, + 'created_at': now, + 'updated_at': now, + } + + +def _external_connection_update_dict( + form_data: ExternalKnowledgeConnectionForm, + existing: dict, +) -> dict: + provider, config = _validate_external_connection_form(form_data) + return { + **existing, + 'name': form_data.name.strip(), + 'provider': provider, + 'endpoint': form_data.endpoint.strip(), + 'auth_config': _external_auth_config(provider, form_data.auth_config, existing.get('auth_config')) or {}, + 'config': config, + 'capabilities': form_data.capabilities or {'retrieve': True}, + 'enabled': form_data.enabled, + 'updated_at': int(time.time()), + } + + +async def _get_external_connection(id: str) -> Optional[dict]: + connections = await _get_external_connections() + return next((connection for connection in connections if connection.get('id') == id), None) + + +async def _count_external_connection_mappings(connection_id: str, db: Optional[AsyncSession] = None) -> int: + count = 0 + for knowledge in await Knowledges.get_knowledge_bases(db=db): + if (knowledge.meta or {}).get('external', {}).get('connection_id') == connection_id: + count += 1 + return count + + +@router.get('/external/connections', response_model=ExternalKnowledgeConnectionListResponse) +async def get_external_knowledge_connections( + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connections = [_sanitize_external_connection(connection) for connection in await _get_external_connections()] + return ExternalKnowledgeConnectionListResponse(items=connections, total=len(connections)) + + +@router.post('/external/connections', response_model=dict) +async def create_external_knowledge_connection( + form_data: ExternalKnowledgeConnectionForm, + user=Depends(get_admin_user), +): + connections = await _get_external_connections() + connection = _external_connection_dict(form_data, user.id) + connections.append(connection) + await _set_external_connections(connections) + return _sanitize_external_connection(connection) + + +@router.get('/external/connections/{id}', response_model=dict) +async def get_external_knowledge_connection( + id: str, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connection = await _get_external_connection(id) + if not connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + return _sanitize_external_connection(connection) + + +@router.patch('/external/connections/{id}', response_model=dict) +async def update_external_knowledge_connection( + id: str, + form_data: ExternalKnowledgeConnectionForm, + user=Depends(get_admin_user), +): + connections = await _get_external_connections() + idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == id), None) + if idx is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + + connection = _external_connection_update_dict(form_data, connections[idx]) + connections[idx] = connection + await _set_external_connections(connections) + return _sanitize_external_connection(connection) + + +@router.delete('/external/connections/{id}', response_model=bool) +async def delete_external_knowledge_connection( + id: str, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connection = await _get_external_connection(id) + if not connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + + if await _count_external_connection_mappings(id, db=db) > 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail='External connection is still used by knowledge bases.', + ) + + connections = [connection for connection in await _get_external_connections() if connection.get('id') != id] + await _set_external_connections(connections) + return True + + +@router.post('/external/connections/{id}/test', response_model=dict) +async def test_external_knowledge_connection( + id: str, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connection = await _get_external_connection(id) + if not connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + + health = { + 'ok': bool(connection.get('enabled') and connection.get('endpoint')), + 'provider': connection.get('provider'), + 'checked_at': int(time.time()), + } + connections = await _get_external_connections() + for item in connections: + if item.get('id') == id: + item['health'] = health + item['updated_at'] = int(time.time()) + break + await _set_external_connections(connections) + return health + + +async def _test_external_source_definition( + request: Request, + connection: dict, + source: ExternalKnowledgeSourceForm, + query: str, + count: int, + user, +) -> dict: + if not query.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query is required.') + + source = _normalize_external_source(source, connection.get('provider')) + test_knowledge = KnowledgeResponse( + id='external-test', + user_id=user.id, + name=connection.get('name'), + description='', + meta={ + 'source': 'external', + 'read_only': True, + 'external': { + 'connection_id': connection.get('id'), + 'source': source.model_dump(), + 'provider': connection.get('provider'), + 'auth_mode': 'service_account', + 'capabilities': {'retrieve': True}, + }, + }, + access_grants=[], + created_at=int(time.time()), + updated_at=int(time.time()), + ) + result = await retrieve_external_knowledge_for_connection( + request, + test_knowledge, + connection, + [query.strip()], + count, + user=user, + ) + return { + 'documents': result.get('documents', [[]])[0], + 'metadatas': result.get('metadatas', [[]])[0], + 'distances': result.get('distances', [[]])[0], + } + + +@router.post('/external/source/test', response_model=dict) +async def test_external_knowledge_source( + request: Request, + form_data: ExternalKnowledgeSourceTestForm, + user=Depends(get_admin_user), +): + if form_data.connection_id: + existing_connection = await _get_external_connection(form_data.connection_id) + if not existing_connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.') + connection = _external_connection_update_dict(form_data.connection, existing_connection) + else: + connection = _external_connection_dict(form_data.connection, user.id, id='external-test') + + return await _test_external_source_definition( + request, + connection, + form_data.source, + form_data.query, + form_data.count, + user, + ) + + +@router.post('/external/connections/{id}/retrieve-test', response_model=dict) +async def test_external_knowledge_retrieval( + request: Request, + id: str, + form_data: ExternalKnowledgeRetrieveTestForm, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connection = await _get_external_connection(id) + if not connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + + source = form_data.source or ExternalKnowledgeSourceForm(name='test', config={'content_field': 'payload.text'}) + return await _test_external_source_definition(request, connection, source, form_data.query, form_data.count, user) + + +@router.post('/external/knowledge/create', response_model=KnowledgeResponse | None) +async def create_external_knowledge( + request: Request, + form_data: ExternalKnowledgeCreateForm, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + connection = await _get_external_connection(form_data.connection_id) + if not connection: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + if not form_data.name.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.') + source = _normalize_external_source(form_data.source, connection.get('provider')) + + form_data.access_grants = await filter_allowed_access_grants( + await Config.get('user.permissions'), + user.id, + user.role, + form_data.access_grants, + 'sharing.public_knowledge', + ) + + knowledge = await Knowledges.insert_new_knowledge( + user.id, + KnowledgeForm( + name=form_data.name.strip(), + description=form_data.description, + access_grants=form_data.access_grants, + ), + db=db, + ) + if not knowledge: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS) + + meta = { + 'source': 'external', + 'read_only': True, + 'external': { + 'connection_id': form_data.connection_id, + 'source': source.model_dump(), + 'provider': connection.get('provider'), + 'auth_mode': 'service_account', + 'capabilities': {'retrieve': True}, + }, + } + knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db) + await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description) + return knowledge + + +@router.post('/external/source/create', response_model=KnowledgeResponse | None) +async def create_external_knowledge_source( + request: Request, + form_data: ExternalKnowledgeSourceCreateForm, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + if not form_data.name.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.') + + connection = _external_connection_dict(form_data.connection, user.id) + source = _normalize_external_source(form_data.source, connection.get('provider')) + test_result = await _test_external_source_definition( + request, + connection, + source, + form_data.test_query, + form_data.test_count, + user, + ) + if not test_result.get('documents'): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.') + + form_data.access_grants = await filter_allowed_access_grants( + await Config.get('user.permissions'), + user.id, + user.role, + form_data.access_grants, + 'sharing.public_knowledge', + ) + + connections = await _get_external_connections() + connections.append(connection) + await _set_external_connections(connections) + + knowledge = await Knowledges.insert_new_knowledge( + user.id, + KnowledgeForm( + name=form_data.name.strip(), + description=form_data.description, + access_grants=form_data.access_grants, + ), + db=db, + ) + if not knowledge: + connections = [item for item in await _get_external_connections() if item.get('id') != connection.get('id')] + await _set_external_connections(connections) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.FILE_EXISTS) + + meta = { + 'source': 'external', + 'read_only': True, + 'external': { + 'connection_id': connection.get('id'), + 'source': source.model_dump(), + 'provider': connection.get('provider'), + 'auth_mode': 'service_account', + 'capabilities': {'retrieve': True}, + }, + } + knowledge = await Knowledges.update_knowledge_meta_by_id(knowledge.id, meta, db=db) + await embed_knowledge_base_metadata(request, knowledge.id, knowledge.name, knowledge.description) + return knowledge + + +@router.patch('/external/source/{id}', response_model=KnowledgeResponse | None) +async def update_external_knowledge_source( + request: Request, + id: str, + form_data: ExternalKnowledgeSourceUpdateForm, + user=Depends(get_admin_user), + db: AsyncSession = Depends(get_async_session), +): + knowledge = await Knowledges.get_knowledge_by_id(id=id, db=db) + if not knowledge or not is_external_knowledge(knowledge): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND) + if not form_data.name.strip(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Knowledge name is required.') + + connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id') + connections = await _get_external_connections() + idx = next((idx for idx, connection in enumerate(connections) if connection.get('id') == connection_id), None) + if idx is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='External connection not found.') + + existing_connection = connections[idx] + connection = _external_connection_update_dict(form_data.connection, existing_connection) + source = _normalize_external_source(form_data.source, connection.get('provider')) + test_result = await _test_external_source_definition( + request, + connection, + source, + form_data.test_query, + form_data.test_count, + user, + ) + if not test_result.get('documents'): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Test query returned no results.') + + form_data.access_grants = await filter_allowed_access_grants( + await Config.get('user.permissions'), + user.id, + user.role, + form_data.access_grants, + 'sharing.public_knowledge', + ) + + connections[idx] = connection + await _set_external_connections(connections) + + updated = await Knowledges.update_knowledge_by_id( + id=id, + form_data=KnowledgeForm( + name=form_data.name.strip(), + description=form_data.description, + access_grants=form_data.access_grants, + ), + db=db, + ) + if not updated: + connections[idx] = existing_connection + await _set_external_connections(connections) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()) + + meta = { + 'source': 'external', + 'read_only': True, + 'external': { + 'connection_id': connection.get('id'), + 'source': source.model_dump(), + 'provider': connection.get('provider'), + 'auth_mode': 'service_account', + 'capabilities': {'retrieve': True}, + }, + } + updated = await Knowledges.update_knowledge_meta_by_id(id, meta, db=db) + if not updated: + connections[idx] = existing_connection + await _set_external_connections(connections) + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT()) + + await embed_knowledge_base_metadata(request, id, updated.name, updated.description) + return updated + + ############################ # GetKnowledgeById ############################ @@ -711,6 +1296,8 @@ async def add_file_to_knowledge_by_id( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id @@ -798,6 +1385,8 @@ async def update_file_from_knowledge_by_id( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id @@ -877,6 +1466,8 @@ async def remove_file_from_knowledge_by_id( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id @@ -1004,11 +1595,21 @@ async def delete_knowledge_by_id( await Models.update_model_by_id(model.id, model_form, db=db) # Clean up vector DB - try: - await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id) - except Exception as e: - log.debug(e) - pass + if is_external_knowledge(knowledge): + connection_id = (knowledge.meta or {}).get('external', {}).get('connection_id') + if connection_id: + connections = [ + connection + for connection in await _get_external_connections() + if connection.get('id') != connection_id + ] + await _set_external_connections(connections) + else: + try: + await ASYNC_VECTOR_DB_CLIENT.delete_collection(collection_name=id) + except Exception as e: + log.debug(e) + pass # Remove knowledge base embedding await remove_knowledge_base_metadata_embedding(id) @@ -1035,6 +1636,8 @@ async def reset_knowledge_by_id( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id @@ -1263,6 +1866,8 @@ async def add_files_to_knowledge_batch( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id @@ -1380,6 +1985,8 @@ async def export_knowledge_by_id(id: str, user=Depends(get_admin_user), db: Asyn status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() files = await Knowledges.get_files_by_id(id, db=db) @@ -1441,6 +2048,8 @@ async def _verify_knowledge_write_access(id: str, user, db: AsyncSession): status_code=status.HTTP_404_NOT_FOUND, detail=ERROR_MESSAGES.NOT_FOUND, ) + if is_external_knowledge(knowledge): + external_knowledge_error() if ( knowledge.user_id != user.id and not await AccessGrants.has_access( diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index c49af3af1b..0017a13b02 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -288,6 +288,7 @@ RETRIEVAL_CONFIG_KEYS = { 'ENABLE_WEB_SEARCH': 'rag.web.search.enable', 'EXA_API_KEY': 'rag.web.search.exa_api_key', 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers', 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', 'EXTERNAL_WEB_LOADER_API_KEY': 'rag.web.loader.external_web_loader_api_key', 'EXTERNAL_WEB_LOADER_URL': 'rag.web.loader.external_web_loader_url', @@ -632,6 +633,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_API_KEY': config.DOCLING_API_KEY, @@ -846,6 +848,7 @@ class ConfigForm(BaseModel): EXTERNAL_DOCUMENT_LOADER_URL: str | None = None EXTERNAL_DOCUMENT_LOADER_API_KEY: str | None = None + EXTERNAL_DOCUMENT_LOADER_HEADERS: dict | None = None TIKA_SERVER_URL: str | None = None DOCLING_SERVER_URL: str | None = None @@ -1021,6 +1024,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None else config.EXTERNAL_DOCUMENT_LOADER_API_KEY ) + config.EXTERNAL_DOCUMENT_LOADER_HEADERS = ( + form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS + if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None + else config.EXTERNAL_DOCUMENT_LOADER_HEADERS + ) config.TIKA_SERVER_URL = ( form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL ) @@ -1336,6 +1344,7 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, 'DOCLING_API_KEY': config.DOCLING_API_KEY, @@ -1843,6 +1852,11 @@ async def process_file( loader_config = await get_loader_config() loader = build_loader_from_config(request, loader_config) loader.user = user + loader.metadata = { + 'file_id': file.id, + 'file_name': file.filename, + 'file_content_type': file.meta.get('content_type'), + } docs = await loader.aload(file.filename, file.meta.get('content_type'), file_path) docs = [ diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index 47d1657df7..8e5011cb7f 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -514,21 +514,14 @@ async def execute_code( elif engine == 'jupyter': from open_webui.utils.code_interpreter import execute_code_jupyter + jupyter_auth = await Config.get('code_interpreter.jupyter.auth') output = await execute_code_jupyter( await Config.get('code_interpreter.jupyter.url'), code, - ( - await Config.get('code_interpreter.jupyter.auth_token') - if jupyter_auth == 'token' - else None - ), - ( - await Config.get('code_interpreter.jupyter.auth_password') - if jupyter_auth == 'password' - else None - ), + (await Config.get('code_interpreter.jupyter.auth_token') if jupyter_auth == 'token' else None), + (await Config.get('code_interpreter.jupyter.auth_password') if jupyter_auth == 'password' else None), await Config.get('code_interpreter.jupyter.timeout'), ) @@ -2385,6 +2378,7 @@ async def query_knowledge_files( from open_webui.models.files import Files from open_webui.models.knowledge import Knowledges from open_webui.models.notes import Notes + from open_webui.retrieval.external import retrieve_external_knowledge from open_webui.retrieval.utils import query_collection user_id = __user__.get('id') @@ -2396,6 +2390,7 @@ async def query_knowledge_files( return json.dumps({'error': 'Embedding function not configured'}) collection_names = [] + external_knowledges = [] note_results = [] # Notes aren't vectorized, handle separately # If model has attached knowledge, use those @@ -2418,7 +2413,10 @@ async def query_knowledge_files( user_group_ids=set(user_group_ids), ) ): - collection_names.append(item_id) + if (knowledge.meta or {}).get('source') == 'external': + external_knowledges.append(knowledge) + else: + collection_names.append(item_id) elif item_type == 'file': # Individual file - use file-{id} as collection name @@ -2464,7 +2462,10 @@ async def query_knowledge_files( user_group_ids=set(user_group_ids), ) ): - collection_names.append(knowledge_id) + if (knowledge.meta or {}).get('source') == 'external': + external_knowledges.append(knowledge) + else: + collection_names.append(knowledge_id) else: # No model knowledge and no specific IDs - search all accessible KBs result = await Knowledges.search_knowledge_bases( @@ -2477,7 +2478,11 @@ async def query_knowledge_files( skip=0, limit=50, ) - collection_names = [knowledge_base.id for knowledge_base in result.items] + for knowledge_base in result.items: + if (knowledge_base.meta or {}).get('source') == 'external': + external_knowledges.append(knowledge_base) + else: + collection_names.append(knowledge_base.id) chunks = [] @@ -2509,6 +2514,31 @@ async def query_knowledge_files( chunk_info['distance'] = distances[idx] chunks.append(chunk_info) + for knowledge in external_knowledges: + query_results = await retrieve_external_knowledge( + __request__, + knowledge, + queries=[query], + count=count, + user=type('UserContext', (), {'id': user_id, 'role': user_role})(), + ) + documents = query_results.get('documents', [[]])[0] + metadatas = query_results.get('metadatas', [[]])[0] + distances = query_results.get('distances', [[]])[0] + + for idx, doc in enumerate(documents): + metadata = metadatas[idx] if idx < len(metadatas) else {} + chunk_info = { + 'content': doc, + 'source': metadata.get('source', metadata.get('name', knowledge.name)), + 'file_id': metadata.get('file_id', f'external-{knowledge.id}'), + 'type': 'external', + 'knowledge_id': knowledge.id, + } + if idx < len(distances): + chunk_info['distance'] = distances[idx] + chunks.append(chunk_info) + # Limit to requested count chunks = chunks[:count] diff --git a/src/lib/apis/knowledge/index.ts b/src/lib/apis/knowledge/index.ts index d99e36f284..feb63ccdd8 100644 --- a/src/lib/apis/knowledge/index.ts +++ b/src/lib/apis/knowledge/index.ts @@ -38,6 +38,304 @@ export const createNewKnowledge = async ( return res; }; +export const getExternalKnowledgeConnections = async (token: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, { + method: 'GET', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const createExternalKnowledgeConnection = async (token: string, connection: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(connection) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updateExternalKnowledgeConnection = async ( + token: string, + id: string, + connection: object +) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, { + method: 'PATCH', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(connection) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const deleteExternalKnowledgeConnection = async (token: string, id: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}`, { + method: 'DELETE', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const testExternalKnowledgeConnection = async (token: string, id: string) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/test`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + } + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const testExternalKnowledgeRetrieval = async ( + token: string, + id: string, + payload: object +) => { + let error = null; + + const res = await fetch( + `${WEBUI_API_BASE_URL}/knowledge/external/connections/${id}/retrieve-test`, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(payload) + } + ) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const testExternalKnowledgeSource = async (token: string, payload: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/test`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(payload) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const createExternalKnowledgeSource = async (token: string, payload: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/create`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(payload) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const updateExternalKnowledgeSource = async (token: string, id: string, payload: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/source/${id}`, { + method: 'PATCH', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(payload) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + +export const createExternalKnowledge = async (token: string, payload: object) => { + let error = null; + + const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/external/knowledge/create`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + authorization: `Bearer ${token}` + }, + body: JSON.stringify(payload) + }) + .then(async (res) => { + if (!res.ok) throw await res.json(); + return res.json(); + }) + .catch((err) => { + error = err.detail; + console.error(err); + return null; + }); + + if (error) { + throw error; + } + + return res; +}; + export const getKnowledgeBases = async (token: string = '', page: number | null = null) => { let error = null; @@ -76,13 +374,15 @@ export const searchKnowledgeBases = async ( token: string = '', query: string | null = null, viewOption: string | null = null, - page: number | null = null + page: number | null = null, + source: string | null = null ) => { let error = null; const searchParams = new URLSearchParams(); if (query) searchParams.append('query', query); if (viewOption) searchParams.append('view_option', viewOption); + if (source) searchParams.append('source', source); if (page) searchParams.append('page', page.toString()); const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/search?${searchParams.toString()}`, { diff --git a/src/lib/apis/retrieval/index.ts b/src/lib/apis/retrieval/index.ts index a84e7b6822..ffab406409 100644 --- a/src/lib/apis/retrieval/index.ts +++ b/src/lib/apis/retrieval/index.ts @@ -54,6 +54,7 @@ type RAGConfigForm = { PDF_EXTRACT_IMAGES?: boolean; ENABLE_GOOGLE_DRIVE_INTEGRATION?: boolean; ENABLE_ONEDRIVE_INTEGRATION?: boolean; + EXTERNAL_DOCUMENT_LOADER_HEADERS?: Record; chunk?: ChunkConfigForm; content_extraction?: ContentExtractConfigForm; web_loader_ssl_verification?: boolean; diff --git a/src/lib/components/admin/Settings.svelte b/src/lib/components/admin/Settings.svelte index 1927e99657..2d75066031 100644 --- a/src/lib/components/admin/Settings.svelte +++ b/src/lib/components/admin/Settings.svelte @@ -155,7 +155,21 @@ id: 'integrations', title: 'Integrations', route: '/admin/settings/integrations', - keywords: ['tools', 'integrations', 'plugins', 'extensions', 'functions', 'openapi', 'server'] + keywords: [ + 'tools', + 'integrations', + 'plugins', + 'extensions', + 'functions', + 'openapi', + 'server', + 'knowledge', + 'vector db', + 'qdrant', + 'rag', + 'retrieval', + 'sources' + ] }, { id: 'documents', diff --git a/src/lib/components/admin/Settings/Documents.svelte b/src/lib/components/admin/Settings/Documents.svelte index 92706ff94d..de40f84cf7 100644 --- a/src/lib/components/admin/Settings/Documents.svelte +++ b/src/lib/components/admin/Settings/Documents.svelte @@ -37,6 +37,7 @@ let showResetConfirm = false; let showResetUploadDirConfirm = false; let showReindexConfirm = false; + let showExternalDocumentLoaderHeadersHint = false; let RAG_EMBEDDING_ENGINE = ''; let RAG_EMBEDDING_MODEL = ''; @@ -149,6 +150,21 @@ toast.error($i18n.t('External Document Loader URL required.')); return; } + if ( + RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external' && + RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS + ) { + try { + const headers = JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS); + if (headers === null || typeof headers !== 'object' || Array.isArray(headers)) { + throw new Error('Headers must be a valid JSON object'); + } + RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS = JSON.stringify(headers, null, 2); + } catch (error) { + toast.error($i18n.t('Headers must be a valid JSON object')); + return; + } + } if (RAGConfig.CONTENT_EXTRACTION_ENGINE === 'tika' && RAGConfig.TIKA_SERVER_URL === '') { toast.error($i18n.t('Tika Server URL required.')); return; @@ -241,6 +257,11 @@ typeof RAGConfig.DOCLING_PARAMS === 'string' && RAGConfig.DOCLING_PARAMS.trim() !== '' ? JSON.parse(RAGConfig.DOCLING_PARAMS) : {}, + EXTERNAL_DOCUMENT_LOADER_HEADERS: + typeof RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'string' && + RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS.trim() !== '' + ? JSON.parse(RAGConfig.EXTERNAL_DOCUMENT_LOADER_HEADERS) + : {}, MINERU_PARAMS: typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== '' ? JSON.parse(RAGConfig.MINERU_PARAMS) @@ -289,6 +310,13 @@ ? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2) : config.MINERU_PARAMS; + config.EXTERNAL_DOCUMENT_LOADER_HEADERS = + typeof config.EXTERNAL_DOCUMENT_LOADER_HEADERS === 'object' + ? Object.keys(config.EXTERNAL_DOCUMENT_LOADER_HEADERS ?? {}).length > 0 + ? JSON.stringify(config.EXTERNAL_DOCUMENT_LOADER_HEADERS, null, 2) + : '' + : config.EXTERNAL_DOCUMENT_LOADER_HEADERS; + config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', '); RAGConfig = config; @@ -581,17 +609,78 @@ {:else if RAGConfig.CONTENT_EXTRACTION_ENGINE === 'external'} -
- - +
+
+ + +
+
+
{$i18n.t('Headers')}
+ + +
+ +
+
+
+ +
+
+ +
+
+
+ +
+
+
+ +
+
+ +
+
+ + {#if sourceForm.provider !== 'pgvector'} +
+
+ +
+
+ +
+
+ {/if} +
+ + {#if sourceForm.provider === 'milvus'} +
+
+
+ +
+
+ +
+
+
+ {/if} + +
+ +
+
+
+ +
+
+ +
+
+
+ + {#if sourceForm.provider === 'pgvector'} +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ {/if} + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+
+ +
+
+ + + + + +
+ +
+ {$i18n.t( + 'External vectors must be generated with the same embedding model configured in Open WebUI.' + )} +
+
+
+ +
+ + + +
+
+ + +
+
+ +
+ + + + +
+
+
+
{$i18n.t('External Knowledge Sources')}
+ {$i18n.t('Experimental')} +
+ + + + +
+ +
+ {#each items as item} + {@const connection = connectionForItem(item)} +
+ +
+
+ + + + +
+ {item.name} + {' '} + + {item?.meta?.external?.provider ?? connection?.provider} + {item?.meta?.external?.source?.name ? `ยท ${item.meta.external.source.name}` : ''} + +
+
+
+ + +
+ + + + + + { + toggleSource(item); + }} + /> + +
+
+ {/each} +
+ + {#if loading} +
+ +
+ {:else if items.length === 0} +
+ {$i18n.t('No external knowledge sources configured.')} +
+ {/if} + +
+
+ {$i18n.t( + 'Create one read-only Knowledge source per external collection. Test must pass before the source is created.' + )} +
+
+
diff --git a/src/lib/components/admin/Settings/Integrations.svelte b/src/lib/components/admin/Settings/Integrations.svelte index 1792104165..81286256f4 100644 --- a/src/lib/components/admin/Settings/Integrations.svelte +++ b/src/lib/components/admin/Settings/Integrations.svelte @@ -1,11 +1,12 @@ @@ -241,6 +265,19 @@ await tick(); }} /> + + @@ -267,11 +304,23 @@
-
- -
+ {#if item?.meta?.source === 'external'} +
+ +
+
+ +
+ {:else} +
+ +
+ {/if} - {#if !item?.write_access} + {#if !item?.write_access && item?.meta?.source !== 'external'}
@@ -282,7 +331,7 @@
{ exportHandler(item); } diff --git a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte index 82935f2080..3beefaf2fb 100644 --- a/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte +++ b/src/lib/components/workspace/Knowledge/KnowledgeBase.svelte @@ -5,7 +5,10 @@ import { PaneGroup, Pane, PaneResizer } from 'paneforge'; import { onMount, getContext, onDestroy, tick } from 'svelte'; - const i18n = getContext('i18n'); + import type { Writable } from 'svelte/store'; + import type { i18n as i18nType } from 'i18next'; + + const i18n = getContext>('i18n'); import { goto } from '$app/navigation'; import { page } from '$app/stores'; @@ -40,7 +43,8 @@ deleteKnowledgeDirectory, moveFileInKnowledge, syncKnowledgeDiff, - syncKnowledgeCleanup + syncKnowledgeCleanup, + testExternalKnowledgeRetrieval } from '$lib/apis/knowledge'; import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval'; @@ -98,11 +102,13 @@ files: any[]; access_grants?: any[]; write_access?: boolean; + meta?: any; }; let id = null; let knowledge: Knowledge | null = null; let knowledgeId = null; + let isExternalKnowledge = false; let selectedFileId = null; let selectedFile = null; @@ -132,6 +138,14 @@ let deleteDirectoryContents = true; let pendingPollTimer: ReturnType | null = null; + let externalTestQuery = ''; + let externalTestResult: { + documents?: string[]; + metadatas?: Record[]; + distances?: number[]; + } | null = null; + + $: isExternalKnowledge = knowledge?.meta?.source === 'external'; const reset = () => { currentPage = 1; @@ -243,6 +257,24 @@ } }; + const externalTestHandler = async () => { + if (!isExternalKnowledge || !externalTestQuery.trim()) return; + + const external = knowledge?.meta?.external ?? {}; + const res = await testExternalKnowledgeRetrieval(localStorage.token, external.connection_id, { + query: externalTestQuery, + source: external.source, + count: 5 + }).catch((e) => { + toast.error(`${e}`); + return null; + }); + + if (res) { + externalTestResult = res; + } + }; + const createFileFromText = (name, content) => { const blob = new Blob([content], { type: 'text/plain' }); const file = blobToFile(blob, `${name}.txt`); @@ -1299,284 +1331,357 @@
-
-
-
- -
- { - selectedFileId = null; - }} - /> - - - - -
-
- -
+ {#if isExternalKnowledge} +
+
+
+ {$i18n.t('Connected')}
- +
+ {$i18n.t('Read Only')} +
+
+ {knowledge?.meta?.external?.provider ?? $i18n.t('Provider')} +
+
+ {$i18n.t('Service Account')} +
+
- {#if knowledge?.write_access} +
- { - if (data.type === 'directory') { - uploadDirectoryHandler(); - } else if (data.type === 'new_directory') { - showNewDirectoryModal = true; - } else if (data.type === 'web') { - showAddWebpageModal = true; - } else if (data.type === 'text') { - showAddTextContentModal = true; - } else { - document.getElementById('files-input').click(); - } - }} - onSync={async () => { - pendingSyncFiles = await collectDirectoryFiles(); - if (pendingSyncFiles?.length) { - showSyncConfirmModal = true; - } - }} - onReset={() => { - showResetConfirm = true; - }} - /> +
{$i18n.t('Mapped Source')}
+
+ {knowledge?.meta?.external?.source?.name ?? $i18n.t('Not configured')} +
- {/if} -
-
- -
-
{ - if (e.deltaY !== 0) { - e.preventDefault(); - e.currentTarget.scrollLeft += e.deltaY; - } - }} - > -
- { - if (value) { - localStorage.workspaceViewOption = value; - } else { - delete localStorage.workspaceViewOption; - } - }} - /> - - - - {#if sortKey} - - {/if} -
-
-
- - {#if currentDirectoryId !== null} -
- navigateToDirectory(dirId)} - onMoveFile={(fileId, dirId) => moveFileToDirectoryHandler(fileId, dirId)} - onMoveDir={(dirId, targetId) => moveDirectoryHandler(dirId, targetId)} - /> -
- {/if} - - {#if syncing} -
-
- -
- {syncing} -
-
-
- {/if} - - {#if fileItems !== null && fileItemsTotal !== null} -
-
-
-
- {#if fileItems.length > 0 || directoryItems.length > 0} -
- { - selectedFileId = fileId; - - if (fileItems) { - const file = fileItems.find((file) => file.id === selectedFileId); - if (file) { - fileSelectHandler(file); - } else { - selectedFile = null; - } - } - }} - onDelete={(fileId) => { - selectedFileId = null; - selectedFile = null; - - deleteFileHandler(fileId); - }} - onRename={(fileId, name) => renameFileHandler(fileId, name)} - onNavigateDirectory={(dirId) => navigateToDirectory(dirId)} - onRenameDirectory={(id, name) => renameDirectoryHandler(id, name)} - onDeleteDirectory={(id) => confirmDeleteDirectory(id)} - onMoveFileToDirectory={(fileId, dirId) => - moveFileToDirectoryHandler(fileId, dirId)} - onMoveDirectoryToDirectory={(dirId, targetId) => - moveDirectoryHandler(dirId, targetId)} - /> -
- - {#if fileItemsTotal > 30} - - {/if} - {:else} -
-
- {$i18n.t('No content found')} -
-
- {/if} +
+
{$i18n.t('Auth Mode')}
+
+ {$i18n.t('Admin-managed service account')}
- {#if selectedFileId !== null} - { - selectedFileId = null; - selectedFile = null; - }} - > -
-
-
-
- -
-
- {selectedFile?.meta?.name} -
+
+ {$i18n.t( + 'This knowledge base retrieves from a connected source. Open WebUI can query it, but cannot upload, sync, edit, delete, reset, or reindex its source data.' + )} +
- {#if knowledge?.write_access} -
- -
- {/if} +
+
{$i18n.t('Test Query')}
+
+ + +
+
+ + {#if externalTestResult} +
+
{$i18n.t('Preview')}
+ {#each externalTestResult.documents ?? [] as document, idx} +
+
{document}
+
+ {externalTestResult.metadatas?.[idx]?.source ?? ''}
- - {#key selectedFile.id} -