diff --git a/backend/open_webui/retrieval/loaders/main.py b/backend/open_webui/retrieval/loaders/main.py index 5f274d4842..eeae076727 100644 --- a/backend/open_webui/retrieval/loaders/main.py +++ b/backend/open_webui/retrieval/loaders/main.py @@ -330,12 +330,12 @@ class DoclingLoader: class Loader: - def __init__(self, engine: str = '', **kwargs): - self.engine = engine - self.user = kwargs.get('user', None) - self.user_groups = kwargs.get('user_groups', None) - self.metadata = kwargs.get('metadata', {}) - self.kwargs = kwargs + def __init__(self, config: dict): + self.config = config + self.engine = config['rag.content_extraction_engine'] + self.user = None + self.user_groups = None + self.metadata = {} def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]: loader = self._get_loader(filename, file_content_type, file_path) @@ -365,7 +365,7 @@ class Loader: # is offloaded to a thread without a running event loop. if self.engine == 'external' and self.user_groups is None: self.user_groups = await get_user_groups_for_custom_headers( - self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_HEADERS'), self.user + self.config['rag.external_document_loader_headers'], self.user ) return await asyncio.to_thread(self.load, filename, file_content_type, file_path) @@ -525,7 +525,7 @@ class Loader: file_ext = filename.split('.')[-1].lower() if file_ext in known_archive_ext or file_content_type in known_archive_content_types: - max_file_size = self.kwargs.get('FILE_MAX_SIZE') + max_file_size = self.config['rag.file.max_size'] try: max_file_size_bytes = int(max_file_size) * 1024 * 1024 if max_file_size else 100 * 1024 * 1024 except (TypeError, ValueError): @@ -547,36 +547,36 @@ class Loader: if ( self.engine == 'external' - and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL') - and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_API_KEY') + and self.config['rag.external_document_loader_url'] + and self.config['rag.external_document_loader_api_key'] ): loader = ExternalDocumentLoader( file_path=file_path, - url=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL'), - api_key=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_API_KEY'), + url=self.config['rag.external_document_loader_url'], + api_key=self.config['rag.external_document_loader_api_key'], mime_type=file_content_type, user=self.user, user_groups=self.user_groups, - headers=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_HEADERS'), + headers=self.config['rag.external_document_loader_headers'], metadata={ **self.metadata, 'file_name': filename, 'file_content_type': file_content_type, }, ) - elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'): + elif self.engine == 'tika' and self.config['rag.tika_server_url']: if self._is_text_file(file_ext, file_content_type): loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) else: loader = TikaLoader( - url=self.kwargs.get('TIKA_SERVER_URL'), + url=self.config['rag.tika_server_url'], file_path=file_path, - server_version=self.kwargs.get('TIKA_SERVER_VERSION'), - extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'), + server_version=self.config['rag.tika_server_version'], + extract_images=self.config['rag.pdf_extract_images'], ) elif ( self.engine == 'datalab_marker' - and self.kwargs.get('DATALAB_MARKER_API_KEY') + and self.config['rag.datalab_marker_api_key'] and file_ext in [ 'pdf', @@ -599,30 +599,30 @@ class Loader: 'tiff', ] ): - api_base_url = self.kwargs.get('DATALAB_MARKER_API_BASE_URL', '') + api_base_url = self.config['rag.datalab_marker_api_base_url'] if not api_base_url or api_base_url.strip() == '': api_base_url = 'https://www.datalab.to/api/v1/marker' # https://github.com/open-webui/open-webui/pull/16867#issuecomment-3218424349 loader = DatalabMarkerLoader( file_path=file_path, - api_key=self.kwargs['DATALAB_MARKER_API_KEY'], + api_key=self.config['rag.datalab_marker_api_key'], api_base_url=api_base_url, - additional_config=self.kwargs.get('DATALAB_MARKER_ADDITIONAL_CONFIG'), - use_llm=self.kwargs.get('DATALAB_MARKER_USE_LLM', False), - skip_cache=self.kwargs.get('DATALAB_MARKER_SKIP_CACHE', False), - force_ocr=self.kwargs.get('DATALAB_MARKER_FORCE_OCR', False), - paginate=self.kwargs.get('DATALAB_MARKER_PAGINATE', False), - strip_existing_ocr=self.kwargs.get('DATALAB_MARKER_STRIP_EXISTING_OCR', False), - disable_image_extraction=self.kwargs.get('DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION', False), - format_lines=self.kwargs.get('DATALAB_MARKER_FORMAT_LINES', False), - output_format=self.kwargs.get('DATALAB_MARKER_OUTPUT_FORMAT', 'markdown'), + additional_config=self.config['rag.datalab_marker_additional_config'], + use_llm=self.config['rag.datalab_marker_use_llm'], + skip_cache=self.config['rag.datalab_marker_skip_cache'], + force_ocr=self.config['rag.datalab_marker_force_ocr'], + paginate=self.config['rag.datalab_marker_paginate'], + strip_existing_ocr=self.config['rag.datalab_marker_strip_existing_ocr'], + disable_image_extraction=self.config['rag.datalab_marker_disable_image_extraction'], + format_lines=self.config['rag.datalab_marker_format_lines'], + output_format=self.config['rag.datalab_marker_output_format'], ) - elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'): + elif self.engine == 'docling' and self.config['rag.docling_server_url']: if self._is_text_file(file_ext, file_content_type): loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path)) else: # Build params for DoclingLoader - params = self.kwargs.get('DOCLING_PARAMS', {}) + params = self.config['rag.docling_params'] if not isinstance(params, dict): try: params = JSONCodec.loads(params) @@ -631,15 +631,15 @@ class Loader: params = {} loader = DoclingLoader( - url=self.kwargs.get('DOCLING_SERVER_URL'), - api_key=self.kwargs.get('DOCLING_API_KEY', None), + url=self.config['rag.docling_server_url'], + api_key=self.config['rag.docling_api_key'], file_path=file_path, mime_type=file_content_type, params=params, ) elif ( self.engine == 'document_intelligence' - and self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT') != '' + and self.config['rag.document_intelligence_endpoint'] != '' and ( file_ext in ['pdf', 'docx', 'ppt', 'pptx'] or file_content_type @@ -650,22 +650,22 @@ class Loader: ] ) ): - if self.kwargs.get('DOCUMENT_INTELLIGENCE_KEY') != '': + if self.config['rag.document_intelligence_key'] != '': loader = DocumentIntelligenceLoader( file_path=file_path, - api_endpoint=self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT'), - api_key=self.kwargs.get('DOCUMENT_INTELLIGENCE_KEY'), - api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'), + api_endpoint=self.config['rag.document_intelligence_endpoint'], + api_key=self.config['rag.document_intelligence_key'], + api_model=self.config['rag.document_intelligence_model'], ) else: loader = DocumentIntelligenceLoader( file_path=file_path, - api_endpoint=self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT'), + api_endpoint=self.config['rag.document_intelligence_endpoint'], azure_credential=DefaultAzureCredential(), - api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'), + api_model=self.config['rag.document_intelligence_model'], ) - elif self.engine == 'mineru' and file_ext in self.kwargs.get('MINERU_FILE_EXTENSIONS', ['pdf']): - mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300) + elif self.engine == 'mineru' and file_ext in self.config['rag.mineru_file_extensions']: + mineru_timeout = self.config['rag.mineru_api_timeout'] if mineru_timeout: try: mineru_timeout = int(mineru_timeout) @@ -673,34 +673,34 @@ class Loader: mineru_timeout = 300 loader = MinerULoader( file_path=file_path, - api_mode=self.kwargs.get('MINERU_API_MODE', 'local'), - api_url=self.kwargs.get('MINERU_API_URL', 'http://localhost:8000'), - api_key=self.kwargs.get('MINERU_API_KEY', ''), - params=self.kwargs.get('MINERU_PARAMS', {}), + api_mode=self.config['rag.mineru_api_mode'], + api_url=self.config['rag.mineru_api_url'], + api_key=self.config['rag.mineru_api_key'], + params=self.config['rag.mineru_params'], timeout=mineru_timeout, max_markdown_bytes=MINERU_MAX_MARKDOWN_BYTES, ) elif ( self.engine == 'mistral_ocr' - and self.kwargs.get('MISTRAL_OCR_API_KEY') != '' + and self.config['rag.mistral_ocr_api_key'] != '' and file_ext in ['pdf'] # Mistral OCR currently only supports PDF and images ): loader = MistralLoader( - base_url=self.kwargs.get('MISTRAL_OCR_API_BASE_URL'), - api_key=self.kwargs.get('MISTRAL_OCR_API_KEY'), + base_url=self.config['rag.mistral_ocr_api_base_url'], + api_key=self.config['rag.mistral_ocr_api_key'], file_path=file_path, - use_base64=self.kwargs.get('MISTRAL_OCR_USE_BASE64', False), + use_base64=self.config['rag.mistral_ocr_use_base64'], user=self.user, ) elif ( self.engine == 'paddleocr_vl' - and self.kwargs.get('PADDLEOCR_VL_BASE_URL') - and self.kwargs.get('PADDLEOCR_VL_TOKEN') + and self.config['rag.paddleocr_vl_base_url'] + and self.config['rag.paddleocr_vl_token'] and file_ext in PADDLEOCR_VL_SUPPORTED_EXTENSIONS ): loader = PaddleOCRVLLoader( - api_url=self.kwargs.get('PADDLEOCR_VL_BASE_URL'), - token=self.kwargs.get('PADDLEOCR_VL_TOKEN'), + api_url=self.config['rag.paddleocr_vl_base_url'], + token=self.config['rag.paddleocr_vl_token'], file_path=file_path, ) else: @@ -720,8 +720,8 @@ class Loader: if file_ext == 'pdf': loader = PDFLoader( file_path, - extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'), - mode=self.kwargs.get('PDF_LOADER_MODE', 'page'), + extract_images=self.config['rag.pdf_extract_images'], + mode=self.config['rag.pdf_loader_mode'], ) elif file_ext == 'csv': loader = CSVLoaderWithSummary( diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 1d2fdfd01a..da9ab85c06 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -80,101 +80,25 @@ def is_youtube_url(url: str) -> bool: return re.match(youtube_regex, url) is not None -LOADER_CONFIG_KEYS = { - 'file_max_size': 'rag.file.max_size', - 'youtube_language': 'rag.youtube_loader_language', - 'youtube_proxy_url': 'rag.youtube_loader_proxy_url', - 'web_loader_ssl_verification': 'web.loader.ssl_verification', - 'web_loader_concurrent_requests': 'web.loader.concurrent_requests', - 'web_search_trust_env': 'web.search.trust_env', - 'web_loader_engine': 'web.loader.engine', - 'web_loader_timeout': 'web.loader.timeout', - 'playwright_ws_url': 'web.loader.playwright_ws_url', - 'playwright_timeout': 'web.loader.playwright_timeout', - 'firecrawl_api_key': 'web.loader.firecrawl_api_key', - 'firecrawl_api_url': 'web.loader.firecrawl_api_url', - 'firecrawl_timeout': 'web.loader.firecrawl_timeout', - 'tavily_api_key': 'web.search.tavily_api_key', - 'tavily_extract_depth': 'web.search.tavily_extract_depth', - 'microsoft_web_iq_api_base_url': 'web.search.microsoft_web_iq_api_base_url', - 'microsoft_web_iq_api_key': 'web.search.microsoft_web_iq_api_key', - 'microsoft_web_iq_language': 'web.search.microsoft_web_iq_language', - 'external_web_loader_url': 'web.loader.external_web_loader_url', - 'external_web_loader_api_key': 'web.loader.external_web_loader_api_key', - 'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine', - 'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key', - 'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url', - 'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config', - 'DATALAB_MARKER_SKIP_CACHE': 'rag.datalab_marker_skip_cache', - 'DATALAB_MARKER_FORCE_OCR': 'rag.datalab_marker_force_ocr', - 'DATALAB_MARKER_PAGINATE': 'rag.datalab_marker_paginate', - 'DATALAB_MARKER_STRIP_EXISTING_OCR': 'rag.datalab_marker_strip_existing_ocr', - 'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': 'rag.datalab_marker_disable_image_extraction', - 'DATALAB_MARKER_FORMAT_LINES': 'rag.datalab_marker_format_lines', - 'DATALAB_MARKER_USE_LLM': 'rag.datalab_marker_use_llm', - 'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format', - 'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url', - 'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key', - 'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers', - 'TIKA_SERVER_URL': 'rag.tika_server_url', - 'TIKA_SERVER_VERSION': 'rag.tika_server_version', - 'DOCLING_SERVER_URL': 'rag.docling_server_url', - 'DOCLING_API_KEY': 'rag.docling_api_key', - 'DOCLING_PARAMS': 'rag.docling_params', - 'PDF_EXTRACT_IMAGES': 'rag.pdf_extract_images', - 'PDF_LOADER_MODE': 'rag.pdf_loader_mode', - 'DOCUMENT_INTELLIGENCE_ENDPOINT': 'rag.document_intelligence_endpoint', - 'DOCUMENT_INTELLIGENCE_KEY': 'rag.document_intelligence_key', - 'DOCUMENT_INTELLIGENCE_MODEL': 'rag.document_intelligence_model', - 'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url', - 'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key', - 'MISTRAL_OCR_USE_BASE64': 'rag.mistral_ocr_use_base64', - 'PADDLEOCR_VL_BASE_URL': 'rag.paddleocr_vl_base_url', - 'PADDLEOCR_VL_TOKEN': 'rag.paddleocr_vl_token', - 'MINERU_API_MODE': 'rag.mineru_api_mode', - 'MINERU_API_URL': 'rag.mineru_api_url', - 'MINERU_API_KEY': 'rag.mineru_api_key', - 'MINERU_API_TIMEOUT': 'rag.mineru_api_timeout', - 'MINERU_PARAMS': 'rag.mineru_params', - 'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions', -} - - -async def get_loader_config(): - values = await Config.get_many(*LOADER_CONFIG_KEYS.values()) - return {name: values.get(key) for name, key in LOADER_CONFIG_KEYS.items()} - - def get_loader(request, url: str, config: dict): if is_youtube_url(url): return YoutubeLoader( url, - language=config.get('youtube_language'), - proxy_url=config.get('youtube_proxy_url'), + language=config['rag.youtube_loader_language'], + proxy_url=config['rag.youtube_loader_proxy_url'], ) - return get_web_loader( - url, - verify_ssl=config.get('web_loader_ssl_verification'), - requests_per_second=config.get('web_loader_concurrent_requests'), - trust_env=config.get('web_search_trust_env'), - loader_config=config, - ) + return get_web_loader(url, config) -def build_loader_from_config(request, config: dict): - """Build a Loader instance with the admin's configured extraction engine settings.""" +def build_loader_from_config(config: dict): + """Build a document loader with the shared retrieval settings.""" from open_webui.retrieval.loaders.main import Loader - loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()} - loader_config['FILE_MAX_SIZE'] = config.get('file_max_size') - return Loader( - engine=loader_config['CONTENT_EXTRACTION_ENGINE'], - **{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'}, - ) + return Loader(config) def _extract_text_from_binary_response( - request, response: requests.Response, url: str, loader_config: dict + request, response: requests.Response, url: str, config: dict ) -> tuple[str, list]: """Download response body to a temp file and extract text using the Loader pipeline.""" import mimetypes @@ -199,7 +123,7 @@ def _extract_text_from_binary_response( suffix = '.' + filename.split('.')[-1].lower() if '.' in filename else '' - max_size = loader_config.get('file_max_size') + max_size = config['rag.file.max_size'] max_bytes = int(max_size) * 1024 * 1024 if max_size else 0 tmp_fd, tmp_path = tempfile.mkstemp(suffix=suffix) @@ -213,7 +137,7 @@ def _extract_text_from_binary_response( raise ValueError(ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB')) tmp.write(chunk) - loader = build_loader_from_config(request, loader_config) + loader = build_loader_from_config(config) docs = loader.load(filename, content_type, tmp_path) for doc in docs: doc.metadata['source'] = url @@ -243,16 +167,75 @@ def _is_text_content_type(content_type: str) -> bool: return ct.endswith(('+xml', '+json')) -async def get_content_from_url(request, url: str) -> str: - loader_config = await get_loader_config() +async def get_content_from_url(request, url: str, *, config: dict | None = None) -> tuple[str, list]: + if config is None: + config = await Config.get_many( + 'rag.content_extraction_engine', + 'rag.datalab_marker_additional_config', + 'rag.datalab_marker_api_base_url', + 'rag.datalab_marker_api_key', + 'rag.datalab_marker_disable_image_extraction', + 'rag.datalab_marker_force_ocr', + 'rag.datalab_marker_format_lines', + 'rag.datalab_marker_output_format', + 'rag.datalab_marker_paginate', + 'rag.datalab_marker_skip_cache', + 'rag.datalab_marker_strip_existing_ocr', + 'rag.datalab_marker_use_llm', + 'rag.docling_api_key', + 'rag.docling_params', + 'rag.docling_server_url', + 'rag.document_intelligence_endpoint', + 'rag.document_intelligence_key', + 'rag.document_intelligence_model', + 'web.loader.ssl_verification', + 'web.search.exa_api_key', + 'rag.external_document_loader_api_key', + 'rag.external_document_loader_headers', + 'rag.external_document_loader_url', + 'web.loader.external_web_loader_api_key', + 'web.loader.external_web_loader_url', + 'rag.file.max_size', + 'web.loader.firecrawl_api_url', + 'web.loader.firecrawl_api_key', + 'web.loader.firecrawl_timeout', + 'web.search.microsoft_web_iq_api_base_url', + 'web.search.microsoft_web_iq_api_key', + 'web.search.microsoft_web_iq_language', + 'rag.mineru_api_key', + 'rag.mineru_api_mode', + 'rag.mineru_api_timeout', + 'rag.mineru_api_url', + 'rag.mineru_file_extensions', + 'rag.mineru_params', + 'rag.mistral_ocr_api_base_url', + 'rag.mistral_ocr_api_key', + 'rag.mistral_ocr_use_base64', + 'rag.paddleocr_vl_base_url', + 'rag.paddleocr_vl_token', + 'rag.pdf_extract_images', + 'rag.pdf_loader_mode', + 'web.loader.playwright_timeout', + 'web.loader.playwright_ws_url', + 'web.search.tavily_api_key', + 'web.search.tavily_extract_depth', + 'rag.tika_server_url', + 'rag.tika_server_version', + 'web.loader.concurrent_requests', + 'web.loader.engine', + 'web.loader.timeout', + 'web.search.trust_env', + 'rag.youtube_loader_language', + 'rag.youtube_loader_proxy_url', + ) # The rest of this function performs synchronous, blocking work: an SSRF-guarded # `requests` probe and a synchronous document loader (`loader.load()`). Run it in a # worker thread so the event loop stays free while waiting on network/parsing. - return await asyncio.to_thread(_get_content_from_url_sync, request, url, loader_config) + return await asyncio.to_thread(_get_content_from_url_sync, request, url, config) -def _get_content_from_url_sync(request, url: str, loader_config): +def _get_content_from_url_sync(request, url: str, config: dict): from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session # Validate URL before making any request (blocks private IPs, non-HTTP, filter list) @@ -265,7 +248,7 @@ def _get_content_from_url_sync(request, url: str, loader_config): # when allow_redirects=False, causing the binary-content path to run # and produce empty docs → HTTP 400. if is_youtube_url(url): - loader = get_loader(request, url, loader_config) + loader = get_loader(request, url, config) docs = loader.load() content = ' '.join([doc.page_content for doc in docs]) return content, docs @@ -289,14 +272,14 @@ def _get_content_from_url_sync(request, url: str, loader_config): if response is None or _is_text_content_type(content_type): if response is not None: response.close() - loader = get_loader(request, url, loader_config) + loader = get_loader(request, url, config) docs = loader.load() content = ' '.join([doc.page_content for doc in docs]) return content, docs # Binary content (PDF, DOCX, XLSX, PPTX, etc.) — download and extract try: - return _extract_text_from_binary_response(request, response, url, loader_config) + return _extract_text_from_binary_response(request, response, url, config) finally: response.close() diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 5cd9a8973a..e124d4328c 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -2,6 +2,7 @@ import asyncio import http.cookiejar import ipaddress import logging +import math import socket import ssl import time @@ -35,22 +36,10 @@ from fastapi.concurrency import run_in_threadpool from langchain_core.document_loaders import BaseLoader from langchain_core.documents import Document from open_webui.config import ( + DEFAULT_CONFIG, ENABLE_LOCAL_WEB_FETCH, - EXTERNAL_WEB_LOADER_API_KEY, - EXTERNAL_WEB_LOADER_URL, - FIRECRAWL_API_BASE_URL, - FIRECRAWL_API_KEY, - FIRECRAWL_TIMEOUT, MICROSOFT_WEB_IQ_API_BASE_URL, - MICROSOFT_WEB_IQ_API_KEY, - MICROSOFT_WEB_IQ_LANGUAGE, - PLAYWRIGHT_TIMEOUT, - PLAYWRIGHT_WS_URL, - TAVILY_API_KEY, - TAVILY_EXTRACT_DEPTH, WEB_FETCH_FILTER_LIST, - WEB_LOADER_ENGINE, - WEB_LOADER_TIMEOUT, ) from open_webui.constants import ERROR_MESSAGES from open_webui.env import ( @@ -381,6 +370,111 @@ class RateLimitMixin: self.last_request_time = datetime.now() +class SafeExaLoader(BaseLoader, RateLimitMixin): + def __init__( + self, + web_paths: Union[str, Sequence[str]], + api_key: str, + timeout: Optional[str] = None, + verify_ssl: bool = True, + trust_env: bool = False, + requests_per_second: Optional[float] = None, + continue_on_failure: bool = True, + ): + if not api_key or not api_key.strip(): + raise ValueError('Exa web loader requires an EXA_API_KEY') + self.web_paths = [web_paths] if isinstance(web_paths, str) else list(web_paths) + self.api_key = api_key + try: + request_timeout = float(timeout) + except (TypeError, ValueError): + request_timeout = 60 + self.timeout = request_timeout if math.isfinite(request_timeout) and request_timeout > 0 else 60 + self.verify_ssl = verify_ssl + self.trust_env = trust_env + self.requests_per_second = requests_per_second + self.last_request_time = None + self.continue_on_failure = continue_on_failure + + def lazy_load(self) -> Iterator[Document]: + # Exa's search models import this module for URL validation. + from open_webui.retrieval.web.exa import EXA_API_BASE + + loaded = 0 + with requests.Session() as session: + session.trust_env = self.trust_env + session.verify = self.verify_ssl + session.headers.update({'Authorization': f'Bearer {self.api_key}'}) + for offset in range(0, len(self.web_paths), 100): + urls = self.web_paths[offset : offset + 100] + self._sync_wait_for_rate_limit() + try: + response = session.post( + f'{EXA_API_BASE}/contents', + json={'urls': urls, 'text': True}, + timeout=self.timeout, + allow_redirects=False, + ) + if response.status_code in (401, 402, 403): + raise PermissionError( + f'Exa web loader authentication or billing failed (HTTP {response.status_code})' + ) + response.raise_for_status() + data = response.json() + if not isinstance(data, dict) or not isinstance(data.get('results'), list): + raise ValueError('Invalid Exa Contents response') + if data.get('statuses') is not None and not isinstance(data['statuses'], list): + raise ValueError('Invalid Exa Contents statuses') + except PermissionError: + raise + except (requests.RequestException, ValueError) as e: + # Do not log provider bodies or exception messages, which can contain credentials. + log.warning('Exa web loader batch failed (%s)', type(e).__name__) + if not self.continue_on_failure: + raise ValueError('Exa web loader request failed') from None + continue + + failed_ids = { + status.get('id') + for status in (data.get('statuses') or []) + if isinstance(status, dict) + and isinstance(status.get('id'), str) + and status.get('status') != 'success' + } + documents = {} + for result in data['results']: + if not isinstance(result, dict): + continue + source = next((result.get(key) for key in ('id', 'url') if result.get(key) in urls), None) + content = result.get('text') + if source is None or source in failed_ids or not isinstance(content, str): + continue + if not content.strip(): + continue + metadata = {'source': source} + if isinstance(result.get('title'), str): + metadata['title'] = result['title'] + documents[source] = Document(page_content=content, metadata=metadata) + + missing = len(set(urls) - documents.keys()) + if missing: + log.warning('Exa web loader could not load %s URL(s)', missing) + if not self.continue_on_failure: + raise ValueError(f'Exa web loader could not load {missing} URL(s)') + for url in urls: + if url in documents: + loaded += 1 + yield documents[url] + + if not loaded: + raise ValueError('Exa web loader could not load any page content') + + async def alazy_load(self) -> AsyncIterator[Document]: + docs = await run_in_threadpool(lambda: list(self.lazy_load())) + for doc in docs: + yield doc + + class URLProcessingMixin: async def _verify_ssl_cert(self, url: str) -> bool: """Verify SSL certificate for a URL.""" @@ -1056,10 +1150,7 @@ class SafeWebBaseLoader(BaseLoader): def get_web_loader( urls: Union[str, Sequence[str]], - verify_ssl: bool = True, - requests_per_second: int = 2, - trust_env: bool = False, - loader_config: Optional[dict] = None, + config: dict, ): # Check if the URLs are valid safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls) @@ -1068,22 +1159,20 @@ def get_web_loader( log.warning(f'All provided URLs were blocked or invalid: {urls}') raise ValueError(ERROR_MESSAGES.INVALID_URL) - loader_config = loader_config or {} + def cfg(key): + # Preserve the web loaders' fallback for legacy null settings. + value = config.get(key) + return DEFAULT_CONFIG[key] if value is None else value - def cfg(key, env_value): - # Admin-saved DB value wins; env constant covers keys never saved. - value = loader_config.get(key) - return env_value if value is None else value - - engine = cfg('web_loader_engine', WEB_LOADER_ENGINE) - web_loader_timeout = cfg('web_loader_timeout', WEB_LOADER_TIMEOUT) + engine = cfg('web.loader.engine') + web_loader_timeout = cfg('web.loader.timeout') web_loader_args = { 'web_paths': safe_urls, - 'verify_ssl': verify_ssl, - 'requests_per_second': requests_per_second, + 'verify_ssl': config['web.loader.ssl_verification'], + 'requests_per_second': config['web.loader.concurrent_requests'], 'continue_on_failure': True, - 'trust_env': trust_env, + 'trust_env': config['web.search.trust_env'], } WebLoaderClass = None @@ -1106,16 +1195,16 @@ def get_web_loader( if engine == 'playwright': WebLoaderClass = SafePlaywrightURLLoader - web_loader_args['playwright_timeout'] = cfg('playwright_timeout', PLAYWRIGHT_TIMEOUT) - playwright_ws_url = cfg('playwright_ws_url', PLAYWRIGHT_WS_URL) + web_loader_args['playwright_timeout'] = cfg('web.loader.playwright_timeout') + playwright_ws_url = cfg('web.loader.playwright_ws_url') if playwright_ws_url: web_loader_args['playwright_ws_url'] = playwright_ws_url if engine == 'firecrawl': WebLoaderClass = SafeFireCrawlLoader - web_loader_args['api_key'] = cfg('firecrawl_api_key', FIRECRAWL_API_KEY) - web_loader_args['api_url'] = cfg('firecrawl_api_url', FIRECRAWL_API_BASE_URL) - firecrawl_timeout = cfg('firecrawl_timeout', FIRECRAWL_TIMEOUT) + web_loader_args['api_key'] = cfg('web.loader.firecrawl_api_key') + web_loader_args['api_url'] = cfg('web.loader.firecrawl_api_url') + firecrawl_timeout = cfg('web.loader.firecrawl_timeout') if firecrawl_timeout: try: web_loader_args['timeout'] = int(firecrawl_timeout) @@ -1124,14 +1213,19 @@ def get_web_loader( if engine == 'tavily': WebLoaderClass = SafeTavilyLoader - web_loader_args['api_key'] = cfg('tavily_api_key', TAVILY_API_KEY) - web_loader_args['extract_depth'] = cfg('tavily_extract_depth', TAVILY_EXTRACT_DEPTH) + web_loader_args['api_key'] = cfg('web.search.tavily_api_key') + web_loader_args['extract_depth'] = cfg('web.search.tavily_extract_depth') + + if engine == 'exa': + WebLoaderClass = SafeExaLoader + web_loader_args['api_key'] = cfg('web.search.exa_api_key') + web_loader_args['timeout'] = web_loader_timeout if engine == 'microsoft_web_iq': WebLoaderClass = SafeMicrosoftWebIQLoader - web_loader_args['api_base_url'] = cfg('microsoft_web_iq_api_base_url', MICROSOFT_WEB_IQ_API_BASE_URL) - web_loader_args['api_key'] = cfg('microsoft_web_iq_api_key', MICROSOFT_WEB_IQ_API_KEY) - web_loader_args['language'] = cfg('microsoft_web_iq_language', MICROSOFT_WEB_IQ_LANGUAGE) + web_loader_args['api_base_url'] = cfg('web.search.microsoft_web_iq_api_base_url') + web_loader_args['api_key'] = cfg('web.search.microsoft_web_iq_api_key') + web_loader_args['language'] = cfg('web.search.microsoft_web_iq_language') if web_loader_timeout: try: web_loader_args['timeout'] = int(web_loader_timeout) @@ -1140,8 +1234,8 @@ def get_web_loader( if engine == 'external': WebLoaderClass = ExternalWebLoader - web_loader_args['external_url'] = cfg('external_web_loader_url', EXTERNAL_WEB_LOADER_URL) - web_loader_args['external_api_key'] = cfg('external_web_loader_api_key', EXTERNAL_WEB_LOADER_API_KEY) + web_loader_args['external_url'] = cfg('web.loader.external_web_loader_url') + web_loader_args['external_api_key'] = cfg('web.loader.external_web_loader_api_key') if WebLoaderClass: web_loader = WebLoaderClass(**web_loader_args) @@ -1156,5 +1250,5 @@ def get_web_loader( else: raise ValueError( f'Invalid WEB_LOADER_ENGINE: {engine}. ' - "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'." + "Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'exa', 'external', or 'microsoft_web_iq'." ) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 226672ec9d..0f38985cff 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -10,7 +10,6 @@ import shutil import uuid from datetime import datetime from pathlib import Path -from types import SimpleNamespace from typing import Callable, Iterator, Optional, Sequence, Union from urllib.parse import unquote, urlparse @@ -36,6 +35,7 @@ from langchain_text_splitters import ( TokenTextSplitter, ) from open_webui.config import ( + Config, DEFAULT_LOCALE, ENV, RAG_EMBEDDING_CONTENT_PREFIX, @@ -65,13 +65,11 @@ from open_webui.events import EVENTS, publish_event from open_webui.internal.db import get_async_db, get_async_session from open_webui.models.files import FileModel, Files, FileUpdateForm from open_webui.models.knowledge import Knowledges -from open_webui.models.config import Config # Document loaders from open_webui.retrieval.loaders.youtube import YoutubeLoader, YoutubeTranscriptError from open_webui.retrieval.utils import ( build_loader_from_config, - get_loader_config, filter_accessible_collections, get_content_from_url, get_embedding_function, @@ -250,8 +248,6 @@ def get_rf( ########################################## -router = APIRouter() - RETRIEVAL_CONFIG_KEYS = { 'ALLOWED_FILE_EXTENSIONS': 'rag.file.allowed_extensions', 'AZURE_AI_SEARCH_API_KEY': 'web.search.azure_ai_search_api_key', @@ -417,32 +413,7 @@ RETRIEVAL_CONFIG_KEYS = { } -class RetrievalConfig(SimpleNamespace): - def __init__(self, values: dict): - super().__init__(**values) - object.__setattr__(self, '_updates', {}) - - def __setattr__(self, key: str, value): - if key.startswith('_'): - object.__setattr__(self, key, value) - return - object.__setattr__(self, key, value) - if key in RETRIEVAL_CONFIG_KEYS: - self._updates[RETRIEVAL_CONFIG_KEYS[key]] = value - - async def save(self) -> None: - if self._updates: - await Config.upsert(dict(self._updates)) - self._updates.clear() - - -async def get_config_values(key_map: dict[str, str]) -> dict: - values = await Config.get_many(*key_map.values()) - return {field: values[storage_key] for field, storage_key in key_map.items() if storage_key in values} - - -async def get_retrieval_config() -> RetrievalConfig: - return RetrievalConfig(await get_config_values(RETRIEVAL_CONFIG_KEYS)) +router = APIRouter() class CollectionNameForm(BaseModel): @@ -469,26 +440,26 @@ class SearchForm(BaseModel): @router.get('/embedding') async def get_embedding_config(request: Request, user=Depends(get_admin_user)): - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) return { 'status': True, - 'RAG_EMBEDDING_ENGINE': config.RAG_EMBEDDING_ENGINE, - 'RAG_EMBEDDING_MODEL': config.RAG_EMBEDDING_MODEL, - 'RAG_EMBEDDING_BATCH_SIZE': config.RAG_EMBEDDING_BATCH_SIZE, - 'ENABLE_ASYNC_EMBEDDING': config.ENABLE_ASYNC_EMBEDDING, - 'RAG_EMBEDDING_CONCURRENT_REQUESTS': config.RAG_EMBEDDING_CONCURRENT_REQUESTS, + 'RAG_EMBEDDING_ENGINE': config['rag.embedding_engine'], + 'RAG_EMBEDDING_MODEL': config['rag.embedding_model'], + 'RAG_EMBEDDING_BATCH_SIZE': config['rag.embedding_batch_size'], + 'ENABLE_ASYNC_EMBEDDING': config['rag.enable_async_embedding'], + 'RAG_EMBEDDING_CONCURRENT_REQUESTS': config['rag.embedding_concurrent_requests'], 'openai_config': { - 'url': config.RAG_OPENAI_API_BASE_URL, - 'key': config.RAG_OPENAI_API_KEY, + 'url': config['rag.openai.api_base_url'], + 'key': config['rag.openai.api_key'], }, 'ollama_config': { - 'url': config.RAG_OLLAMA_BASE_URL, - 'key': config.RAG_OLLAMA_API_KEY, + 'url': config['rag.ollama.base_url'], + 'key': config['rag.ollama.api_key'], }, 'azure_openai_config': { - 'url': config.RAG_AZURE_OPENAI_BASE_URL, - 'key': config.RAG_AZURE_OPENAI_API_KEY, - 'version': config.RAG_AZURE_OPENAI_API_VERSION, + 'url': config['rag.azure_openai.base_url'], + 'key': config['rag.azure_openai.api_key'], + 'version': config['rag.azure_openai.api_version'], }, } @@ -521,8 +492,8 @@ class EmbeddingModelUpdateForm(BaseModel): async def unload_embedding_model(request: Request): - config = await get_retrieval_config() - if config.RAG_EMBEDDING_ENGINE == '': + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) + if config['rag.embedding_engine'] == '': # unloads current internal embedding model and clears VRAM cache request.app.state.ef = None request.app.state.EMBEDDING_FUNCTION = None @@ -540,84 +511,84 @@ async def unload_embedding_model(request: Request): async def update_embedding_config(request: Request, form_data: EmbeddingModelUpdateForm, user=Depends(get_admin_user)): if USE_SLIM and form_data.RAG_EMBEDDING_ENGINE == '': raise HTTPException(400, 'Slim requires an external embedding engine (openai, ollama, azure_openai).') - config = await get_retrieval_config() - log.info('Updating embedding model: %s to %s', config.RAG_EMBEDDING_MODEL, form_data.RAG_EMBEDDING_MODEL) + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) + log.info('Updating embedding model: %s to %s', config['rag.embedding_model'], form_data.RAG_EMBEDDING_MODEL) await unload_embedding_model(request) try: - config.RAG_EMBEDDING_ENGINE = form_data.RAG_EMBEDDING_ENGINE - config.RAG_EMBEDDING_MODEL = form_data.RAG_EMBEDDING_MODEL.strip() - config.RAG_EMBEDDING_BATCH_SIZE = form_data.RAG_EMBEDDING_BATCH_SIZE - config.ENABLE_ASYNC_EMBEDDING = form_data.ENABLE_ASYNC_EMBEDDING - config.RAG_EMBEDDING_CONCURRENT_REQUESTS = form_data.RAG_EMBEDDING_CONCURRENT_REQUESTS - - if config.RAG_EMBEDDING_ENGINE == 'openai' and form_data.openai_config is not None: - config.RAG_OPENAI_API_BASE_URL = form_data.openai_config.url or '' - config.RAG_OPENAI_API_KEY = form_data.openai_config.key or '' - - if config.RAG_EMBEDDING_ENGINE == 'ollama' and form_data.ollama_config is not None: - config.RAG_OLLAMA_BASE_URL = form_data.ollama_config.url or '' - config.RAG_OLLAMA_API_KEY = form_data.ollama_config.key or '' - - if config.RAG_EMBEDDING_ENGINE == 'azure_openai' and form_data.azure_openai_config is not None: - config.RAG_AZURE_OPENAI_BASE_URL = form_data.azure_openai_config.url or '' - config.RAG_AZURE_OPENAI_API_KEY = form_data.azure_openai_config.key or '' - config.RAG_AZURE_OPENAI_API_VERSION = form_data.azure_openai_config.version or '' + updates = { + 'rag.embedding_engine': form_data.RAG_EMBEDDING_ENGINE, + 'rag.embedding_model': form_data.RAG_EMBEDDING_MODEL.strip(), + 'rag.embedding_batch_size': form_data.RAG_EMBEDDING_BATCH_SIZE, + 'rag.enable_async_embedding': form_data.ENABLE_ASYNC_EMBEDDING, + 'rag.embedding_concurrent_requests': form_data.RAG_EMBEDDING_CONCURRENT_REQUESTS, + } + if form_data.RAG_EMBEDDING_ENGINE == 'openai' and form_data.openai_config is not None: + updates['rag.openai.api_base_url'] = form_data.openai_config.url or '' + updates['rag.openai.api_key'] = form_data.openai_config.key or '' + if form_data.RAG_EMBEDDING_ENGINE == 'ollama' and form_data.ollama_config is not None: + updates['rag.ollama.base_url'] = form_data.ollama_config.url or '' + updates['rag.ollama.api_key'] = form_data.ollama_config.key or '' + if form_data.RAG_EMBEDDING_ENGINE == 'azure_openai' and form_data.azure_openai_config is not None: + updates['rag.azure_openai.base_url'] = form_data.azure_openai_config.url or '' + updates['rag.azure_openai.api_key'] = form_data.azure_openai_config.key or '' + updates['rag.azure_openai.api_version'] = form_data.azure_openai_config.version or '' + config.update(updates) request.app.state.ef = get_ef( - config.RAG_EMBEDDING_ENGINE, - config.RAG_EMBEDDING_MODEL, + config['rag.embedding_engine'], + config['rag.embedding_model'], ) request.app.state.EMBEDDING_FUNCTION = get_embedding_function( - config.RAG_EMBEDDING_ENGINE, - config.RAG_EMBEDDING_MODEL, + config['rag.embedding_engine'], + config['rag.embedding_model'], request.app.state.ef, ( - config.RAG_OPENAI_API_BASE_URL - if config.RAG_EMBEDDING_ENGINE == 'openai' + config['rag.openai.api_base_url'] + if config['rag.embedding_engine'] == 'openai' else ( - config.RAG_OLLAMA_BASE_URL - if config.RAG_EMBEDDING_ENGINE == 'ollama' - else config.RAG_AZURE_OPENAI_BASE_URL + config['rag.ollama.base_url'] + if config['rag.embedding_engine'] == 'ollama' + else config['rag.azure_openai.base_url'] ) ), ( - config.RAG_OPENAI_API_KEY - if config.RAG_EMBEDDING_ENGINE == 'openai' + config['rag.openai.api_key'] + if config['rag.embedding_engine'] == 'openai' else ( - config.RAG_OLLAMA_API_KEY - if config.RAG_EMBEDDING_ENGINE == 'ollama' - else config.RAG_AZURE_OPENAI_API_KEY + config['rag.ollama.api_key'] + if config['rag.embedding_engine'] == 'ollama' + else config['rag.azure_openai.api_key'] ) ), - config.RAG_EMBEDDING_BATCH_SIZE, + config['rag.embedding_batch_size'], azure_api_version=( - config.RAG_AZURE_OPENAI_API_VERSION if config.RAG_EMBEDDING_ENGINE == 'azure_openai' else None + config['rag.azure_openai.api_version'] if config['rag.embedding_engine'] == 'azure_openai' else None ), - enable_async=config.ENABLE_ASYNC_EMBEDDING, - concurrent_requests=config.RAG_EMBEDDING_CONCURRENT_REQUESTS, + enable_async=config['rag.enable_async_embedding'], + concurrent_requests=config['rag.embedding_concurrent_requests'], ) - await config.save() + await Config.upsert(updates) return { 'status': True, - 'RAG_EMBEDDING_ENGINE': config.RAG_EMBEDDING_ENGINE, - 'RAG_EMBEDDING_MODEL': config.RAG_EMBEDDING_MODEL, - 'RAG_EMBEDDING_BATCH_SIZE': config.RAG_EMBEDDING_BATCH_SIZE, - 'ENABLE_ASYNC_EMBEDDING': config.ENABLE_ASYNC_EMBEDDING, - 'RAG_EMBEDDING_CONCURRENT_REQUESTS': config.RAG_EMBEDDING_CONCURRENT_REQUESTS, + 'RAG_EMBEDDING_ENGINE': config['rag.embedding_engine'], + 'RAG_EMBEDDING_MODEL': config['rag.embedding_model'], + 'RAG_EMBEDDING_BATCH_SIZE': config['rag.embedding_batch_size'], + 'ENABLE_ASYNC_EMBEDDING': config['rag.enable_async_embedding'], + 'RAG_EMBEDDING_CONCURRENT_REQUESTS': config['rag.embedding_concurrent_requests'], 'openai_config': { - 'url': config.RAG_OPENAI_API_BASE_URL, - 'key': config.RAG_OPENAI_API_KEY, + 'url': config['rag.openai.api_base_url'], + 'key': config['rag.openai.api_key'], }, 'ollama_config': { - 'url': config.RAG_OLLAMA_BASE_URL, - 'key': config.RAG_OLLAMA_API_KEY, + 'url': config['rag.ollama.base_url'], + 'key': config['rag.ollama.api_key'], }, 'azure_openai_config': { - 'url': config.RAG_AZURE_OPENAI_BASE_URL, - 'key': config.RAG_AZURE_OPENAI_API_KEY, - 'version': config.RAG_AZURE_OPENAI_API_VERSION, + 'url': config['rag.azure_openai.base_url'], + 'key': config['rag.azure_openai.api_key'], + 'version': config['rag.azure_openai.api_version'], }, } except Exception as e: @@ -630,164 +601,163 @@ async def update_embedding_config(request: Request, form_data: EmbeddingModelUpd @router.get('/config') async def get_rag_config(request: Request, user=Depends(get_admin_user)): - config = await get_retrieval_config() - await config.save() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) return { 'status': True, # RAG settings - 'RAG_TEMPLATE': config.RAG_TEMPLATE, - 'TOP_K': config.TOP_K, - 'BYPASS_EMBEDDING_AND_RETRIEVAL': config.BYPASS_EMBEDDING_AND_RETRIEVAL, - 'RAG_FULL_CONTEXT': config.RAG_FULL_CONTEXT, + 'RAG_TEMPLATE': config['rag.template'], + 'TOP_K': config['rag.top_k'], + 'BYPASS_EMBEDDING_AND_RETRIEVAL': config['rag.bypass_embedding_and_retrieval'], + 'RAG_FULL_CONTEXT': config['rag.full_context'], # Hybrid search settings - 'ENABLE_RAG_HYBRID_SEARCH': config.ENABLE_RAG_HYBRID_SEARCH, - 'ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS': config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS, - 'TOP_K_RERANKER': config.TOP_K_RERANKER, - 'RELEVANCE_THRESHOLD': config.RELEVANCE_THRESHOLD, - 'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT, + 'ENABLE_RAG_HYBRID_SEARCH': config['rag.enable_hybrid_search'], + 'ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS': config['rag.enable_hybrid_search_enriched_texts'], + 'TOP_K_RERANKER': config['rag.top_k_reranker'], + 'RELEVANCE_THRESHOLD': config['rag.relevance_threshold'], + 'HYBRID_BM25_WEIGHT': config['rag.hybrid_bm25_weight'], # Content extraction settings - 'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE, - 'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES, - 'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES, - 'PDF_LOADER_MODE': config.PDF_LOADER_MODE, - 'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY, - 'DATALAB_MARKER_API_BASE_URL': config.DATALAB_MARKER_API_BASE_URL, - 'DATALAB_MARKER_ADDITIONAL_CONFIG': config.DATALAB_MARKER_ADDITIONAL_CONFIG, - 'DATALAB_MARKER_SKIP_CACHE': config.DATALAB_MARKER_SKIP_CACHE, - 'DATALAB_MARKER_FORCE_OCR': config.DATALAB_MARKER_FORCE_OCR, - 'DATALAB_MARKER_PAGINATE': config.DATALAB_MARKER_PAGINATE, - 'DATALAB_MARKER_STRIP_EXISTING_OCR': config.DATALAB_MARKER_STRIP_EXISTING_OCR, - 'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION, - 'DATALAB_MARKER_FORMAT_LINES': config.DATALAB_MARKER_FORMAT_LINES, - 'DATALAB_MARKER_USE_LLM': config.DATALAB_MARKER_USE_LLM, - 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, - 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, - 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, - 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, - 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, - 'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION, - 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, - 'DOCLING_API_KEY': config.DOCLING_API_KEY, - 'DOCLING_PARAMS': config.DOCLING_PARAMS, - 'DOCUMENT_INTELLIGENCE_ENDPOINT': config.DOCUMENT_INTELLIGENCE_ENDPOINT, - 'DOCUMENT_INTELLIGENCE_KEY': config.DOCUMENT_INTELLIGENCE_KEY, - 'DOCUMENT_INTELLIGENCE_MODEL': config.DOCUMENT_INTELLIGENCE_MODEL, - 'MISTRAL_OCR_API_BASE_URL': config.MISTRAL_OCR_API_BASE_URL, - 'MISTRAL_OCR_API_KEY': config.MISTRAL_OCR_API_KEY, - 'MISTRAL_OCR_USE_BASE64': config.MISTRAL_OCR_USE_BASE64, - 'PADDLEOCR_VL_BASE_URL': config.PADDLEOCR_VL_BASE_URL, - 'PADDLEOCR_VL_TOKEN': config.PADDLEOCR_VL_TOKEN, + 'CONTENT_EXTRACTION_ENGINE': config['rag.content_extraction_engine'], + 'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config['rag.content_extraction.supported_media_mime_types'], + 'PDF_EXTRACT_IMAGES': config['rag.pdf_extract_images'], + 'PDF_LOADER_MODE': config['rag.pdf_loader_mode'], + 'DATALAB_MARKER_API_KEY': config['rag.datalab_marker_api_key'], + 'DATALAB_MARKER_API_BASE_URL': config['rag.datalab_marker_api_base_url'], + 'DATALAB_MARKER_ADDITIONAL_CONFIG': config['rag.datalab_marker_additional_config'], + 'DATALAB_MARKER_SKIP_CACHE': config['rag.datalab_marker_skip_cache'], + 'DATALAB_MARKER_FORCE_OCR': config['rag.datalab_marker_force_ocr'], + 'DATALAB_MARKER_PAGINATE': config['rag.datalab_marker_paginate'], + 'DATALAB_MARKER_STRIP_EXISTING_OCR': config['rag.datalab_marker_strip_existing_ocr'], + 'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': config['rag.datalab_marker_disable_image_extraction'], + 'DATALAB_MARKER_FORMAT_LINES': config['rag.datalab_marker_format_lines'], + 'DATALAB_MARKER_USE_LLM': config['rag.datalab_marker_use_llm'], + 'DATALAB_MARKER_OUTPUT_FORMAT': config['rag.datalab_marker_output_format'], + 'EXTERNAL_DOCUMENT_LOADER_URL': config['rag.external_document_loader_url'], + 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config['rag.external_document_loader_api_key'], + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config['rag.external_document_loader_headers'], + 'TIKA_SERVER_URL': config['rag.tika_server_url'], + 'TIKA_SERVER_VERSION': config['rag.tika_server_version'], + 'DOCLING_SERVER_URL': config['rag.docling_server_url'], + 'DOCLING_API_KEY': config['rag.docling_api_key'], + 'DOCLING_PARAMS': config['rag.docling_params'], + 'DOCUMENT_INTELLIGENCE_ENDPOINT': config['rag.document_intelligence_endpoint'], + 'DOCUMENT_INTELLIGENCE_KEY': config['rag.document_intelligence_key'], + 'DOCUMENT_INTELLIGENCE_MODEL': config['rag.document_intelligence_model'], + 'MISTRAL_OCR_API_BASE_URL': config['rag.mistral_ocr_api_base_url'], + 'MISTRAL_OCR_API_KEY': config['rag.mistral_ocr_api_key'], + 'MISTRAL_OCR_USE_BASE64': config['rag.mistral_ocr_use_base64'], + 'PADDLEOCR_VL_BASE_URL': config['rag.paddleocr_vl_base_url'], + 'PADDLEOCR_VL_TOKEN': config['rag.paddleocr_vl_token'], # MinerU settings - 'MINERU_API_MODE': config.MINERU_API_MODE, - 'MINERU_API_URL': config.MINERU_API_URL, - 'MINERU_API_KEY': config.MINERU_API_KEY, - 'MINERU_API_TIMEOUT': config.MINERU_API_TIMEOUT, - 'MINERU_PARAMS': config.MINERU_PARAMS, - 'MINERU_FILE_EXTENSIONS': config.MINERU_FILE_EXTENSIONS, + 'MINERU_API_MODE': config['rag.mineru_api_mode'], + 'MINERU_API_URL': config['rag.mineru_api_url'], + 'MINERU_API_KEY': config['rag.mineru_api_key'], + 'MINERU_API_TIMEOUT': config['rag.mineru_api_timeout'], + 'MINERU_PARAMS': config['rag.mineru_params'], + 'MINERU_FILE_EXTENSIONS': config['rag.mineru_file_extensions'], # Reranking settings - 'RAG_RERANKING_MODEL': config.RAG_RERANKING_MODEL, - 'RAG_RERANKING_ENGINE': config.RAG_RERANKING_ENGINE, - 'RAG_RERANKING_BATCH_SIZE': config.RAG_RERANKING_BATCH_SIZE, - 'RAG_EXTERNAL_RERANKER_URL': config.RAG_EXTERNAL_RERANKER_URL, - 'RAG_EXTERNAL_RERANKER_API_KEY': config.RAG_EXTERNAL_RERANKER_API_KEY, - 'RAG_EXTERNAL_RERANKER_TIMEOUT': config.RAG_EXTERNAL_RERANKER_TIMEOUT, + 'RAG_RERANKING_MODEL': config['rag.reranking_model'], + 'RAG_RERANKING_ENGINE': config['rag.reranking_engine'], + 'RAG_RERANKING_BATCH_SIZE': config['rag.reranking_batch_size'], + 'RAG_EXTERNAL_RERANKER_URL': config['rag.external_reranker_url'], + 'RAG_EXTERNAL_RERANKER_API_KEY': config['rag.external_reranker_api_key'], + 'RAG_EXTERNAL_RERANKER_TIMEOUT': config['rag.external_reranker_timeout'], # Chunking settings - 'TEXT_SPLITTER': config.TEXT_SPLITTER, - 'RAG_TOKENIZER_MODEL': config.RAG_TOKENIZER_MODEL, - 'ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER': config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER, - 'CHUNK_SIZE': config.CHUNK_SIZE, - 'CHUNK_MIN_SIZE_TARGET': config.CHUNK_MIN_SIZE_TARGET, - 'CHUNK_OVERLAP': config.CHUNK_OVERLAP, + 'TEXT_SPLITTER': config['rag.text_splitter'], + 'RAG_TOKENIZER_MODEL': config['rag.tokenizer_model'], + 'ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER': config['rag.enable_markdown_header_text_splitter'], + 'CHUNK_SIZE': config['rag.chunk_size'], + 'CHUNK_MIN_SIZE_TARGET': config['rag.chunk_min_size_target'], + 'CHUNK_OVERLAP': config['rag.chunk_overlap'], # File upload settings - 'FILE_MAX_SIZE': config.FILE_MAX_SIZE, - 'FILE_MAX_COUNT': config.FILE_MAX_COUNT, - 'FILE_IMAGE_COMPRESSION_WIDTH': config.FILE_IMAGE_COMPRESSION_WIDTH, - 'FILE_IMAGE_COMPRESSION_HEIGHT': config.FILE_IMAGE_COMPRESSION_HEIGHT, - 'ALLOWED_FILE_EXTENSIONS': config.ALLOWED_FILE_EXTENSIONS, + 'FILE_MAX_SIZE': config['rag.file.max_size'], + 'FILE_MAX_COUNT': config['rag.file.max_count'], + 'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'], + 'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'], + 'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'], # Integration settings - 'ENABLE_GOOGLE_DRIVE_INTEGRATION': config.ENABLE_GOOGLE_DRIVE_INTEGRATION, - 'ENABLE_ONEDRIVE_INTEGRATION': config.ENABLE_ONEDRIVE_INTEGRATION, + 'ENABLE_GOOGLE_DRIVE_INTEGRATION': config['google_drive.enable'], + 'ENABLE_ONEDRIVE_INTEGRATION': config['onedrive.enable'], # Web search settings 'web': { - 'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH, - 'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION, - 'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT, - 'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE, - 'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV, - 'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT, - 'WEB_SEARCH_CONCURRENT_REQUESTS': config.WEB_SEARCH_CONCURRENT_REQUESTS, - 'WEB_FETCH_MAX_CONTENT_LENGTH': config.WEB_FETCH_MAX_CONTENT_LENGTH, - 'WEB_LOADER_CONCURRENT_REQUESTS': config.WEB_LOADER_CONCURRENT_REQUESTS, - 'WEB_SEARCH_DOMAIN_FILTER_LIST': config.WEB_SEARCH_DOMAIN_FILTER_LIST, - 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL, - 'BYPASS_WEB_SEARCH_WEB_LOADER': config.BYPASS_WEB_SEARCH_WEB_LOADER, - 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY, - 'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL, - 'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE, - 'OPENSERP_BASE_URL': config.OPENSERP_BASE_URL, - 'YACY_QUERY_URL': config.YACY_QUERY_URL, - 'YACY_USERNAME': config.YACY_USERNAME, - 'YACY_PASSWORD': config.YACY_PASSWORD, - 'GOOGLE_PSE_API_KEY': config.GOOGLE_PSE_API_KEY, - 'GOOGLE_PSE_ENGINE_ID': config.GOOGLE_PSE_ENGINE_ID, - 'BRAVE_SEARCH_API_KEY': config.BRAVE_SEARCH_API_KEY, - 'BRAVE_SEARCH_CONTEXT_TOKENS': config.BRAVE_SEARCH_CONTEXT_TOKENS, - 'KAGI_SEARCH_API_KEY': config.KAGI_SEARCH_API_KEY, - 'MOJEEK_SEARCH_API_KEY': config.MOJEEK_SEARCH_API_KEY, - 'BOCHA_SEARCH_API_KEY': config.BOCHA_SEARCH_API_KEY, - 'SERPSTACK_API_KEY': config.SERPSTACK_API_KEY, - 'SERPSTACK_HTTPS': config.SERPSTACK_HTTPS, - 'SERPER_API_KEY': config.SERPER_API_KEY, - 'SERPHOUSE_API_KEY': config.SERPHOUSE_API_KEY, - 'SERPHOUSE_DOMAIN': config.SERPHOUSE_DOMAIN, - 'SERPLY_API_KEY': config.SERPLY_API_KEY, - 'DDGS_BACKEND': config.DDGS_BACKEND, - 'TAVILY_API_KEY': config.TAVILY_API_KEY, - 'STAAN_API_KEY': config.STAAN_API_KEY, - 'STAAN_MARKET': config.STAAN_MARKET, - 'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS, - 'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY, - 'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE, - 'SERPAPI_API_KEY': config.SERPAPI_API_KEY, - 'SERPAPI_ENGINE': config.SERPAPI_ENGINE, - 'JINA_API_KEY': config.JINA_API_KEY, - 'JINA_API_BASE_URL': config.JINA_API_BASE_URL, - 'BING_SEARCH_V7_ENDPOINT': config.BING_SEARCH_V7_ENDPOINT, - 'BING_SEARCH_V7_SUBSCRIPTION_KEY': config.BING_SEARCH_V7_SUBSCRIPTION_KEY, - 'EXA_API_KEY': config.EXA_API_KEY, - 'EXA_MAX_CONTENT_LENGTH': config.EXA_MAX_CONTENT_LENGTH, - 'PERPLEXITY_API_KEY': config.PERPLEXITY_API_KEY, - 'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL, - 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE, - 'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL, - 'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL, - 'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY, - 'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE, - 'SOUGOU_API_SID': config.SOUGOU_API_SID, - 'SOUGOU_API_SK': config.SOUGOU_API_SK, - 'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE, - 'WEB_LOADER_TIMEOUT': config.WEB_LOADER_TIMEOUT, - 'ENABLE_WEB_LOADER_SSL_VERIFICATION': config.ENABLE_WEB_LOADER_SSL_VERIFICATION, - 'PLAYWRIGHT_WS_URL': config.PLAYWRIGHT_WS_URL, - 'PLAYWRIGHT_TIMEOUT': config.PLAYWRIGHT_TIMEOUT, - 'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY, - 'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL, - 'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT, - 'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH, - 'TAVILY_SEARCH_DEPTH': config.TAVILY_SEARCH_DEPTH, - 'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL, - 'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY, - 'EXTERNAL_WEB_LOADER_URL': config.EXTERNAL_WEB_LOADER_URL, - 'EXTERNAL_WEB_LOADER_API_KEY': config.EXTERNAL_WEB_LOADER_API_KEY, - 'YOUTUBE_LOADER_LANGUAGE': config.YOUTUBE_LOADER_LANGUAGE, - 'YOUTUBE_LOADER_PROXY_URL': config.YOUTUBE_LOADER_PROXY_URL, + 'ENABLE_WEB_SEARCH': config['web.search.enable'], + 'ENABLE_WEB_SEARCH_CONFIRMATION': config['web.search.confirmation.enable'], + 'WEB_SEARCH_CONFIRMATION_CONTENT': config['web.search.confirmation.content'], + 'WEB_SEARCH_ENGINE': config['web.search.engine'], + 'WEB_SEARCH_TRUST_ENV': config['web.search.trust_env'], + 'WEB_SEARCH_RESULT_COUNT': config['web.search.result_count'], + 'WEB_SEARCH_CONCURRENT_REQUESTS': config['web.search.concurrent_requests'], + 'WEB_FETCH_MAX_CONTENT_LENGTH': config['web.fetch.max_content_length'], + 'WEB_LOADER_CONCURRENT_REQUESTS': config['web.loader.concurrent_requests'], + 'WEB_SEARCH_DOMAIN_FILTER_LIST': config['web.search.domain.filter_list'], + 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': config['web.search.bypass_embedding_and_retrieval'], + 'BYPASS_WEB_SEARCH_WEB_LOADER': config['web.search.bypass_web_loader'], + 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config['web.search.ollama_cloud_api_key'], + 'SEARXNG_QUERY_URL': config['web.search.searxng_query_url'], + 'SEARXNG_LANGUAGE': config['web.search.searxng_language'], + 'OPENSERP_BASE_URL': config['web.search.openserp_base_url'], + 'YACY_QUERY_URL': config['web.search.yacy_query_url'], + 'YACY_USERNAME': config['web.search.yacy_username'], + 'YACY_PASSWORD': config['web.search.yacy_password'], + 'GOOGLE_PSE_API_KEY': config['web.search.google_pse_api_key'], + 'GOOGLE_PSE_ENGINE_ID': config['web.search.google_pse_engine_id'], + 'BRAVE_SEARCH_API_KEY': config['web.search.brave_search_api_key'], + 'BRAVE_SEARCH_CONTEXT_TOKENS': config['web.search.brave_search_context_tokens'], + 'KAGI_SEARCH_API_KEY': config['web.search.kagi_search_api_key'], + 'MOJEEK_SEARCH_API_KEY': config['web.search.mojeek_search_api_key'], + 'BOCHA_SEARCH_API_KEY': config['web.search.bocha_search_api_key'], + 'SERPSTACK_API_KEY': config['web.search.serpstack_api_key'], + 'SERPSTACK_HTTPS': config['web.search.serpstack_https'], + 'SERPER_API_KEY': config['web.search.serper_api_key'], + 'SERPHOUSE_API_KEY': config['web.search.serphouse_api_key'], + 'SERPHOUSE_DOMAIN': config['web.search.serphouse_domain'], + 'SERPLY_API_KEY': config['web.search.serply_api_key'], + 'DDGS_BACKEND': config['web.search.ddgs_backend'], + 'TAVILY_API_KEY': config['web.search.tavily_api_key'], + 'STAAN_API_KEY': config['web.search.staan_api_key'], + 'STAAN_MARKET': config['web.search.staan_market'], + 'STAAN_MAX_SNIPPETS': config['web.search.staan_max_snippets'], + 'SEARCHAPI_API_KEY': config['web.search.searchapi_api_key'], + 'SEARCHAPI_ENGINE': config['web.search.searchapi_engine'], + 'SERPAPI_API_KEY': config['web.search.serpapi_api_key'], + 'SERPAPI_ENGINE': config['web.search.serpapi_engine'], + 'JINA_API_KEY': config['web.search.jina_api_key'], + 'JINA_API_BASE_URL': config['web.search.jina_api_base_url'], + 'BING_SEARCH_V7_ENDPOINT': config['web.search.bing_search_v7_endpoint'], + 'BING_SEARCH_V7_SUBSCRIPTION_KEY': config['web.search.bing_search_v7_subscription_key'], + 'EXA_API_KEY': config['web.search.exa_api_key'], + 'EXA_MAX_CONTENT_LENGTH': config['web.search.exa_max_content_length'], + 'PERPLEXITY_API_KEY': config['web.search.perplexity_api_key'], + 'PERPLEXITY_MODEL': config['web.search.perplexity_model'], + 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config['web.search.perplexity_search_context_usage'], + 'PERPLEXITY_SEARCH_API_URL': config['web.search.perplexity_search_api_url'], + 'MICROSOFT_WEB_IQ_API_BASE_URL': config['web.search.microsoft_web_iq_api_base_url'], + 'MICROSOFT_WEB_IQ_API_KEY': config['web.search.microsoft_web_iq_api_key'], + 'MICROSOFT_WEB_IQ_LANGUAGE': config['web.search.microsoft_web_iq_language'], + 'SOUGOU_API_SID': config['web.search.sougou_api_sid'], + 'SOUGOU_API_SK': config['web.search.sougou_api_sk'], + 'WEB_LOADER_ENGINE': config['web.loader.engine'], + 'WEB_LOADER_TIMEOUT': config['web.loader.timeout'], + 'ENABLE_WEB_LOADER_SSL_VERIFICATION': config['web.loader.ssl_verification'], + 'PLAYWRIGHT_WS_URL': config['web.loader.playwright_ws_url'], + 'PLAYWRIGHT_TIMEOUT': config['web.loader.playwright_timeout'], + 'FIRECRAWL_API_KEY': config['web.loader.firecrawl_api_key'], + 'FIRECRAWL_API_BASE_URL': config['web.loader.firecrawl_api_url'], + 'FIRECRAWL_TIMEOUT': config['web.loader.firecrawl_timeout'], + 'TAVILY_EXTRACT_DEPTH': config['web.search.tavily_extract_depth'], + 'TAVILY_SEARCH_DEPTH': config['web.search.tavily_search_depth'], + 'EXTERNAL_WEB_SEARCH_URL': config['web.search.external_web_search_url'], + 'EXTERNAL_WEB_SEARCH_API_KEY': config['web.search.external_web_search_api_key'], + 'EXTERNAL_WEB_LOADER_URL': config['web.loader.external_web_loader_url'], + 'EXTERNAL_WEB_LOADER_API_KEY': config['web.loader.external_web_loader_api_key'], + 'YOUTUBE_LOADER_LANGUAGE': config['rag.youtube_loader_language'], + 'YOUTUBE_LOADER_PROXY_URL': config['rag.youtube_loader_proxy_url'], 'YOUTUBE_LOADER_TRANSLATION': request.app.state.YOUTUBE_LOADER_TRANSLATION, - 'YANDEX_WEB_SEARCH_URL': config.YANDEX_WEB_SEARCH_URL, - 'YANDEX_WEB_SEARCH_API_KEY': config.YANDEX_WEB_SEARCH_API_KEY, - 'YANDEX_WEB_SEARCH_CONFIG': config.YANDEX_WEB_SEARCH_CONFIG, - 'YOUCOM_API_KEY': config.YOUCOM_API_KEY, - 'LINKUP_API_KEY': config.LINKUP_API_KEY, - 'LINKUP_SEARCH_PARAMS': config.LINKUP_SEARCH_PARAMS, + 'YANDEX_WEB_SEARCH_URL': config['web.search.yandex_web_search_url'], + 'YANDEX_WEB_SEARCH_API_KEY': config['web.search.yandex_web_search_api_key'], + 'YANDEX_WEB_SEARCH_CONFIG': config['web.search.yandex_web_search_config'], + 'YOUCOM_API_KEY': config['web.search.youcom_api_key'], + 'LINKUP_API_KEY': config['web.search.linkup_api_key'], + 'LINKUP_SEARCH_PARAMS': config['web.search.linkup_search_params'], }, } @@ -966,12 +936,12 @@ class ConfigForm(BaseModel): @router.post('/config/update') async def update_rag_config(request: Request, form_data: ConfigForm, user=Depends(get_admin_user)): # RAG settings - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) if USE_SLIM: if ( form_data.web and form_data.web.WEB_SEARCH_ENGINE == 'duckduckgo' - and config.WEB_SEARCH_ENGINE != 'duckduckgo' + and config['web.search.engine'] != 'duckduckgo' ): raise HTTPException( 400, @@ -980,217 +950,46 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend if ( form_data.web and form_data.web.WEB_LOADER_ENGINE == 'playwright' - and config.WEB_LOADER_ENGINE != 'playwright' + and config['web.loader.engine'] != 'playwright' ): raise HTTPException( 400, 'Playwright is unavailable in slim. Use basic HTTP fetching or an external web loader.' ) - if form_data.TEXT_SPLITTER == 'token_transformers' and config.TEXT_SPLITTER != 'token_transformers': + if form_data.TEXT_SPLITTER == 'token_transformers' and config['rag.text_splitter'] != 'token_transformers': raise HTTPException( 400, 'Transformers tokenization is unavailable in slim. Use character or token splitting.' ) reranker_engine = ( form_data.RAG_RERANKING_ENGINE if form_data.RAG_RERANKING_ENGINE is not None - else config.RAG_RERANKING_ENGINE + else config['rag.reranking_engine'] ) reranker_model = ( - form_data.RAG_RERANKING_MODEL if form_data.RAG_RERANKING_MODEL is not None else config.RAG_RERANKING_MODEL + form_data.RAG_RERANKING_MODEL + if form_data.RAG_RERANKING_MODEL is not None + else config['rag.reranking_model'] ) if ( reranker_engine != 'external' and reranker_model - and (reranker_engine != config.RAG_RERANKING_ENGINE or reranker_model != config.RAG_RERANKING_MODEL) + and (reranker_engine != config['rag.reranking_engine'] or reranker_model != config['rag.reranking_model']) ): raise HTTPException( 400, 'Slim requires an external reranker, or an empty reranking model for cosine scoring.' ) - config.RAG_TEMPLATE = form_data.RAG_TEMPLATE if form_data.RAG_TEMPLATE is not None else config.RAG_TEMPLATE - config.TOP_K = form_data.TOP_K if form_data.TOP_K is not None else config.TOP_K - config.BYPASS_EMBEDDING_AND_RETRIEVAL = ( - form_data.BYPASS_EMBEDDING_AND_RETRIEVAL - if form_data.BYPASS_EMBEDDING_AND_RETRIEVAL is not None - else config.BYPASS_EMBEDDING_AND_RETRIEVAL - ) - config.RAG_FULL_CONTEXT = ( - form_data.RAG_FULL_CONTEXT if form_data.RAG_FULL_CONTEXT is not None else config.RAG_FULL_CONTEXT - ) + # Only write submitted values; unrelated settings may have changed since this read. + updates = { + RETRIEVAL_CONFIG_KEYS[field]: value + for field, value in form_data.model_dump(exclude={'web'}, exclude_none=True).items() + } + if form_data.RAG_TOKENIZER_MODEL is not None: + updates['rag.tokenizer_model'] = form_data.RAG_TOKENIZER_MODEL.strip() + for field in ('FILE_MAX_SIZE', 'FILE_MAX_COUNT', 'FILE_IMAGE_COMPRESSION_WIDTH', 'FILE_IMAGE_COMPRESSION_HEIGHT'): + if getattr(form_data, field) == '': + updates[RETRIEVAL_CONFIG_KEYS[field]] = None - # Hybrid search settings - config.ENABLE_RAG_HYBRID_SEARCH = ( - form_data.ENABLE_RAG_HYBRID_SEARCH - if form_data.ENABLE_RAG_HYBRID_SEARCH is not None - else config.ENABLE_RAG_HYBRID_SEARCH - ) - config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = ( - form_data.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS - if form_data.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS is not None - else config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS - ) - - config.TOP_K_RERANKER = form_data.TOP_K_RERANKER if form_data.TOP_K_RERANKER is not None else config.TOP_K_RERANKER - config.RELEVANCE_THRESHOLD = ( - form_data.RELEVANCE_THRESHOLD if form_data.RELEVANCE_THRESHOLD is not None else config.RELEVANCE_THRESHOLD - ) - config.HYBRID_BM25_WEIGHT = ( - form_data.HYBRID_BM25_WEIGHT if form_data.HYBRID_BM25_WEIGHT is not None else config.HYBRID_BM25_WEIGHT - ) - - # Content extraction settings - config.CONTENT_EXTRACTION_ENGINE = ( - form_data.CONTENT_EXTRACTION_ENGINE - if form_data.CONTENT_EXTRACTION_ENGINE is not None - else config.CONTENT_EXTRACTION_ENGINE - ) - config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES = ( - form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES - if form_data.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES is not None - else config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES - ) - config.PDF_EXTRACT_IMAGES = ( - form_data.PDF_EXTRACT_IMAGES if form_data.PDF_EXTRACT_IMAGES is not None else config.PDF_EXTRACT_IMAGES - ) - config.PDF_LOADER_MODE = ( - form_data.PDF_LOADER_MODE if form_data.PDF_LOADER_MODE is not None else config.PDF_LOADER_MODE - ) - config.DATALAB_MARKER_API_KEY = ( - form_data.DATALAB_MARKER_API_KEY - if form_data.DATALAB_MARKER_API_KEY is not None - else config.DATALAB_MARKER_API_KEY - ) - config.DATALAB_MARKER_API_BASE_URL = ( - form_data.DATALAB_MARKER_API_BASE_URL - if form_data.DATALAB_MARKER_API_BASE_URL is not None - else config.DATALAB_MARKER_API_BASE_URL - ) - config.DATALAB_MARKER_ADDITIONAL_CONFIG = ( - form_data.DATALAB_MARKER_ADDITIONAL_CONFIG - if form_data.DATALAB_MARKER_ADDITIONAL_CONFIG is not None - else config.DATALAB_MARKER_ADDITIONAL_CONFIG - ) - config.DATALAB_MARKER_SKIP_CACHE = ( - form_data.DATALAB_MARKER_SKIP_CACHE - if form_data.DATALAB_MARKER_SKIP_CACHE is not None - else config.DATALAB_MARKER_SKIP_CACHE - ) - config.DATALAB_MARKER_FORCE_OCR = ( - form_data.DATALAB_MARKER_FORCE_OCR - if form_data.DATALAB_MARKER_FORCE_OCR is not None - else config.DATALAB_MARKER_FORCE_OCR - ) - config.DATALAB_MARKER_PAGINATE = ( - form_data.DATALAB_MARKER_PAGINATE - if form_data.DATALAB_MARKER_PAGINATE is not None - else config.DATALAB_MARKER_PAGINATE - ) - config.DATALAB_MARKER_STRIP_EXISTING_OCR = ( - form_data.DATALAB_MARKER_STRIP_EXISTING_OCR - if form_data.DATALAB_MARKER_STRIP_EXISTING_OCR is not None - else config.DATALAB_MARKER_STRIP_EXISTING_OCR - ) - config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION = ( - form_data.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION - if form_data.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION is not None - else config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION - ) - config.DATALAB_MARKER_FORMAT_LINES = ( - form_data.DATALAB_MARKER_FORMAT_LINES - if form_data.DATALAB_MARKER_FORMAT_LINES is not None - else config.DATALAB_MARKER_FORMAT_LINES - ) - config.DATALAB_MARKER_OUTPUT_FORMAT = ( - form_data.DATALAB_MARKER_OUTPUT_FORMAT - if form_data.DATALAB_MARKER_OUTPUT_FORMAT is not None - else config.DATALAB_MARKER_OUTPUT_FORMAT - ) - config.DATALAB_MARKER_USE_LLM = ( - form_data.DATALAB_MARKER_USE_LLM - if form_data.DATALAB_MARKER_USE_LLM is not None - else config.DATALAB_MARKER_USE_LLM - ) - config.EXTERNAL_DOCUMENT_LOADER_URL = ( - form_data.EXTERNAL_DOCUMENT_LOADER_URL - if form_data.EXTERNAL_DOCUMENT_LOADER_URL is not None - else config.EXTERNAL_DOCUMENT_LOADER_URL - ) - config.EXTERNAL_DOCUMENT_LOADER_API_KEY = ( - form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY - if form_data.EXTERNAL_DOCUMENT_LOADER_API_KEY is not None - else config.EXTERNAL_DOCUMENT_LOADER_API_KEY - ) - config.EXTERNAL_DOCUMENT_LOADER_HEADERS = ( - form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS - if form_data.EXTERNAL_DOCUMENT_LOADER_HEADERS is not None - else config.EXTERNAL_DOCUMENT_LOADER_HEADERS - ) - config.TIKA_SERVER_URL = ( - form_data.TIKA_SERVER_URL if form_data.TIKA_SERVER_URL is not None else config.TIKA_SERVER_URL - ) - config.TIKA_SERVER_VERSION = ( - form_data.TIKA_SERVER_VERSION if form_data.TIKA_SERVER_VERSION is not None else config.TIKA_SERVER_VERSION - ) - config.DOCLING_SERVER_URL = ( - form_data.DOCLING_SERVER_URL if form_data.DOCLING_SERVER_URL is not None else config.DOCLING_SERVER_URL - ) - config.DOCLING_API_KEY = ( - form_data.DOCLING_API_KEY if form_data.DOCLING_API_KEY is not None else config.DOCLING_API_KEY - ) - config.DOCLING_PARAMS = form_data.DOCLING_PARAMS if form_data.DOCLING_PARAMS is not None else config.DOCLING_PARAMS - config.DOCUMENT_INTELLIGENCE_ENDPOINT = ( - form_data.DOCUMENT_INTELLIGENCE_ENDPOINT - if form_data.DOCUMENT_INTELLIGENCE_ENDPOINT is not None - else config.DOCUMENT_INTELLIGENCE_ENDPOINT - ) - config.DOCUMENT_INTELLIGENCE_KEY = ( - form_data.DOCUMENT_INTELLIGENCE_KEY - if form_data.DOCUMENT_INTELLIGENCE_KEY is not None - else config.DOCUMENT_INTELLIGENCE_KEY - ) - config.DOCUMENT_INTELLIGENCE_MODEL = ( - form_data.DOCUMENT_INTELLIGENCE_MODEL - if form_data.DOCUMENT_INTELLIGENCE_MODEL is not None - else config.DOCUMENT_INTELLIGENCE_MODEL - ) - - config.MISTRAL_OCR_API_BASE_URL = ( - form_data.MISTRAL_OCR_API_BASE_URL - if form_data.MISTRAL_OCR_API_BASE_URL is not None - else config.MISTRAL_OCR_API_BASE_URL - ) - config.MISTRAL_OCR_API_KEY = ( - form_data.MISTRAL_OCR_API_KEY if form_data.MISTRAL_OCR_API_KEY is not None else config.MISTRAL_OCR_API_KEY - ) - config.MISTRAL_OCR_USE_BASE64 = ( - form_data.MISTRAL_OCR_USE_BASE64 - if form_data.MISTRAL_OCR_USE_BASE64 is not None - else config.MISTRAL_OCR_USE_BASE64 - ) - config.PADDLEOCR_VL_BASE_URL = ( - form_data.PADDLEOCR_VL_BASE_URL if form_data.PADDLEOCR_VL_BASE_URL is not None else config.PADDLEOCR_VL_BASE_URL - ) - config.PADDLEOCR_VL_TOKEN = ( - form_data.PADDLEOCR_VL_TOKEN if form_data.PADDLEOCR_VL_TOKEN is not None else config.PADDLEOCR_VL_TOKEN - ) - - # MinerU settings - config.MINERU_API_MODE = ( - form_data.MINERU_API_MODE if form_data.MINERU_API_MODE is not None else config.MINERU_API_MODE - ) - config.MINERU_API_URL = form_data.MINERU_API_URL if form_data.MINERU_API_URL is not None else config.MINERU_API_URL - config.MINERU_API_KEY = form_data.MINERU_API_KEY if form_data.MINERU_API_KEY is not None else config.MINERU_API_KEY - config.MINERU_API_TIMEOUT = ( - form_data.MINERU_API_TIMEOUT if form_data.MINERU_API_TIMEOUT is not None else config.MINERU_API_TIMEOUT - ) - config.MINERU_PARAMS = form_data.MINERU_PARAMS if form_data.MINERU_PARAMS is not None else config.MINERU_PARAMS - config.MINERU_FILE_EXTENSIONS = ( - form_data.MINERU_FILE_EXTENSIONS - if form_data.MINERU_FILE_EXTENSIONS is not None - else config.MINERU_FILE_EXTENSIONS - ) - - # Reranking settings - if config.RAG_RERANKING_ENGINE == '': - # Unloading the internal reranker and clear VRAM memory + # Unload the previous internal reranker before applying its new settings. + if config['rag.reranking_engine'] == '': request.app.state.rf = None request.app.state.RERANKING_FUNCTION = None import gc @@ -1201,358 +1000,193 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend if torch.cuda.is_available(): torch.cuda.empty_cache() - config.RAG_RERANKING_ENGINE = ( - form_data.RAG_RERANKING_ENGINE if form_data.RAG_RERANKING_ENGINE is not None else config.RAG_RERANKING_ENGINE - ) - - config.RAG_EXTERNAL_RERANKER_URL = ( - form_data.RAG_EXTERNAL_RERANKER_URL - if form_data.RAG_EXTERNAL_RERANKER_URL is not None - else config.RAG_EXTERNAL_RERANKER_URL - ) - - config.RAG_EXTERNAL_RERANKER_API_KEY = ( - form_data.RAG_EXTERNAL_RERANKER_API_KEY - if form_data.RAG_EXTERNAL_RERANKER_API_KEY is not None - else config.RAG_EXTERNAL_RERANKER_API_KEY - ) - - config.RAG_EXTERNAL_RERANKER_TIMEOUT = ( - form_data.RAG_EXTERNAL_RERANKER_TIMEOUT - if form_data.RAG_EXTERNAL_RERANKER_TIMEOUT is not None - else config.RAG_EXTERNAL_RERANKER_TIMEOUT - ) - - config.RAG_RERANKING_BATCH_SIZE = ( - form_data.RAG_RERANKING_BATCH_SIZE - if form_data.RAG_RERANKING_BATCH_SIZE is not None - else config.RAG_RERANKING_BATCH_SIZE - ) if form_data.RAG_RERANKING_MODEL is not None: - log.info('Updating reranking model: %s to %s', config.RAG_RERANKING_MODEL, form_data.RAG_RERANKING_MODEL) + log.info('Updating reranking model: %s to %s', config['rag.reranking_model'], form_data.RAG_RERANKING_MODEL) + config.update(updates) try: - config.RAG_RERANKING_MODEL = ( - form_data.RAG_RERANKING_MODEL if form_data.RAG_RERANKING_MODEL is not None else config.RAG_RERANKING_MODEL - ) - - try: - if config.ENABLE_RAG_HYBRID_SEARCH and not config.BYPASS_EMBEDDING_AND_RETRIEVAL: - request.app.state.rf = get_rf( - config.RAG_RERANKING_ENGINE, - config.RAG_RERANKING_MODEL, - config.RAG_EXTERNAL_RERANKER_URL, - config.RAG_EXTERNAL_RERANKER_API_KEY, - config.RAG_EXTERNAL_RERANKER_TIMEOUT, - ) - - request.app.state.RERANKING_FUNCTION = get_reranking_function( - config.RAG_RERANKING_ENGINE, - config.RAG_RERANKING_MODEL, - request.app.state.rf, - reranking_batch_size=config.RAG_RERANKING_BATCH_SIZE, - ) - except Exception as e: - log.error(f'Error loading reranking model: {e}') - config.ENABLE_RAG_HYBRID_SEARCH = False + if config['rag.enable_hybrid_search'] and not config['rag.bypass_embedding_and_retrieval']: + request.app.state.rf = get_rf( + config['rag.reranking_engine'], + config['rag.reranking_model'], + config['rag.external_reranker_url'], + config['rag.external_reranker_api_key'], + config['rag.external_reranker_timeout'], + ) + request.app.state.RERANKING_FUNCTION = get_reranking_function( + config['rag.reranking_engine'], + config['rag.reranking_model'], + request.app.state.rf, + reranking_batch_size=config['rag.reranking_batch_size'], + ) except Exception as e: - log.exception(f'Problem updating reranking model: {e}') - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=ERROR_MESSAGES.DEFAULT(e, 'Error updating reranking configuration'), - ) - - # Chunking settings - config.TEXT_SPLITTER = form_data.TEXT_SPLITTER if form_data.TEXT_SPLITTER is not None else config.TEXT_SPLITTER - config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ( - form_data.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER - if form_data.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER is not None - else config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER - ) - config.CHUNK_SIZE = form_data.CHUNK_SIZE if form_data.CHUNK_SIZE is not None else config.CHUNK_SIZE - config.CHUNK_MIN_SIZE_TARGET = ( - form_data.CHUNK_MIN_SIZE_TARGET if form_data.CHUNK_MIN_SIZE_TARGET is not None else config.CHUNK_MIN_SIZE_TARGET - ) - config.CHUNK_OVERLAP = form_data.CHUNK_OVERLAP if form_data.CHUNK_OVERLAP is not None else config.CHUNK_OVERLAP - config.RAG_TOKENIZER_MODEL = ( - form_data.RAG_TOKENIZER_MODEL.strip() - if form_data.RAG_TOKENIZER_MODEL is not None - else config.RAG_TOKENIZER_MODEL - ) - - # File upload settings - # Empty string means "clear to None" (unlimited/no compression), - # None means "don't change", int means "set to this value" - if form_data.FILE_MAX_SIZE is not None: - config.FILE_MAX_SIZE = None if form_data.FILE_MAX_SIZE == '' else form_data.FILE_MAX_SIZE - if form_data.FILE_MAX_COUNT is not None: - config.FILE_MAX_COUNT = None if form_data.FILE_MAX_COUNT == '' else form_data.FILE_MAX_COUNT - if form_data.FILE_IMAGE_COMPRESSION_WIDTH is not None: - config.FILE_IMAGE_COMPRESSION_WIDTH = ( - None if form_data.FILE_IMAGE_COMPRESSION_WIDTH == '' else form_data.FILE_IMAGE_COMPRESSION_WIDTH - ) - if form_data.FILE_IMAGE_COMPRESSION_HEIGHT is not None: - config.FILE_IMAGE_COMPRESSION_HEIGHT = ( - None if form_data.FILE_IMAGE_COMPRESSION_HEIGHT == '' else form_data.FILE_IMAGE_COMPRESSION_HEIGHT - ) - - config.ALLOWED_FILE_EXTENSIONS = ( - form_data.ALLOWED_FILE_EXTENSIONS - if form_data.ALLOWED_FILE_EXTENSIONS is not None - else config.ALLOWED_FILE_EXTENSIONS - ) - - # Integration settings - config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ( - form_data.ENABLE_GOOGLE_DRIVE_INTEGRATION - if form_data.ENABLE_GOOGLE_DRIVE_INTEGRATION is not None - else config.ENABLE_GOOGLE_DRIVE_INTEGRATION - ) - config.ENABLE_ONEDRIVE_INTEGRATION = ( - form_data.ENABLE_ONEDRIVE_INTEGRATION - if form_data.ENABLE_ONEDRIVE_INTEGRATION is not None - else config.ENABLE_ONEDRIVE_INTEGRATION - ) + log.error(f'Error loading reranking model: {e}') + updates['rag.enable_hybrid_search'] = False + config['rag.enable_hybrid_search'] = False if form_data.web is not None: - # Web search settings - config.ENABLE_WEB_SEARCH = form_data.web.ENABLE_WEB_SEARCH - config.ENABLE_WEB_SEARCH_CONFIRMATION = form_data.web.ENABLE_WEB_SEARCH_CONFIRMATION - config.WEB_SEARCH_CONFIRMATION_CONTENT = form_data.web.WEB_SEARCH_CONFIRMATION_CONTENT - config.WEB_SEARCH_ENGINE = form_data.web.WEB_SEARCH_ENGINE - config.WEB_SEARCH_TRUST_ENV = form_data.web.WEB_SEARCH_TRUST_ENV - config.WEB_SEARCH_RESULT_COUNT = form_data.web.WEB_SEARCH_RESULT_COUNT - config.WEB_SEARCH_CONCURRENT_REQUESTS = form_data.web.WEB_SEARCH_CONCURRENT_REQUESTS - config.WEB_FETCH_MAX_CONTENT_LENGTH = form_data.web.WEB_FETCH_MAX_CONTENT_LENGTH - config.WEB_LOADER_CONCURRENT_REQUESTS = form_data.web.WEB_LOADER_CONCURRENT_REQUESTS - config.WEB_SEARCH_DOMAIN_FILTER_LIST = form_data.web.WEB_SEARCH_DOMAIN_FILTER_LIST - config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = form_data.web.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL - config.BYPASS_WEB_SEARCH_WEB_LOADER = form_data.web.BYPASS_WEB_SEARCH_WEB_LOADER - config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = form_data.web.OLLAMA_CLOUD_WEB_SEARCH_API_KEY - config.SEARXNG_QUERY_URL = form_data.web.SEARXNG_QUERY_URL - config.SEARXNG_LANGUAGE = form_data.web.SEARXNG_LANGUAGE - config.OPENSERP_BASE_URL = form_data.web.OPENSERP_BASE_URL - config.YACY_QUERY_URL = form_data.web.YACY_QUERY_URL - config.YACY_USERNAME = form_data.web.YACY_USERNAME - config.YACY_PASSWORD = form_data.web.YACY_PASSWORD - config.GOOGLE_PSE_API_KEY = form_data.web.GOOGLE_PSE_API_KEY - config.GOOGLE_PSE_ENGINE_ID = form_data.web.GOOGLE_PSE_ENGINE_ID - config.BRAVE_SEARCH_API_KEY = form_data.web.BRAVE_SEARCH_API_KEY - if form_data.web.BRAVE_SEARCH_CONTEXT_TOKENS is not None: - config.BRAVE_SEARCH_CONTEXT_TOKENS = form_data.web.BRAVE_SEARCH_CONTEXT_TOKENS - config.KAGI_SEARCH_API_KEY = form_data.web.KAGI_SEARCH_API_KEY - config.MOJEEK_SEARCH_API_KEY = form_data.web.MOJEEK_SEARCH_API_KEY - config.BOCHA_SEARCH_API_KEY = form_data.web.BOCHA_SEARCH_API_KEY - config.SERPSTACK_API_KEY = form_data.web.SERPSTACK_API_KEY - config.SERPSTACK_HTTPS = form_data.web.SERPSTACK_HTTPS - config.SERPER_API_KEY = form_data.web.SERPER_API_KEY - config.SERPHOUSE_API_KEY = form_data.web.SERPHOUSE_API_KEY - config.SERPHOUSE_DOMAIN = form_data.web.SERPHOUSE_DOMAIN - config.SERPLY_API_KEY = form_data.web.SERPLY_API_KEY - config.DDGS_BACKEND = form_data.web.DDGS_BACKEND - config.TAVILY_API_KEY = form_data.web.TAVILY_API_KEY - config.STAAN_API_KEY = form_data.web.STAAN_API_KEY - config.STAAN_MARKET = form_data.web.STAAN_MARKET - config.STAAN_MAX_SNIPPETS = form_data.web.STAAN_MAX_SNIPPETS - config.SEARCHAPI_API_KEY = form_data.web.SEARCHAPI_API_KEY - config.SEARCHAPI_ENGINE = form_data.web.SEARCHAPI_ENGINE - config.SERPAPI_API_KEY = form_data.web.SERPAPI_API_KEY - config.SERPAPI_ENGINE = form_data.web.SERPAPI_ENGINE - config.JINA_API_KEY = form_data.web.JINA_API_KEY - config.JINA_API_BASE_URL = form_data.web.JINA_API_BASE_URL - config.BING_SEARCH_V7_ENDPOINT = form_data.web.BING_SEARCH_V7_ENDPOINT - config.BING_SEARCH_V7_SUBSCRIPTION_KEY = form_data.web.BING_SEARCH_V7_SUBSCRIPTION_KEY - config.EXA_API_KEY = form_data.web.EXA_API_KEY - config.EXA_MAX_CONTENT_LENGTH = form_data.web.EXA_MAX_CONTENT_LENGTH - config.PERPLEXITY_API_KEY = form_data.web.PERPLEXITY_API_KEY - config.PERPLEXITY_MODEL = form_data.web.PERPLEXITY_MODEL - config.PERPLEXITY_SEARCH_CONTEXT_USAGE = form_data.web.PERPLEXITY_SEARCH_CONTEXT_USAGE - config.PERPLEXITY_SEARCH_API_URL = form_data.web.PERPLEXITY_SEARCH_API_URL - config.MICROSOFT_WEB_IQ_API_BASE_URL = form_data.web.MICROSOFT_WEB_IQ_API_BASE_URL - config.MICROSOFT_WEB_IQ_API_KEY = form_data.web.MICROSOFT_WEB_IQ_API_KEY - config.MICROSOFT_WEB_IQ_LANGUAGE = form_data.web.MICROSOFT_WEB_IQ_LANGUAGE - config.SOUGOU_API_SID = form_data.web.SOUGOU_API_SID - config.SOUGOU_API_SK = form_data.web.SOUGOU_API_SK + for field, value in form_data.web.model_dump(exclude_unset=True).items(): + if field == 'YOUTUBE_LOADER_TRANSLATION': + request.app.state.YOUTUBE_LOADER_TRANSLATION = value + elif field == 'BRAVE_SEARCH_CONTEXT_TOKENS' and value is None: + continue + else: + updates[RETRIEVAL_CONFIG_KEYS[field]] = value + config.update(updates) - # Web loader settings - config.WEB_LOADER_ENGINE = form_data.web.WEB_LOADER_ENGINE - config.WEB_LOADER_TIMEOUT = form_data.web.WEB_LOADER_TIMEOUT - - config.ENABLE_WEB_LOADER_SSL_VERIFICATION = form_data.web.ENABLE_WEB_LOADER_SSL_VERIFICATION - config.PLAYWRIGHT_WS_URL = form_data.web.PLAYWRIGHT_WS_URL - config.PLAYWRIGHT_TIMEOUT = form_data.web.PLAYWRIGHT_TIMEOUT - config.FIRECRAWL_API_KEY = form_data.web.FIRECRAWL_API_KEY - config.FIRECRAWL_API_BASE_URL = form_data.web.FIRECRAWL_API_BASE_URL - config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT - config.EXTERNAL_WEB_SEARCH_URL = form_data.web.EXTERNAL_WEB_SEARCH_URL - config.EXTERNAL_WEB_SEARCH_API_KEY = form_data.web.EXTERNAL_WEB_SEARCH_API_KEY - config.EXTERNAL_WEB_LOADER_URL = form_data.web.EXTERNAL_WEB_LOADER_URL - config.EXTERNAL_WEB_LOADER_API_KEY = form_data.web.EXTERNAL_WEB_LOADER_API_KEY - config.TAVILY_EXTRACT_DEPTH = form_data.web.TAVILY_EXTRACT_DEPTH - config.TAVILY_SEARCH_DEPTH = form_data.web.TAVILY_SEARCH_DEPTH - config.YOUTUBE_LOADER_LANGUAGE = form_data.web.YOUTUBE_LOADER_LANGUAGE - config.YOUTUBE_LOADER_PROXY_URL = form_data.web.YOUTUBE_LOADER_PROXY_URL - request.app.state.YOUTUBE_LOADER_TRANSLATION = form_data.web.YOUTUBE_LOADER_TRANSLATION - config.YANDEX_WEB_SEARCH_URL = form_data.web.YANDEX_WEB_SEARCH_URL - config.YANDEX_WEB_SEARCH_API_KEY = form_data.web.YANDEX_WEB_SEARCH_API_KEY - config.YANDEX_WEB_SEARCH_CONFIG = form_data.web.YANDEX_WEB_SEARCH_CONFIG - config.YOUCOM_API_KEY = form_data.web.YOUCOM_API_KEY - config.LINKUP_API_KEY = form_data.web.LINKUP_API_KEY - config.LINKUP_SEARCH_PARAMS = form_data.web.LINKUP_SEARCH_PARAMS - - await config.save() + await Config.upsert(updates) return { 'status': True, # RAG settings - 'RAG_TEMPLATE': config.RAG_TEMPLATE, - 'TOP_K': config.TOP_K, - 'BYPASS_EMBEDDING_AND_RETRIEVAL': config.BYPASS_EMBEDDING_AND_RETRIEVAL, - 'RAG_FULL_CONTEXT': config.RAG_FULL_CONTEXT, + 'RAG_TEMPLATE': config['rag.template'], + 'TOP_K': config['rag.top_k'], + 'BYPASS_EMBEDDING_AND_RETRIEVAL': config['rag.bypass_embedding_and_retrieval'], + 'RAG_FULL_CONTEXT': config['rag.full_context'], # Hybrid search settings - 'ENABLE_RAG_HYBRID_SEARCH': config.ENABLE_RAG_HYBRID_SEARCH, - 'TOP_K_RERANKER': config.TOP_K_RERANKER, - 'RELEVANCE_THRESHOLD': config.RELEVANCE_THRESHOLD, - 'HYBRID_BM25_WEIGHT': config.HYBRID_BM25_WEIGHT, + 'ENABLE_RAG_HYBRID_SEARCH': config['rag.enable_hybrid_search'], + 'TOP_K_RERANKER': config['rag.top_k_reranker'], + 'RELEVANCE_THRESHOLD': config['rag.relevance_threshold'], + 'HYBRID_BM25_WEIGHT': config['rag.hybrid_bm25_weight'], # Content extraction settings - 'CONTENT_EXTRACTION_ENGINE': config.CONTENT_EXTRACTION_ENGINE, - 'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config.CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES, - 'PDF_EXTRACT_IMAGES': config.PDF_EXTRACT_IMAGES, - 'PDF_LOADER_MODE': config.PDF_LOADER_MODE, - 'DATALAB_MARKER_API_KEY': config.DATALAB_MARKER_API_KEY, - 'DATALAB_MARKER_API_BASE_URL': config.DATALAB_MARKER_API_BASE_URL, - 'DATALAB_MARKER_ADDITIONAL_CONFIG': config.DATALAB_MARKER_ADDITIONAL_CONFIG, - 'DATALAB_MARKER_SKIP_CACHE': config.DATALAB_MARKER_SKIP_CACHE, - 'DATALAB_MARKER_FORCE_OCR': config.DATALAB_MARKER_FORCE_OCR, - 'DATALAB_MARKER_PAGINATE': config.DATALAB_MARKER_PAGINATE, - 'DATALAB_MARKER_STRIP_EXISTING_OCR': config.DATALAB_MARKER_STRIP_EXISTING_OCR, - 'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION, - 'DATALAB_MARKER_USE_LLM': config.DATALAB_MARKER_USE_LLM, - 'DATALAB_MARKER_OUTPUT_FORMAT': config.DATALAB_MARKER_OUTPUT_FORMAT, - 'EXTERNAL_DOCUMENT_LOADER_URL': config.EXTERNAL_DOCUMENT_LOADER_URL, - 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config.EXTERNAL_DOCUMENT_LOADER_API_KEY, - 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config.EXTERNAL_DOCUMENT_LOADER_HEADERS, - 'TIKA_SERVER_URL': config.TIKA_SERVER_URL, - 'TIKA_SERVER_VERSION': config.TIKA_SERVER_VERSION, - 'DOCLING_SERVER_URL': config.DOCLING_SERVER_URL, - 'DOCLING_API_KEY': config.DOCLING_API_KEY, - 'DOCLING_PARAMS': config.DOCLING_PARAMS, - 'DOCUMENT_INTELLIGENCE_ENDPOINT': config.DOCUMENT_INTELLIGENCE_ENDPOINT, - 'DOCUMENT_INTELLIGENCE_KEY': config.DOCUMENT_INTELLIGENCE_KEY, - 'DOCUMENT_INTELLIGENCE_MODEL': config.DOCUMENT_INTELLIGENCE_MODEL, - 'MISTRAL_OCR_API_BASE_URL': config.MISTRAL_OCR_API_BASE_URL, - 'MISTRAL_OCR_API_KEY': config.MISTRAL_OCR_API_KEY, - 'MISTRAL_OCR_USE_BASE64': config.MISTRAL_OCR_USE_BASE64, - 'PADDLEOCR_VL_BASE_URL': config.PADDLEOCR_VL_BASE_URL, - 'PADDLEOCR_VL_TOKEN': config.PADDLEOCR_VL_TOKEN, + 'CONTENT_EXTRACTION_ENGINE': config['rag.content_extraction_engine'], + 'CONTENT_EXTRACTION_SUPPORTED_MEDIA_MIME_TYPES': config['rag.content_extraction.supported_media_mime_types'], + 'PDF_EXTRACT_IMAGES': config['rag.pdf_extract_images'], + 'PDF_LOADER_MODE': config['rag.pdf_loader_mode'], + 'DATALAB_MARKER_API_KEY': config['rag.datalab_marker_api_key'], + 'DATALAB_MARKER_API_BASE_URL': config['rag.datalab_marker_api_base_url'], + 'DATALAB_MARKER_ADDITIONAL_CONFIG': config['rag.datalab_marker_additional_config'], + 'DATALAB_MARKER_SKIP_CACHE': config['rag.datalab_marker_skip_cache'], + 'DATALAB_MARKER_FORCE_OCR': config['rag.datalab_marker_force_ocr'], + 'DATALAB_MARKER_PAGINATE': config['rag.datalab_marker_paginate'], + 'DATALAB_MARKER_STRIP_EXISTING_OCR': config['rag.datalab_marker_strip_existing_ocr'], + 'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': config['rag.datalab_marker_disable_image_extraction'], + 'DATALAB_MARKER_USE_LLM': config['rag.datalab_marker_use_llm'], + 'DATALAB_MARKER_OUTPUT_FORMAT': config['rag.datalab_marker_output_format'], + 'EXTERNAL_DOCUMENT_LOADER_URL': config['rag.external_document_loader_url'], + 'EXTERNAL_DOCUMENT_LOADER_API_KEY': config['rag.external_document_loader_api_key'], + 'EXTERNAL_DOCUMENT_LOADER_HEADERS': config['rag.external_document_loader_headers'], + 'TIKA_SERVER_URL': config['rag.tika_server_url'], + 'TIKA_SERVER_VERSION': config['rag.tika_server_version'], + 'DOCLING_SERVER_URL': config['rag.docling_server_url'], + 'DOCLING_API_KEY': config['rag.docling_api_key'], + 'DOCLING_PARAMS': config['rag.docling_params'], + 'DOCUMENT_INTELLIGENCE_ENDPOINT': config['rag.document_intelligence_endpoint'], + 'DOCUMENT_INTELLIGENCE_KEY': config['rag.document_intelligence_key'], + 'DOCUMENT_INTELLIGENCE_MODEL': config['rag.document_intelligence_model'], + 'MISTRAL_OCR_API_BASE_URL': config['rag.mistral_ocr_api_base_url'], + 'MISTRAL_OCR_API_KEY': config['rag.mistral_ocr_api_key'], + 'MISTRAL_OCR_USE_BASE64': config['rag.mistral_ocr_use_base64'], + 'PADDLEOCR_VL_BASE_URL': config['rag.paddleocr_vl_base_url'], + 'PADDLEOCR_VL_TOKEN': config['rag.paddleocr_vl_token'], # MinerU settings - 'MINERU_API_MODE': config.MINERU_API_MODE, - 'MINERU_API_URL': config.MINERU_API_URL, - 'MINERU_API_KEY': config.MINERU_API_KEY, - 'MINERU_API_TIMEOUT': config.MINERU_API_TIMEOUT, - 'MINERU_PARAMS': config.MINERU_PARAMS, + 'MINERU_API_MODE': config['rag.mineru_api_mode'], + 'MINERU_API_URL': config['rag.mineru_api_url'], + 'MINERU_API_KEY': config['rag.mineru_api_key'], + 'MINERU_API_TIMEOUT': config['rag.mineru_api_timeout'], + 'MINERU_PARAMS': config['rag.mineru_params'], # Reranking settings - 'RAG_RERANKING_MODEL': config.RAG_RERANKING_MODEL, - 'RAG_RERANKING_ENGINE': config.RAG_RERANKING_ENGINE, - 'RAG_EXTERNAL_RERANKER_URL': config.RAG_EXTERNAL_RERANKER_URL, - 'RAG_EXTERNAL_RERANKER_API_KEY': config.RAG_EXTERNAL_RERANKER_API_KEY, - 'RAG_EXTERNAL_RERANKER_TIMEOUT': config.RAG_EXTERNAL_RERANKER_TIMEOUT, + 'RAG_RERANKING_MODEL': config['rag.reranking_model'], + 'RAG_RERANKING_ENGINE': config['rag.reranking_engine'], + 'RAG_EXTERNAL_RERANKER_URL': config['rag.external_reranker_url'], + 'RAG_EXTERNAL_RERANKER_API_KEY': config['rag.external_reranker_api_key'], + 'RAG_EXTERNAL_RERANKER_TIMEOUT': config['rag.external_reranker_timeout'], # Chunking settings - 'TEXT_SPLITTER': config.TEXT_SPLITTER, - 'RAG_TOKENIZER_MODEL': config.RAG_TOKENIZER_MODEL, - 'CHUNK_SIZE': config.CHUNK_SIZE, - 'CHUNK_MIN_SIZE_TARGET': config.CHUNK_MIN_SIZE_TARGET, - 'ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER': config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER, - 'CHUNK_OVERLAP': config.CHUNK_OVERLAP, + 'TEXT_SPLITTER': config['rag.text_splitter'], + 'RAG_TOKENIZER_MODEL': config['rag.tokenizer_model'], + 'CHUNK_SIZE': config['rag.chunk_size'], + 'CHUNK_MIN_SIZE_TARGET': config['rag.chunk_min_size_target'], + 'ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER': config['rag.enable_markdown_header_text_splitter'], + 'CHUNK_OVERLAP': config['rag.chunk_overlap'], # File upload settings - 'FILE_MAX_SIZE': config.FILE_MAX_SIZE, - 'FILE_MAX_COUNT': config.FILE_MAX_COUNT, - 'FILE_IMAGE_COMPRESSION_WIDTH': config.FILE_IMAGE_COMPRESSION_WIDTH, - 'FILE_IMAGE_COMPRESSION_HEIGHT': config.FILE_IMAGE_COMPRESSION_HEIGHT, - 'ALLOWED_FILE_EXTENSIONS': config.ALLOWED_FILE_EXTENSIONS, + 'FILE_MAX_SIZE': config['rag.file.max_size'], + 'FILE_MAX_COUNT': config['rag.file.max_count'], + 'FILE_IMAGE_COMPRESSION_WIDTH': config['file.image_compression_width'], + 'FILE_IMAGE_COMPRESSION_HEIGHT': config['file.image_compression_height'], + 'ALLOWED_FILE_EXTENSIONS': config['rag.file.allowed_extensions'], # Integration settings - 'ENABLE_GOOGLE_DRIVE_INTEGRATION': config.ENABLE_GOOGLE_DRIVE_INTEGRATION, - 'ENABLE_ONEDRIVE_INTEGRATION': config.ENABLE_ONEDRIVE_INTEGRATION, + 'ENABLE_GOOGLE_DRIVE_INTEGRATION': config['google_drive.enable'], + 'ENABLE_ONEDRIVE_INTEGRATION': config['onedrive.enable'], # Web search settings 'web': { - 'ENABLE_WEB_SEARCH': config.ENABLE_WEB_SEARCH, - 'ENABLE_WEB_SEARCH_CONFIRMATION': config.ENABLE_WEB_SEARCH_CONFIRMATION, - 'WEB_SEARCH_CONFIRMATION_CONTENT': config.WEB_SEARCH_CONFIRMATION_CONTENT, - 'WEB_SEARCH_ENGINE': config.WEB_SEARCH_ENGINE, - 'WEB_SEARCH_TRUST_ENV': config.WEB_SEARCH_TRUST_ENV, - 'WEB_SEARCH_RESULT_COUNT': config.WEB_SEARCH_RESULT_COUNT, - 'WEB_SEARCH_CONCURRENT_REQUESTS': config.WEB_SEARCH_CONCURRENT_REQUESTS, - 'WEB_FETCH_MAX_CONTENT_LENGTH': config.WEB_FETCH_MAX_CONTENT_LENGTH, - 'WEB_LOADER_CONCURRENT_REQUESTS': config.WEB_LOADER_CONCURRENT_REQUESTS, - 'WEB_SEARCH_DOMAIN_FILTER_LIST': config.WEB_SEARCH_DOMAIN_FILTER_LIST, - 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL, - 'BYPASS_WEB_SEARCH_WEB_LOADER': config.BYPASS_WEB_SEARCH_WEB_LOADER, - 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY, - 'SEARXNG_QUERY_URL': config.SEARXNG_QUERY_URL, - 'SEARXNG_LANGUAGE': config.SEARXNG_LANGUAGE, - 'OPENSERP_BASE_URL': config.OPENSERP_BASE_URL, - 'YACY_QUERY_URL': config.YACY_QUERY_URL, - 'YACY_USERNAME': config.YACY_USERNAME, - 'YACY_PASSWORD': config.YACY_PASSWORD, - 'GOOGLE_PSE_API_KEY': config.GOOGLE_PSE_API_KEY, - 'GOOGLE_PSE_ENGINE_ID': config.GOOGLE_PSE_ENGINE_ID, - 'BRAVE_SEARCH_API_KEY': config.BRAVE_SEARCH_API_KEY, - 'BRAVE_SEARCH_CONTEXT_TOKENS': config.BRAVE_SEARCH_CONTEXT_TOKENS, - 'KAGI_SEARCH_API_KEY': config.KAGI_SEARCH_API_KEY, - 'MOJEEK_SEARCH_API_KEY': config.MOJEEK_SEARCH_API_KEY, - 'BOCHA_SEARCH_API_KEY': config.BOCHA_SEARCH_API_KEY, - 'SERPSTACK_API_KEY': config.SERPSTACK_API_KEY, - 'SERPSTACK_HTTPS': config.SERPSTACK_HTTPS, - 'SERPER_API_KEY': config.SERPER_API_KEY, - 'SERPHOUSE_API_KEY': config.SERPHOUSE_API_KEY, - 'SERPHOUSE_DOMAIN': config.SERPHOUSE_DOMAIN, - 'SERPLY_API_KEY': config.SERPLY_API_KEY, - 'TAVILY_API_KEY': config.TAVILY_API_KEY, - 'STAAN_API_KEY': config.STAAN_API_KEY, - 'STAAN_MARKET': config.STAAN_MARKET, - 'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS, - 'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY, - 'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE, - 'SERPAPI_API_KEY': config.SERPAPI_API_KEY, - 'SERPAPI_ENGINE': config.SERPAPI_ENGINE, - 'JINA_API_KEY': config.JINA_API_KEY, - 'JINA_API_BASE_URL': config.JINA_API_BASE_URL, - 'BING_SEARCH_V7_ENDPOINT': config.BING_SEARCH_V7_ENDPOINT, - 'BING_SEARCH_V7_SUBSCRIPTION_KEY': config.BING_SEARCH_V7_SUBSCRIPTION_KEY, - 'EXA_API_KEY': config.EXA_API_KEY, - 'EXA_MAX_CONTENT_LENGTH': config.EXA_MAX_CONTENT_LENGTH, - 'PERPLEXITY_API_KEY': config.PERPLEXITY_API_KEY, - 'PERPLEXITY_MODEL': config.PERPLEXITY_MODEL, - 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config.PERPLEXITY_SEARCH_CONTEXT_USAGE, - 'PERPLEXITY_SEARCH_API_URL': config.PERPLEXITY_SEARCH_API_URL, - 'MICROSOFT_WEB_IQ_API_BASE_URL': config.MICROSOFT_WEB_IQ_API_BASE_URL, - 'MICROSOFT_WEB_IQ_API_KEY': config.MICROSOFT_WEB_IQ_API_KEY, - 'MICROSOFT_WEB_IQ_LANGUAGE': config.MICROSOFT_WEB_IQ_LANGUAGE, - 'SOUGOU_API_SID': config.SOUGOU_API_SID, - 'SOUGOU_API_SK': config.SOUGOU_API_SK, - 'WEB_LOADER_ENGINE': config.WEB_LOADER_ENGINE, - 'WEB_LOADER_TIMEOUT': config.WEB_LOADER_TIMEOUT, - 'ENABLE_WEB_LOADER_SSL_VERIFICATION': config.ENABLE_WEB_LOADER_SSL_VERIFICATION, - 'PLAYWRIGHT_WS_URL': config.PLAYWRIGHT_WS_URL, - 'PLAYWRIGHT_TIMEOUT': config.PLAYWRIGHT_TIMEOUT, - 'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY, - 'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL, - 'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT, - 'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH, - 'TAVILY_SEARCH_DEPTH': config.TAVILY_SEARCH_DEPTH, - 'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL, - 'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY, - 'EXTERNAL_WEB_LOADER_URL': config.EXTERNAL_WEB_LOADER_URL, - 'EXTERNAL_WEB_LOADER_API_KEY': config.EXTERNAL_WEB_LOADER_API_KEY, - 'YOUTUBE_LOADER_LANGUAGE': config.YOUTUBE_LOADER_LANGUAGE, - 'YOUTUBE_LOADER_PROXY_URL': config.YOUTUBE_LOADER_PROXY_URL, + 'ENABLE_WEB_SEARCH': config['web.search.enable'], + 'ENABLE_WEB_SEARCH_CONFIRMATION': config['web.search.confirmation.enable'], + 'WEB_SEARCH_CONFIRMATION_CONTENT': config['web.search.confirmation.content'], + 'WEB_SEARCH_ENGINE': config['web.search.engine'], + 'WEB_SEARCH_TRUST_ENV': config['web.search.trust_env'], + 'WEB_SEARCH_RESULT_COUNT': config['web.search.result_count'], + 'WEB_SEARCH_CONCURRENT_REQUESTS': config['web.search.concurrent_requests'], + 'WEB_FETCH_MAX_CONTENT_LENGTH': config['web.fetch.max_content_length'], + 'WEB_LOADER_CONCURRENT_REQUESTS': config['web.loader.concurrent_requests'], + 'WEB_SEARCH_DOMAIN_FILTER_LIST': config['web.search.domain.filter_list'], + 'BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL': config['web.search.bypass_embedding_and_retrieval'], + 'BYPASS_WEB_SEARCH_WEB_LOADER': config['web.search.bypass_web_loader'], + 'OLLAMA_CLOUD_WEB_SEARCH_API_KEY': config['web.search.ollama_cloud_api_key'], + 'SEARXNG_QUERY_URL': config['web.search.searxng_query_url'], + 'SEARXNG_LANGUAGE': config['web.search.searxng_language'], + 'OPENSERP_BASE_URL': config['web.search.openserp_base_url'], + 'YACY_QUERY_URL': config['web.search.yacy_query_url'], + 'YACY_USERNAME': config['web.search.yacy_username'], + 'YACY_PASSWORD': config['web.search.yacy_password'], + 'GOOGLE_PSE_API_KEY': config['web.search.google_pse_api_key'], + 'GOOGLE_PSE_ENGINE_ID': config['web.search.google_pse_engine_id'], + 'BRAVE_SEARCH_API_KEY': config['web.search.brave_search_api_key'], + 'BRAVE_SEARCH_CONTEXT_TOKENS': config['web.search.brave_search_context_tokens'], + 'KAGI_SEARCH_API_KEY': config['web.search.kagi_search_api_key'], + 'MOJEEK_SEARCH_API_KEY': config['web.search.mojeek_search_api_key'], + 'BOCHA_SEARCH_API_KEY': config['web.search.bocha_search_api_key'], + 'SERPSTACK_API_KEY': config['web.search.serpstack_api_key'], + 'SERPSTACK_HTTPS': config['web.search.serpstack_https'], + 'SERPER_API_KEY': config['web.search.serper_api_key'], + 'SERPHOUSE_API_KEY': config['web.search.serphouse_api_key'], + 'SERPHOUSE_DOMAIN': config['web.search.serphouse_domain'], + 'SERPLY_API_KEY': config['web.search.serply_api_key'], + 'TAVILY_API_KEY': config['web.search.tavily_api_key'], + 'STAAN_API_KEY': config['web.search.staan_api_key'], + 'STAAN_MARKET': config['web.search.staan_market'], + 'STAAN_MAX_SNIPPETS': config['web.search.staan_max_snippets'], + 'SEARCHAPI_API_KEY': config['web.search.searchapi_api_key'], + 'SEARCHAPI_ENGINE': config['web.search.searchapi_engine'], + 'SERPAPI_API_KEY': config['web.search.serpapi_api_key'], + 'SERPAPI_ENGINE': config['web.search.serpapi_engine'], + 'JINA_API_KEY': config['web.search.jina_api_key'], + 'JINA_API_BASE_URL': config['web.search.jina_api_base_url'], + 'BING_SEARCH_V7_ENDPOINT': config['web.search.bing_search_v7_endpoint'], + 'BING_SEARCH_V7_SUBSCRIPTION_KEY': config['web.search.bing_search_v7_subscription_key'], + 'EXA_API_KEY': config['web.search.exa_api_key'], + 'EXA_MAX_CONTENT_LENGTH': config['web.search.exa_max_content_length'], + 'PERPLEXITY_API_KEY': config['web.search.perplexity_api_key'], + 'PERPLEXITY_MODEL': config['web.search.perplexity_model'], + 'PERPLEXITY_SEARCH_CONTEXT_USAGE': config['web.search.perplexity_search_context_usage'], + 'PERPLEXITY_SEARCH_API_URL': config['web.search.perplexity_search_api_url'], + 'MICROSOFT_WEB_IQ_API_BASE_URL': config['web.search.microsoft_web_iq_api_base_url'], + 'MICROSOFT_WEB_IQ_API_KEY': config['web.search.microsoft_web_iq_api_key'], + 'MICROSOFT_WEB_IQ_LANGUAGE': config['web.search.microsoft_web_iq_language'], + 'SOUGOU_API_SID': config['web.search.sougou_api_sid'], + 'SOUGOU_API_SK': config['web.search.sougou_api_sk'], + 'WEB_LOADER_ENGINE': config['web.loader.engine'], + 'WEB_LOADER_TIMEOUT': config['web.loader.timeout'], + 'ENABLE_WEB_LOADER_SSL_VERIFICATION': config['web.loader.ssl_verification'], + 'PLAYWRIGHT_WS_URL': config['web.loader.playwright_ws_url'], + 'PLAYWRIGHT_TIMEOUT': config['web.loader.playwright_timeout'], + 'FIRECRAWL_API_KEY': config['web.loader.firecrawl_api_key'], + 'FIRECRAWL_API_BASE_URL': config['web.loader.firecrawl_api_url'], + 'FIRECRAWL_TIMEOUT': config['web.loader.firecrawl_timeout'], + 'TAVILY_EXTRACT_DEPTH': config['web.search.tavily_extract_depth'], + 'TAVILY_SEARCH_DEPTH': config['web.search.tavily_search_depth'], + 'EXTERNAL_WEB_SEARCH_URL': config['web.search.external_web_search_url'], + 'EXTERNAL_WEB_SEARCH_API_KEY': config['web.search.external_web_search_api_key'], + 'EXTERNAL_WEB_LOADER_URL': config['web.loader.external_web_loader_url'], + 'EXTERNAL_WEB_LOADER_API_KEY': config['web.loader.external_web_loader_api_key'], + 'YOUTUBE_LOADER_LANGUAGE': config['rag.youtube_loader_language'], + 'YOUTUBE_LOADER_PROXY_URL': config['rag.youtube_loader_proxy_url'], 'YOUTUBE_LOADER_TRANSLATION': request.app.state.YOUTUBE_LOADER_TRANSLATION, - 'YANDEX_WEB_SEARCH_URL': config.YANDEX_WEB_SEARCH_URL, - 'YANDEX_WEB_SEARCH_API_KEY': config.YANDEX_WEB_SEARCH_API_KEY, - 'YANDEX_WEB_SEARCH_CONFIG': config.YANDEX_WEB_SEARCH_CONFIG, - 'YOUCOM_API_KEY': config.YOUCOM_API_KEY, - 'LINKUP_API_KEY': config.LINKUP_API_KEY, - 'LINKUP_SEARCH_PARAMS': config.LINKUP_SEARCH_PARAMS, + 'YANDEX_WEB_SEARCH_URL': config['web.search.yandex_web_search_url'], + 'YANDEX_WEB_SEARCH_API_KEY': config['web.search.yandex_web_search_api_key'], + 'YANDEX_WEB_SEARCH_CONFIG': config['web.search.yandex_web_search_config'], + 'YOUCOM_API_KEY': config['web.search.youcom_api_key'], + 'LINKUP_API_KEY': config['web.search.linkup_api_key'], + 'LINKUP_SEARCH_PARAMS': config['web.search.linkup_search_params'], }, } @@ -1580,7 +1214,7 @@ def can_merge_chunks(a: Document, b: Document) -> bool: def merge_docs_to_target_size( request: Request, chunks: list[Document], - config: RetrievalConfig, + config: dict, ) -> list[Document]: """ Best-effort normalization of chunk sizes. @@ -1593,8 +1227,8 @@ def merge_docs_to_target_size( backward merging (append into the previous emitted chunk) for undersized chunks that can't grow forward. """ - min_size = config.CHUNK_MIN_SIZE_TARGET - max_size = config.CHUNK_SIZE + min_size = config['rag.chunk_min_size_target'] + max_size = config['rag.chunk_size'] if min_size <= 0: return chunks @@ -1652,13 +1286,13 @@ def merge_docs_to_target_size( return result -def get_transformers_tokenizer(request: Request, config: RetrievalConfig): +def get_transformers_tokenizer(request: Request, config: dict): if USE_SLIM: raise HTTPException(503, 'Transformers tokenization is unavailable in slim. Use character or token splitting.') - if config.RAG_TOKENIZER_MODEL: + if config['rag.tokenizer_model']: from transformers import AutoTokenizer - tokenizer_model = config.RAG_TOKENIZER_MODEL + tokenizer_model = config['rag.tokenizer_model'] if not os.path.exists(tokenizer_model) and '/' not in tokenizer_model: tokenizer_model = f'sentence-transformers/{tokenizer_model}' @@ -1687,13 +1321,13 @@ def get_transformers_tokenizer(request: Request, config: RetrievalConfig): def get_splitter_length_function( request: Request, - config: RetrievalConfig, + config: dict, ) -> Callable[[str], int]: - if config.TEXT_SPLITTER == 'token': - encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME)) + if config['rag.text_splitter'] == 'token': + encoding = tiktoken.get_encoding(str(config['rag.tiktoken_encoding_name'])) return lambda text: len(encoding.encode(text, disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL)) - if config.TEXT_SPLITTER == 'token_transformers': + if config['rag.text_splitter'] == 'token_transformers': tokenizer = get_transformers_tokenizer(request, config) return lambda text: len(tokenizer.encode(text)) @@ -1733,7 +1367,7 @@ def save_docs_to_vector_db( request: Request, docs, collection_name, - config: RetrievalConfig, + config: dict, metadata: dict | None = None, overwrite: bool = False, split: bool = True, @@ -1765,7 +1399,7 @@ def save_docs_to_vector_db( raise ValueError(ERROR_MESSAGES.DUPLICATE_CONTENT) if split: - if config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER: + if config['rag.enable_markdown_header_text_splitter']: log.info('Using markdown header text splitter') # Define headers to split on - covering most common markdown header levels markdown_splitter = MarkdownHeaderTextSplitter( @@ -1793,34 +1427,34 @@ def save_docs_to_vector_db( ) docs = split_docs - if config.CHUNK_MIN_SIZE_TARGET > 0: + if config['rag.chunk_min_size_target'] > 0: docs = merge_docs_to_target_size(request, docs, config) - if config.TEXT_SPLITTER in ['', 'character']: + if config['rag.text_splitter'] in ['', 'character']: text_splitter = RecursiveCharacterTextSplitter( - chunk_size=config.CHUNK_SIZE, - chunk_overlap=config.CHUNK_OVERLAP, + chunk_size=config['rag.chunk_size'], + chunk_overlap=config['rag.chunk_overlap'], add_start_index=True, ) docs = text_splitter.split_documents(docs) - elif config.TEXT_SPLITTER == 'token': - log.info('Using token text splitter: %s', config.TIKTOKEN_ENCODING_NAME) + elif config['rag.text_splitter'] == 'token': + log.info('Using token text splitter: %s', config['rag.tiktoken_encoding_name']) - tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME)) + tiktoken.get_encoding(str(config['rag.tiktoken_encoding_name'])) text_splitter = TokenTextSplitter( - encoding_name=str(config.TIKTOKEN_ENCODING_NAME), - chunk_size=config.CHUNK_SIZE, - chunk_overlap=config.CHUNK_OVERLAP, + encoding_name=str(config['rag.tiktoken_encoding_name']), + chunk_size=config['rag.chunk_size'], + chunk_overlap=config['rag.chunk_overlap'], add_start_index=True, disallowed_special=TIKTOKEN_DISALLOWED_SPECIAL, ) docs = text_splitter.split_documents(docs) - elif config.TEXT_SPLITTER == 'token_transformers': + elif config['rag.text_splitter'] == 'token_transformers': log.info('Using transformers token text splitter') text_splitter = RecursiveCharacterTextSplitter( - chunk_size=config.CHUNK_SIZE, - chunk_overlap=config.CHUNK_OVERLAP, + chunk_size=config['rag.chunk_size'], + chunk_overlap=config['rag.chunk_overlap'], length_function=get_splitter_length_function(request, config), add_start_index=True, ) @@ -1837,8 +1471,8 @@ def save_docs_to_vector_db( **doc.metadata, **(metadata if metadata else {}), 'embedding_config': { - 'engine': config.RAG_EMBEDDING_ENGINE, - 'model': config.RAG_EMBEDDING_MODEL, + 'engine': config['rag.embedding_engine'], + 'model': config['rag.embedding_model'], }, } for doc in docs @@ -1857,33 +1491,33 @@ def save_docs_to_vector_db( log.info('generating embeddings for %s', collection_name) embedding_function = get_embedding_function( - config.RAG_EMBEDDING_ENGINE, - config.RAG_EMBEDDING_MODEL, + config['rag.embedding_engine'], + config['rag.embedding_model'], request.app.state.ef, ( - config.RAG_OPENAI_API_BASE_URL - if config.RAG_EMBEDDING_ENGINE == 'openai' + config['rag.openai.api_base_url'] + if config['rag.embedding_engine'] == 'openai' else ( - config.RAG_OLLAMA_BASE_URL - if config.RAG_EMBEDDING_ENGINE == 'ollama' - else config.RAG_AZURE_OPENAI_BASE_URL + config['rag.ollama.base_url'] + if config['rag.embedding_engine'] == 'ollama' + else config['rag.azure_openai.base_url'] ) ), ( - config.RAG_OPENAI_API_KEY - if config.RAG_EMBEDDING_ENGINE == 'openai' + config['rag.openai.api_key'] + if config['rag.embedding_engine'] == 'openai' else ( - config.RAG_OLLAMA_API_KEY - if config.RAG_EMBEDDING_ENGINE == 'ollama' - else config.RAG_AZURE_OPENAI_API_KEY + config['rag.ollama.api_key'] + if config['rag.embedding_engine'] == 'ollama' + else config['rag.azure_openai.api_key'] ) ), - config.RAG_EMBEDDING_BATCH_SIZE, + config['rag.embedding_batch_size'], azure_api_version=( - config.RAG_AZURE_OPENAI_API_VERSION if config.RAG_EMBEDDING_ENGINE == 'azure_openai' else None + config['rag.azure_openai.api_version'] if config['rag.embedding_engine'] == 'azure_openai' else None ), - enable_async=config.ENABLE_ASYNC_EMBEDDING, - concurrent_requests=config.RAG_EMBEDDING_CONCURRENT_REQUESTS, + enable_async=config['rag.enable_async_embedding'], + concurrent_requests=config['rag.embedding_concurrent_requests'], ) # Run async embedding in sync context using the main event loop @@ -1946,7 +1580,7 @@ async def process_file( Note: granular session management is used to prevent connection pool exhaustion. The session is committed before external API calls, and updates use a fresh session. """ - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) file = await Files.get_file_by_id(form_data.file_id, db=db) if file and file.user_id != user.id and user.role != 'admin': if not await has_access_to_file(file.id, 'write', user, db=db): @@ -2033,8 +1667,7 @@ async def process_file( file_path = file.path if file_path: file_path = await asyncio.to_thread(Storage.get_file, file_path) - loader_config = await get_loader_config() - loader = build_loader_from_config(request, loader_config) + loader = build_loader_from_config(config) loader.user = user loader.metadata = { 'file_id': file.id, @@ -2080,7 +1713,7 @@ async def process_file( ) hash = calculate_sha256_string(text_content) - if config.BYPASS_EMBEDDING_AND_RETRIEVAL: + if config['rag.bypass_embedding_and_retrieval']: await Files.update_file_data_by_id(file.id, {'status': 'completed', 'error': None}, db=db) await Files.update_file_hash_by_id(file.id, hash, db=db) await publish_event( @@ -2231,7 +1864,7 @@ async def process_text( text_content = form_data.content log.debug('text_content: %s', text_content) - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) result = await run_in_threadpool(save_docs_to_vector_db, request, docs, collection_name, config, user=user) if result: await publish_event( @@ -2386,9 +2019,9 @@ async def process_url( 'content': result.get('content'), } - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) try: - url_result = await _fetch_url(form_data.url, config.FILE_MAX_SIZE) + url_result = await _fetch_url(form_data.url, config['rag.file.max_size']) except HTTPException: raise except Exception as e: @@ -2458,10 +2091,10 @@ async def process_web( overwrite: bool = Query(True, description='Whether to overwrite existing collection'), user=Depends(get_verified_user), ): - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) try: - content, docs = await get_content_from_url(request, form_data.url) + content, docs = await get_content_from_url(request, form_data.url, config=config) except HTTPException: raise except YoutubeTranscriptError as e: @@ -2494,7 +2127,7 @@ async def process_web( else: await _validate_collection_access([collection_name], user, access_type='write') - if not config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: + if not config['web.search.bypass_embedding_and_retrieval']: await run_in_threadpool( save_docs_to_vector_db, request, @@ -2538,7 +2171,9 @@ async def process_web( ) -async def search_web(request: Request, engine: str, query: str, user=None) -> list[SearchResult]: +async def search_web( + request: Request, engine: str, query: str, user=None, *, config: dict | None = None +) -> list[SearchResult]: """Dispatch a web search query to the configured engine and return results. Providers that have been migrated to async (aiohttp) are awaited natively. @@ -2547,171 +2182,172 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li """ # TODO: add playwright to search the web - config = await get_retrieval_config() + if config is None: + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) if engine == 'ollama_cloud': return await asyncio.to_thread( search_ollama_cloud, 'https://ollama.com', - config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY, + config['web.search.ollama_cloud_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) elif engine == 'perplexity_search': - if config.PERPLEXITY_API_KEY: + if config['web.search.perplexity_api_key']: return await asyncio.to_thread( search_perplexity_search, - config.PERPLEXITY_API_KEY, + config['web.search.perplexity_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - config.PERPLEXITY_SEARCH_API_URL, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + config['web.search.perplexity_search_api_url'], user, ) else: raise Exception('No PERPLEXITY_API_KEY found in environment variables') elif engine == 'searxng': - if config.SEARXNG_QUERY_URL: - searxng_kwargs = {'language': config.SEARXNG_LANGUAGE} + if config['web.search.searxng_query_url']: + searxng_kwargs = {'language': config['web.search.searxng_language']} return await search_searxng( - config.SEARXNG_QUERY_URL, + config['web.search.searxng_query_url'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], **searxng_kwargs, ) else: raise Exception('No SEARXNG_QUERY_URL found in environment variables') elif engine == 'openserp': - if config.OPENSERP_BASE_URL: + if config['web.search.openserp_base_url']: return await search_openserp( - config.OPENSERP_BASE_URL, + config['web.search.openserp_base_url'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No OPENSERP_BASE_URL found in environment variables') elif engine == 'yacy': - if config.YACY_QUERY_URL: + if config['web.search.yacy_query_url']: return await asyncio.to_thread( search_yacy, - config.YACY_QUERY_URL, - config.YACY_USERNAME, - config.YACY_PASSWORD, + config['web.search.yacy_query_url'], + config['web.search.yacy_username'], + config['web.search.yacy_password'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No YACY_QUERY_URL found in environment variables') elif engine == 'google_pse': - if config.GOOGLE_PSE_API_KEY and config.GOOGLE_PSE_ENGINE_ID: + if config['web.search.google_pse_api_key'] and config['web.search.google_pse_engine_id']: return await search_google_pse( - config.GOOGLE_PSE_API_KEY, - config.GOOGLE_PSE_ENGINE_ID, + config['web.search.google_pse_api_key'], + config['web.search.google_pse_engine_id'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - referer=config.WEBUI_URL, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + referer=config['webui.url'], ) else: raise Exception('No GOOGLE_PSE_API_KEY or GOOGLE_PSE_ENGINE_ID found in environment variables') elif engine == 'brave': - if config.BRAVE_SEARCH_API_KEY: + if config['web.search.brave_search_api_key']: return await search_brave( - config.BRAVE_SEARCH_API_KEY, + config['web.search.brave_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No BRAVE_SEARCH_API_KEY found in environment variables') elif engine == 'brave_llm_context': - if config.BRAVE_SEARCH_API_KEY: + if config['web.search.brave_search_api_key']: return await asyncio.to_thread( search_brave_llm_context, - config.BRAVE_SEARCH_API_KEY, + config['web.search.brave_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - config.BRAVE_SEARCH_CONTEXT_TOKENS, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + config['web.search.brave_search_context_tokens'], ) else: raise Exception('No BRAVE_SEARCH_API_KEY found in environment variables') elif engine == 'kagi': - if config.KAGI_SEARCH_API_KEY: + if config['web.search.kagi_search_api_key']: return await asyncio.to_thread( search_kagi, - config.KAGI_SEARCH_API_KEY, + config['web.search.kagi_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No KAGI_SEARCH_API_KEY found in environment variables') elif engine == 'mojeek': - if config.MOJEEK_SEARCH_API_KEY: + if config['web.search.mojeek_search_api_key']: return await asyncio.to_thread( search_mojeek, - config.MOJEEK_SEARCH_API_KEY, + config['web.search.mojeek_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No MOJEEK_SEARCH_API_KEY found in environment variables') elif engine == 'bocha': - if config.BOCHA_SEARCH_API_KEY: + if config['web.search.bocha_search_api_key']: return await asyncio.to_thread( search_bocha, - config.BOCHA_SEARCH_API_KEY, + config['web.search.bocha_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No BOCHA_SEARCH_API_KEY found in environment variables') elif engine == 'serpstack': - if config.SERPSTACK_API_KEY: + if config['web.search.serpstack_api_key']: return await search_serpstack( - config.SERPSTACK_API_KEY, + config['web.search.serpstack_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - https_enabled=config.SERPSTACK_HTTPS, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + https_enabled=config['web.search.serpstack_https'], ) else: raise Exception('No SERPSTACK_API_KEY found in environment variables') elif engine == 'serper': - if config.SERPER_API_KEY: + if config['web.search.serper_api_key']: return await search_serper( - config.SERPER_API_KEY, + config['web.search.serper_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No SERPER_API_KEY found in environment variables') elif engine == 'serphouse': - if config.SERPHOUSE_API_KEY: + if config['web.search.serphouse_api_key']: return await search_serphouse( - config.SERPHOUSE_API_KEY, - config.SERPHOUSE_DOMAIN, + config['web.search.serphouse_api_key'], + config['web.search.serphouse_domain'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No SERPHOUSE_API_KEY found in environment variables') elif engine == 'serply': - if config.SERPLY_API_KEY: + if config['web.search.serply_api_key']: return await asyncio.to_thread( search_serply, - config.SERPLY_API_KEY, + config['web.search.serply_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - filter_list=config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + filter_list=config['web.search.domain.filter_list'], ) else: raise Exception('No SERPLY_API_KEY found in environment variables') @@ -2719,100 +2355,104 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li return await asyncio.to_thread( search_duckduckgo, query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - concurrent_requests=config.WEB_SEARCH_CONCURRENT_REQUESTS, - backend=config.DDGS_BACKEND, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + concurrent_requests=config['web.search.concurrent_requests'], + backend=config['web.search.ddgs_backend'], ) elif engine == 'tavily': - if config.TAVILY_API_KEY: + if config['web.search.tavily_api_key']: return await asyncio.to_thread( search_tavily, - config.TAVILY_API_KEY, + config['web.search.tavily_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - search_depth=config.TAVILY_SEARCH_DEPTH, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + search_depth=config['web.search.tavily_search_depth'], ) else: raise Exception('No TAVILY_API_KEY found in environment variables') elif engine == 'staan': - if config.STAAN_API_KEY: + if config['web.search.staan_api_key']: return await asyncio.to_thread( search_staan, - config.STAAN_API_KEY, + config['web.search.staan_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - market=config.STAAN_MARKET, - max_snippets=config.STAAN_MAX_SNIPPETS, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + market=config['web.search.staan_market'], + max_snippets=config['web.search.staan_max_snippets'], ) else: raise Exception('No STAAN_API_KEY found in environment variables') elif engine == 'exa': - if config.EXA_API_KEY: + if config['web.search.exa_api_key']: return await asyncio.to_thread( search_exa, - config.EXA_API_KEY, + config['web.search.exa_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - max_content_length=config.EXA_MAX_CONTENT_LENGTH, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + max_content_length=config['web.search.exa_max_content_length'], ) else: raise Exception('No EXA_API_KEY found in environment variables') elif engine == 'searchapi': - if config.SEARCHAPI_API_KEY: + if config['web.search.searchapi_api_key']: return await asyncio.to_thread( search_searchapi, - config.SEARCHAPI_API_KEY, - config.SEARCHAPI_ENGINE, + config['web.search.searchapi_api_key'], + config['web.search.searchapi_engine'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No SEARCHAPI_API_KEY found in environment variables') elif engine == 'serpapi': - if config.SERPAPI_API_KEY: + if config['web.search.serpapi_api_key']: return await asyncio.to_thread( search_serpapi, - config.SERPAPI_API_KEY, - config.SERPAPI_ENGINE, + config['web.search.serpapi_api_key'], + config['web.search.serpapi_engine'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No SERPAPI_API_KEY found in environment variables') elif engine == 'jina': return await asyncio.to_thread( search_jina, - config.JINA_API_KEY, + config['web.search.jina_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.JINA_API_BASE_URL, + config['web.search.result_count'], + config['web.search.jina_api_base_url'], ) elif engine == 'bing': return await asyncio.to_thread( search_bing, - config.BING_SEARCH_V7_SUBSCRIPTION_KEY, - config.BING_SEARCH_V7_ENDPOINT, + config['web.search.bing_search_v7_subscription_key'], + config['web.search.bing_search_v7_endpoint'], str(DEFAULT_LOCALE), query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) elif engine == 'azure': - if config.AZURE_AI_SEARCH_API_KEY and config.AZURE_AI_SEARCH_ENDPOINT and config.AZURE_AI_SEARCH_INDEX_NAME: + if ( + config['web.search.azure_ai_search_api_key'] + and config['web.search.azure_ai_search_endpoint'] + and config['web.search.azure_ai_search_index_name'] + ): return await asyncio.to_thread( search_azure, - config.AZURE_AI_SEARCH_API_KEY, - config.AZURE_AI_SEARCH_ENDPOINT, - config.AZURE_AI_SEARCH_INDEX_NAME, + config['web.search.azure_ai_search_api_key'], + config['web.search.azure_ai_search_endpoint'], + config['web.search.azure_ai_search_index_name'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception( @@ -2821,88 +2461,88 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li elif engine == 'perplexity': return await asyncio.to_thread( search_perplexity, - config.PERPLEXITY_API_KEY, + config['web.search.perplexity_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - model=config.PERPLEXITY_MODEL, - search_context_usage=config.PERPLEXITY_SEARCH_CONTEXT_USAGE, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + model=config['web.search.perplexity_model'], + search_context_usage=config['web.search.perplexity_search_context_usage'], ) elif engine == 'microsoft_web_iq': - if config.MICROSOFT_WEB_IQ_API_KEY: + if config['web.search.microsoft_web_iq_api_key']: return await asyncio.to_thread( search_microsoft_web_iq, - config.MICROSOFT_WEB_IQ_API_BASE_URL, - config.MICROSOFT_WEB_IQ_API_KEY, + config['web.search.microsoft_web_iq_api_base_url'], + config['web.search.microsoft_web_iq_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, - config.MICROSOFT_WEB_IQ_LANGUAGE, + config['web.search.result_count'], + config['web.search.domain.filter_list'], + config['web.search.microsoft_web_iq_language'], user, ) else: raise Exception('No MICROSOFT_WEB_IQ_API_KEY found in environment variables') elif engine == 'sougou': - if config.SOUGOU_API_SID and config.SOUGOU_API_SK: + if config['web.search.sougou_api_sid'] and config['web.search.sougou_api_sk']: return await asyncio.to_thread( search_sougou, - config.SOUGOU_API_SID, - config.SOUGOU_API_SK, + config['web.search.sougou_api_sid'], + config['web.search.sougou_api_sk'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) else: raise Exception('No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables') elif engine == 'firecrawl': return await asyncio.to_thread( search_firecrawl, - config.FIRECRAWL_API_BASE_URL, - config.FIRECRAWL_API_KEY, + config['web.loader.firecrawl_api_url'], + config['web.loader.firecrawl_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) elif engine == 'external': return await asyncio.to_thread( search_external, request, - config.EXTERNAL_WEB_SEARCH_URL, - config.EXTERNAL_WEB_SEARCH_API_KEY, + config['web.search.external_web_search_url'], + config['web.search.external_web_search_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], user=user, ) elif engine == 'yandex': return await asyncio.to_thread( search_yandex, request, - config.YANDEX_WEB_SEARCH_URL, - config.YANDEX_WEB_SEARCH_API_KEY, - config.YANDEX_WEB_SEARCH_CONFIG, + config['web.search.yandex_web_search_url'], + config['web.search.yandex_web_search_api_key'], + config['web.search.yandex_web_search_config'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], user=user, ) elif engine == 'youcom': return await asyncio.to_thread( search_youcom, - config.YOUCOM_API_KEY, + config['web.search.youcom_api_key'], query, - config.WEB_SEARCH_RESULT_COUNT, - config.WEB_SEARCH_DOMAIN_FILTER_LIST, + config['web.search.result_count'], + config['web.search.domain.filter_list'], ) elif engine == 'linkup': - if config.LINKUP_API_KEY: + if config['web.search.linkup_api_key']: return await asyncio.to_thread( search_linkup, - api_key=config.LINKUP_API_KEY, + api_key=config['web.search.linkup_api_key'], query=query, - count=config.WEB_SEARCH_RESULT_COUNT, - filter_list=config.WEB_SEARCH_DOMAIN_FILTER_LIST, - params=config.LINKUP_SEARCH_PARAMS, + count=config['web.search.result_count'], + filter_list=config['web.search.domain.filter_list'], + params=config['web.search.linkup_search_params'], ) else: raise Exception('No LINKUP_API_KEY found in environment variables') @@ -2912,14 +2552,14 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li @router.post('/process/web/search') async def process_web_search(request: Request, form_data: SearchForm, user=Depends(get_verified_user)): - config = await get_retrieval_config() - if not config.ENABLE_WEB_SEARCH: + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) + if not config['web.search.enable']: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.ACCESS_PROHIBITED, ) - if user.role != 'admin' and not await has_permission(user.id, 'features.web_search', config.USER_PERMISSIONS): + if user.role != 'admin' and not await has_permission(user.id, 'features.web_search', config['user.permissions']): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=ERROR_MESSAGES.ACCESS_PROHIBITED, @@ -2929,12 +2569,12 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen result_items = [] try: - logging.debug('trying to web search with %s', (config.WEB_SEARCH_ENGINE, form_data.queries)) + logging.debug('trying to web search with %s', (config['web.search.engine'], form_data.queries)) # Use semaphore to limit concurrent requests based on WEB_SEARCH_CONCURRENT_REQUESTS # 0 or None = unlimited (previous behavior), positive number = limited concurrency # Set to 1 for sequential execution (rate-limited APIs like Brave free tier) - concurrent_limit = config.WEB_SEARCH_CONCURRENT_REQUESTS + concurrent_limit = config['web.search.concurrent_requests'] if concurrent_limit: # Limited concurrency with semaphore @@ -2944,9 +2584,10 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen async with semaphore: return await search_web( request, - config.WEB_SEARCH_ENGINE, + config['web.search.engine'], query, user, + config=config, ) search_tasks = [search_query_with_semaphore(query) for query in form_data.queries] @@ -2955,9 +2596,10 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen search_tasks = [ search_web( request, - config.WEB_SEARCH_ENGINE, + config['web.search.engine'], query, user, + config=config, ) for query in form_data.queries ] @@ -2988,7 +2630,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen ) try: - if config.BYPASS_WEB_SEARCH_WEB_LOADER: + if config['web.search.bypass_web_loader']: search_results = [item for result in search_results for item in result if result] docs = [ @@ -3005,14 +2647,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen if hasattr(result, 'snippet') and result.snippet is not None ] else: - loader_config = await get_loader_config() - loader = get_web_loader( - urls, - verify_ssl=loader_config.get('web_loader_ssl_verification'), - requests_per_second=loader_config.get('web_loader_concurrent_requests'), - trust_env=loader_config.get('web_search_trust_env'), - loader_config=loader_config, - ) + loader = get_web_loader(urls, config) docs = await loader.aload() urls = [ @@ -3023,7 +2658,7 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen dict(item) for item in result_items if item.link in url_set ] # only keep the search results that have been loaded - if config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: + if config['web.search.bypass_embedding_and_retrieval']: return { 'status': True, 'collection_name': None, @@ -3116,11 +2751,11 @@ async def query_doc_handler( form_data: QueryDocForm, user=Depends(get_verified_user), ): - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) await _validate_collection_access([form_data.collection_name], user) try: - if config.ENABLE_RAG_HYBRID_SEARCH and (form_data.hybrid is None or form_data.hybrid): + if config['rag.enable_hybrid_search'] and (form_data.hybrid is None or form_data.hybrid): return await query_doc_with_hybrid_search( collection_name=form_data.collection_name, collection_result=None, @@ -3128,18 +2763,18 @@ async def query_doc_handler( embedding_function=lambda query, prefix: request.app.state.EMBEDDING_FUNCTION( query, prefix=prefix, user=user ), - k=form_data.k if form_data.k else config.TOP_K, + k=form_data.k if form_data.k else config['rag.top_k'], reranking_function=( (lambda query, documents: request.app.state.RERANKING_FUNCTION(query, documents, user=user)) if request.app.state.RERANKING_FUNCTION else None ), - k_reranker=form_data.k_reranker or config.TOP_K_RERANKER, - r=(form_data.r if form_data.r else config.RELEVANCE_THRESHOLD), + k_reranker=form_data.k_reranker or config['rag.top_k_reranker'], + r=(form_data.r if form_data.r else config['rag.relevance_threshold']), hybrid_bm25_weight=( form_data.hybrid_bm25_weight if form_data.hybrid_bm25_weight is not None - else config.HYBRID_BM25_WEIGHT + else config['rag.hybrid_bm25_weight'] ), ) else: @@ -3152,7 +2787,7 @@ async def query_doc_handler( query_doc, collection_name=form_data.collection_name, query_embedding=query_embedding, - k=form_data.k if form_data.k else config.TOP_K, + k=form_data.k if form_data.k else config['rag.top_k'], user=user, ) except HTTPException: @@ -3182,34 +2817,34 @@ async def query_collection_handler( form_data: QueryCollectionsForm, user=Depends(get_verified_user), ): - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) await _validate_collection_access(form_data.collection_names, user) try: - if config.ENABLE_RAG_HYBRID_SEARCH and (form_data.hybrid is None or form_data.hybrid): + if config['rag.enable_hybrid_search'] and (form_data.hybrid is None or form_data.hybrid): return await query_collection_with_hybrid_search( collection_names=form_data.collection_names, queries=[form_data.query], embedding_function=lambda query, prefix: request.app.state.EMBEDDING_FUNCTION( query, prefix=prefix, user=user ), - k=form_data.k if form_data.k else config.TOP_K, + k=form_data.k if form_data.k else config['rag.top_k'], reranking_function=( (lambda query, documents: request.app.state.RERANKING_FUNCTION(query, documents, user=user)) if request.app.state.RERANKING_FUNCTION else None ), - k_reranker=form_data.k_reranker or config.TOP_K_RERANKER, - r=(form_data.r if form_data.r else config.RELEVANCE_THRESHOLD), + k_reranker=form_data.k_reranker or config['rag.top_k_reranker'], + r=(form_data.r if form_data.r else config['rag.relevance_threshold']), hybrid_bm25_weight=( form_data.hybrid_bm25_weight if form_data.hybrid_bm25_weight is not None - else config.HYBRID_BM25_WEIGHT + else config['rag.hybrid_bm25_weight'] ), enable_enriched_texts=( form_data.enable_enriched_texts if form_data.enable_enriched_texts is not None - else config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS + else config['rag.enable_hybrid_search_enriched_texts'] ), ) else: @@ -3220,7 +2855,7 @@ async def query_collection_handler( embedding_function=lambda query, prefix: request.app.state.EMBEDDING_FUNCTION( query, prefix=prefix, user=user ), - k=form_data.k if form_data.k else config.TOP_K, + k=form_data.k if form_data.k else config['rag.top_k'], user=user, ) @@ -3394,7 +3029,7 @@ async def process_files_batch( embedding (Files.update_file_by_id) manage their own short-lived sessions. """ - config = await get_retrieval_config() + config = await Config.get_many(*RETRIEVAL_CONFIG_KEYS.values()) collection_name = form_data.collection_name if collection_name: diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 674e02ca47..3842e737d9 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -50,7 +50,14 @@ 'linkup', 'openserp' ]; - let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'microsoft_web_iq', 'external']; + let webLoaderEngines = [ + 'playwright', + 'firecrawl', + 'tavily', + 'exa', + 'microsoft_web_iq', + 'external' + ]; let webConfig: any = null; const inputClass = @@ -1258,7 +1265,23 @@ - {#if webConfig.WEB_LOADER_ENGINE === '' || webConfig.WEB_LOADER_ENGINE === 'safe_web'} + {#if webConfig.WEB_LOADER_ENGINE === 'exa'} + {#if webConfig.WEB_SEARCH_ENGINE === 'exa'} +

+ {$i18n.t('Exa search and web loading share the API key configured above.')} +

+ {:else} + + + + {/if} + {/if} + + {#if ['', 'safe_web', 'exa'].includes(webConfig.WEB_LOADER_ENGINE)}