scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-10-10 03:27:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refac

Browse source
This commit is contained in:
Timothy Jaeryang Baek 2026-10-09 21:42:26 +04:00
parent 3dd1db1479
commit 09dcb60887
5 changed files with 927 additions and 1192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
backend/open_webui/retrieval/loaders/main.py
View file

@ -330,12 +330,12 @@ class DoclingLoader:
class Loader:
def __init__(self, engine: str = '', **kwargs):
self.engine = engine
self.user = kwargs.get('user', None)
self.user_groups = kwargs.get('user_groups', None)
self.metadata = kwargs.get('metadata', {})
self.kwargs = kwargs
def __init__(self, config: dict):
self.config = config
self.engine = config['rag.content_extraction_engine']
self.user = None
self.user_groups = None
self.metadata = {}
def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]:
loader = self._get_loader(filename, file_content_type, file_path)
@ -365,7 +365,7 @@ class Loader:
# is offloaded to a thread without a running event loop.
if self.engine == 'external' and self.user_groups is None:
self.user_groups = await get_user_groups_for_custom_headers(
self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_HEADERS'), self.user
self.config['rag.external_document_loader_headers'], self.user
)
return await asyncio.to_thread(self.load, filename, file_content_type, file_path)
@ -525,7 +525,7 @@ class Loader:
file_ext = filename.split('.')[-1].lower()
if file_ext in known_archive_ext or file_content_type in known_archive_content_types:
max_file_size = self.kwargs.get('FILE_MAX_SIZE')
max_file_size = self.config['rag.file.max_size']
try:
max_file_size_bytes = int(max_file_size) * 1024 * 1024 if max_file_size else 100 * 1024 * 1024
except (TypeError, ValueError):
@ -547,36 +547,36 @@ class Loader:
if (
self.engine == 'external'
and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL')
and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_API_KEY')
and self.config['rag.external_document_loader_url']
and self.config['rag.external_document_loader_api_key']
):
loader = ExternalDocumentLoader(
file_path=file_path,
url=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL'),
api_key=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_API_KEY'),
url=self.config['rag.external_document_loader_url'],
api_key=self.config['rag.external_document_loader_api_key'],
mime_type=file_content_type,
user=self.user,
user_groups=self.user_groups,
headers=self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_HEADERS'),
headers=self.config['rag.external_document_loader_headers'],
metadata={
**self.metadata,
'file_name': filename,
'file_content_type': file_content_type,
},
)
elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'):
elif self.engine == 'tika' and self.config['rag.tika_server_url']:
if self._is_text_file(file_ext, file_content_type):
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
else:
loader = TikaLoader(
url=self.kwargs.get('TIKA_SERVER_URL'),
url=self.config['rag.tika_server_url'],
file_path=file_path,
server_version=self.kwargs.get('TIKA_SERVER_VERSION'),
extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'),
server_version=self.config['rag.tika_server_version'],
extract_images=self.config['rag.pdf_extract_images'],
)
elif (
self.engine == 'datalab_marker'
and self.kwargs.get('DATALAB_MARKER_API_KEY')
and self.config['rag.datalab_marker_api_key']
and file_ext
in [
'pdf',
@ -599,30 +599,30 @@ class Loader:
'tiff',
]
):
api_base_url = self.kwargs.get('DATALAB_MARKER_API_BASE_URL', '')
api_base_url = self.config['rag.datalab_marker_api_base_url']
if not api_base_url or api_base_url.strip() == '':
api_base_url = 'https://www.datalab.to/api/v1/marker' # https://github.com/open-webui/open-webui/pull/16867#issuecomment-3218424349
loader = DatalabMarkerLoader(
file_path=file_path,
api_key=self.kwargs['DATALAB_MARKER_API_KEY'],
api_key=self.config['rag.datalab_marker_api_key'],
api_base_url=api_base_url,
additional_config=self.kwargs.get('DATALAB_MARKER_ADDITIONAL_CONFIG'),
use_llm=self.kwargs.get('DATALAB_MARKER_USE_LLM', False),
skip_cache=self.kwargs.get('DATALAB_MARKER_SKIP_CACHE', False),
force_ocr=self.kwargs.get('DATALAB_MARKER_FORCE_OCR', False),
paginate=self.kwargs.get('DATALAB_MARKER_PAGINATE', False),
strip_existing_ocr=self.kwargs.get('DATALAB_MARKER_STRIP_EXISTING_OCR', False),
disable_image_extraction=self.kwargs.get('DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION', False),
format_lines=self.kwargs.get('DATALAB_MARKER_FORMAT_LINES', False),
output_format=self.kwargs.get('DATALAB_MARKER_OUTPUT_FORMAT', 'markdown'),
additional_config=self.config['rag.datalab_marker_additional_config'],
use_llm=self.config['rag.datalab_marker_use_llm'],
skip_cache=self.config['rag.datalab_marker_skip_cache'],
force_ocr=self.config['rag.datalab_marker_force_ocr'],
paginate=self.config['rag.datalab_marker_paginate'],
strip_existing_ocr=self.config['rag.datalab_marker_strip_existing_ocr'],
disable_image_extraction=self.config['rag.datalab_marker_disable_image_extraction'],
format_lines=self.config['rag.datalab_marker_format_lines'],
output_format=self.config['rag.datalab_marker_output_format'],
)
elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'):
elif self.engine == 'docling' and self.config['rag.docling_server_url']:
if self._is_text_file(file_ext, file_content_type):
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
else:
# Build params for DoclingLoader
params = self.kwargs.get('DOCLING_PARAMS', {})
params = self.config['rag.docling_params']
if not isinstance(params, dict):
try:
params = JSONCodec.loads(params)
@ -631,15 +631,15 @@ class Loader:
params = {}
loader = DoclingLoader(
url=self.kwargs.get('DOCLING_SERVER_URL'),
api_key=self.kwargs.get('DOCLING_API_KEY', None),
url=self.config['rag.docling_server_url'],
api_key=self.config['rag.docling_api_key'],
file_path=file_path,
mime_type=file_content_type,
params=params,
)
elif (
self.engine == 'document_intelligence'
and self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT') != ''
and self.config['rag.document_intelligence_endpoint'] != ''
and (
file_ext in ['pdf', 'docx', 'ppt', 'pptx']
or file_content_type
@ -650,22 +650,22 @@ class Loader:
]
)
):
if self.kwargs.get('DOCUMENT_INTELLIGENCE_KEY') != '':
if self.config['rag.document_intelligence_key'] != '':
loader = DocumentIntelligenceLoader(
file_path=file_path,
api_endpoint=self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT'),
api_key=self.kwargs.get('DOCUMENT_INTELLIGENCE_KEY'),
api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'),
api_endpoint=self.config['rag.document_intelligence_endpoint'],
api_key=self.config['rag.document_intelligence_key'],
api_model=self.config['rag.document_intelligence_model'],
)
else:
loader = DocumentIntelligenceLoader(
file_path=file_path,
api_endpoint=self.kwargs.get('DOCUMENT_INTELLIGENCE_ENDPOINT'),
api_endpoint=self.config['rag.document_intelligence_endpoint'],
azure_credential=DefaultAzureCredential(),
api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'),
api_model=self.config['rag.document_intelligence_model'],
)
elif self.engine == 'mineru' and file_ext in self.kwargs.get('MINERU_FILE_EXTENSIONS', ['pdf']):
mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300)
elif self.engine == 'mineru' and file_ext in self.config['rag.mineru_file_extensions']:
mineru_timeout = self.config['rag.mineru_api_timeout']
if mineru_timeout:
try:
mineru_timeout = int(mineru_timeout)
@ -673,34 +673,34 @@ class Loader:
mineru_timeout = 300
loader = MinerULoader(
file_path=file_path,
api_mode=self.kwargs.get('MINERU_API_MODE', 'local'),
api_url=self.kwargs.get('MINERU_API_URL', 'http://localhost:8000'),
api_key=self.kwargs.get('MINERU_API_KEY', ''),
params=self.kwargs.get('MINERU_PARAMS', {}),
api_mode=self.config['rag.mineru_api_mode'],
api_url=self.config['rag.mineru_api_url'],
api_key=self.config['rag.mineru_api_key'],
params=self.config['rag.mineru_params'],
timeout=mineru_timeout,
max_markdown_bytes=MINERU_MAX_MARKDOWN_BYTES,
)
elif (
self.engine == 'mistral_ocr'
and self.kwargs.get('MISTRAL_OCR_API_KEY') != ''
and self.config['rag.mistral_ocr_api_key'] != ''
and file_ext in ['pdf'] # Mistral OCR currently only supports PDF and images
):
loader = MistralLoader(
base_url=self.kwargs.get('MISTRAL_OCR_API_BASE_URL'),
api_key=self.kwargs.get('MISTRAL_OCR_API_KEY'),
base_url=self.config['rag.mistral_ocr_api_base_url'],
api_key=self.config['rag.mistral_ocr_api_key'],
file_path=file_path,
use_base64=self.kwargs.get('MISTRAL_OCR_USE_BASE64', False),
use_base64=self.config['rag.mistral_ocr_use_base64'],
user=self.user,
)
elif (
self.engine == 'paddleocr_vl'
and self.kwargs.get('PADDLEOCR_VL_BASE_URL')
and self.kwargs.get('PADDLEOCR_VL_TOKEN')
and self.config['rag.paddleocr_vl_base_url']
and self.config['rag.paddleocr_vl_token']
and file_ext in PADDLEOCR_VL_SUPPORTED_EXTENSIONS
):
loader = PaddleOCRVLLoader(
api_url=self.kwargs.get('PADDLEOCR_VL_BASE_URL'),
token=self.kwargs.get('PADDLEOCR_VL_TOKEN'),
api_url=self.config['rag.paddleocr_vl_base_url'],
token=self.config['rag.paddleocr_vl_token'],
file_path=file_path,
)
else:
@ -720,8 +720,8 @@ class Loader:
if file_ext == 'pdf':
loader = PDFLoader(
file_path,
extract_images=self.kwargs.get('PDF_EXTRACT_IMAGES'),
mode=self.kwargs.get('PDF_LOADER_MODE', 'page'),
extract_images=self.config['rag.pdf_extract_images'],
mode=self.config['rag.pdf_loader_mode'],
)
elif file_ext == 'csv':
loader = CSVLoaderWithSummary(

167
backend/open_webui/retrieval/utils.py
View file

@ -80,101 +80,25 @@ def is_youtube_url(url: str) -> bool:
return re.match(youtube_regex, url) is not None
LOADER_CONFIG_KEYS = {
'file_max_size': 'rag.file.max_size',
'youtube_language': 'rag.youtube_loader_language',
'youtube_proxy_url': 'rag.youtube_loader_proxy_url',
'web_loader_ssl_verification': 'web.loader.ssl_verification',
'web_loader_concurrent_requests': 'web.loader.concurrent_requests',
'web_search_trust_env': 'web.search.trust_env',
'web_loader_engine': 'web.loader.engine',
'web_loader_timeout': 'web.loader.timeout',
'playwright_ws_url': 'web.loader.playwright_ws_url',
'playwright_timeout': 'web.loader.playwright_timeout',
'firecrawl_api_key': 'web.loader.firecrawl_api_key',
'firecrawl_api_url': 'web.loader.firecrawl_api_url',
'firecrawl_timeout': 'web.loader.firecrawl_timeout',
'tavily_api_key': 'web.search.tavily_api_key',
'tavily_extract_depth': 'web.search.tavily_extract_depth',
'microsoft_web_iq_api_base_url': 'web.search.microsoft_web_iq_api_base_url',
'microsoft_web_iq_api_key': 'web.search.microsoft_web_iq_api_key',
'microsoft_web_iq_language': 'web.search.microsoft_web_iq_language',
'external_web_loader_url': 'web.loader.external_web_loader_url',
'external_web_loader_api_key': 'web.loader.external_web_loader_api_key',
'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key',
'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config',
'DATALAB_MARKER_SKIP_CACHE': 'rag.datalab_marker_skip_cache',
'DATALAB_MARKER_FORCE_OCR': 'rag.datalab_marker_force_ocr',
'DATALAB_MARKER_PAGINATE': 'rag.datalab_marker_paginate',
'DATALAB_MARKER_STRIP_EXISTING_OCR': 'rag.datalab_marker_strip_existing_ocr',
'DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION': 'rag.datalab_marker_disable_image_extraction',
'DATALAB_MARKER_FORMAT_LINES': 'rag.datalab_marker_format_lines',
'DATALAB_MARKER_USE_LLM': 'rag.datalab_marker_use_llm',
'DATALAB_MARKER_OUTPUT_FORMAT': 'rag.datalab_marker_output_format',
'EXTERNAL_DOCUMENT_LOADER_URL': 'rag.external_document_loader_url',
'EXTERNAL_DOCUMENT_LOADER_API_KEY': 'rag.external_document_loader_api_key',
'EXTERNAL_DOCUMENT_LOADER_HEADERS': 'rag.external_document_loader_headers',
'TIKA_SERVER_URL': 'rag.tika_server_url',
'TIKA_SERVER_VERSION': 'rag.tika_server_version',
'DOCLING_SERVER_URL': 'rag.docling_server_url',
'DOCLING_API_KEY': 'rag.docling_api_key',
'DOCLING_PARAMS': 'rag.docling_params',
'PDF_EXTRACT_IMAGES': 'rag.pdf_extract_images',
'PDF_LOADER_MODE': 'rag.pdf_loader_mode',
'DOCUMENT_INTELLIGENCE_ENDPOINT': 'rag.document_intelligence_endpoint',
'DOCUMENT_INTELLIGENCE_KEY': 'rag.document_intelligence_key',
'DOCUMENT_INTELLIGENCE_MODEL': 'rag.document_intelligence_model',
'MISTRAL_OCR_API_BASE_URL': 'rag.mistral_ocr_api_base_url',
'MISTRAL_OCR_API_KEY': 'rag.mistral_ocr_api_key',
'MISTRAL_OCR_USE_BASE64': 'rag.mistral_ocr_use_base64',
'PADDLEOCR_VL_BASE_URL': 'rag.paddleocr_vl_base_url',
'PADDLEOCR_VL_TOKEN': 'rag.paddleocr_vl_token',
'MINERU_API_MODE': 'rag.mineru_api_mode',
'MINERU_API_URL': 'rag.mineru_api_url',
'MINERU_API_KEY': 'rag.mineru_api_key',
'MINERU_API_TIMEOUT': 'rag.mineru_api_timeout',
'MINERU_PARAMS': 'rag.mineru_params',
'MINERU_FILE_EXTENSIONS': 'rag.mineru_file_extensions',
}
async def get_loader_config():
values = await Config.get_many(*LOADER_CONFIG_KEYS.values())
return {name: values.get(key) for name, key in LOADER_CONFIG_KEYS.items()}
def get_loader(request, url: str, config: dict):
if is_youtube_url(url):
return YoutubeLoader(
url,
language=config.get('youtube_language'),
proxy_url=config.get('youtube_proxy_url'),
language=config['rag.youtube_loader_language'],
proxy_url=config['rag.youtube_loader_proxy_url'],
)
return get_web_loader(
url,
verify_ssl=config.get('web_loader_ssl_verification'),
requests_per_second=config.get('web_loader_concurrent_requests'),
trust_env=config.get('web_search_trust_env'),
loader_config=config,
)
return get_web_loader(url, config)
def build_loader_from_config(request, config: dict):
"""Build a Loader instance with the admin's configured extraction engine settings."""
def build_loader_from_config(config: dict):
"""Build a document loader with the shared retrieval settings."""
from open_webui.retrieval.loaders.main import Loader
loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
loader_config['FILE_MAX_SIZE'] = config.get('file_max_size')
return Loader(
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
)
return Loader(config)
def _extract_text_from_binary_response(
request, response: requests.Response, url: str, loader_config: dict
request, response: requests.Response, url: str, config: dict
) -> tuple[str, list]:
"""Download response body to a temp file and extract text using the Loader pipeline."""
import mimetypes
@ -199,7 +123,7 @@ def _extract_text_from_binary_response(
suffix = '.' + filename.split('.')[-1].lower() if '.' in filename else ''
max_size = loader_config.get('file_max_size')
max_size = config['rag.file.max_size']
max_bytes = int(max_size) * 1024 * 1024 if max_size else 0
tmp_fd, tmp_path = tempfile.mkstemp(suffix=suffix)
@ -213,7 +137,7 @@ def _extract_text_from_binary_response(
raise ValueError(ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_size} MB'))
tmp.write(chunk)
loader = build_loader_from_config(request, loader_config)
loader = build_loader_from_config(config)
docs = loader.load(filename, content_type, tmp_path)
for doc in docs:
doc.metadata['source'] = url
@ -243,16 +167,75 @@ def _is_text_content_type(content_type: str) -> bool:
return ct.endswith(('+xml', '+json'))
async def get_content_from_url(request, url: str) -> str:
loader_config = await get_loader_config()
async def get_content_from_url(request, url: str, *, config: dict | None = None) -> tuple[str, list]:
if config is None:
config = await Config.get_many(
'rag.content_extraction_engine',
'rag.datalab_marker_additional_config',
'rag.datalab_marker_api_base_url',
'rag.datalab_marker_api_key',
'rag.datalab_marker_disable_image_extraction',
'rag.datalab_marker_force_ocr',
'rag.datalab_marker_format_lines',
'rag.datalab_marker_output_format',
'rag.datalab_marker_paginate',
'rag.datalab_marker_skip_cache',
'rag.datalab_marker_strip_existing_ocr',
'rag.datalab_marker_use_llm',
'rag.docling_api_key',
'rag.docling_params',
'rag.docling_server_url',
'rag.document_intelligence_endpoint',
'rag.document_intelligence_key',
'rag.document_intelligence_model',
'web.loader.ssl_verification',
'web.search.exa_api_key',
'rag.external_document_loader_api_key',
'rag.external_document_loader_headers',
'rag.external_document_loader_url',
'web.loader.external_web_loader_api_key',
'web.loader.external_web_loader_url',
'rag.file.max_size',
'web.loader.firecrawl_api_url',
'web.loader.firecrawl_api_key',
'web.loader.firecrawl_timeout',
'web.search.microsoft_web_iq_api_base_url',
'web.search.microsoft_web_iq_api_key',
'web.search.microsoft_web_iq_language',
'rag.mineru_api_key',
'rag.mineru_api_mode',
'rag.mineru_api_timeout',
'rag.mineru_api_url',
'rag.mineru_file_extensions',
'rag.mineru_params',
'rag.mistral_ocr_api_base_url',
'rag.mistral_ocr_api_key',
'rag.mistral_ocr_use_base64',
'rag.paddleocr_vl_base_url',
'rag.paddleocr_vl_token',
'rag.pdf_extract_images',
'rag.pdf_loader_mode',
'web.loader.playwright_timeout',
'web.loader.playwright_ws_url',
'web.search.tavily_api_key',
'web.search.tavily_extract_depth',
'rag.tika_server_url',
'rag.tika_server_version',
'web.loader.concurrent_requests',
'web.loader.engine',
'web.loader.timeout',
'web.search.trust_env',
'rag.youtube_loader_language',
'rag.youtube_loader_proxy_url',
)
# The rest of this function performs synchronous, blocking work: an SSRF-guarded
# `requests` probe and a synchronous document loader (`loader.load()`). Run it in a
# worker thread so the event loop stays free while waiting on network/parsing.
return await asyncio.to_thread(_get_content_from_url_sync, request, url, loader_config)
return await asyncio.to_thread(_get_content_from_url_sync, request, url, config)
def _get_content_from_url_sync(request, url: str, loader_config):
def _get_content_from_url_sync(request, url: str, config: dict):
from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session
# Validate URL before making any request (blocks private IPs, non-HTTP, filter list)
@ -265,7 +248,7 @@ def _get_content_from_url_sync(request, url: str, loader_config):
# when allow_redirects=False, causing the binary-content path to run
# and produce empty docs → HTTP 400.
if is_youtube_url(url):
loader = get_loader(request, url, loader_config)
loader = get_loader(request, url, config)
docs = loader.load()
content = ' '.join([doc.page_content for doc in docs])
return content, docs
@ -289,14 +272,14 @@ def _get_content_from_url_sync(request, url: str, loader_config):
if response is None or _is_text_content_type(content_type):
if response is not None:
response.close()
loader = get_loader(request, url, loader_config)
loader = get_loader(request, url, config)
docs = loader.load()
content = ' '.join([doc.page_content for doc in docs])
return content, docs
# Binary content (PDF, DOCX, XLSX, PPTX, etc.) — download and extract
try:
return _extract_text_from_binary_response(request, response, url, loader_config)
return _extract_text_from_binary_response(request, response, url, config)
finally:
response.close()

176
backend/open_webui/retrieval/web/utils.py
View file

@ -2,6 +2,7 @@ import asyncio
import http.cookiejar
import ipaddress
import logging
import math
import socket
import ssl
import time
@ -35,22 +36,10 @@ from fastapi.concurrency import run_in_threadpool
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document
from open_webui.config import (
DEFAULT_CONFIG,
ENABLE_LOCAL_WEB_FETCH,
EXTERNAL_WEB_LOADER_API_KEY,
EXTERNAL_WEB_LOADER_URL,
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
FIRECRAWL_TIMEOUT,
MICROSOFT_WEB_IQ_API_BASE_URL,
MICROSOFT_WEB_IQ_API_KEY,
MICROSOFT_WEB_IQ_LANGUAGE,
PLAYWRIGHT_TIMEOUT,
PLAYWRIGHT_WS_URL,
TAVILY_API_KEY,
TAVILY_EXTRACT_DEPTH,
WEB_FETCH_FILTER_LIST,
WEB_LOADER_ENGINE,
WEB_LOADER_TIMEOUT,
)
from open_webui.constants import ERROR_MESSAGES
from open_webui.env import (
@ -381,6 +370,111 @@ class RateLimitMixin:
self.last_request_time = datetime.now()
class SafeExaLoader(BaseLoader, RateLimitMixin):
def __init__(
self,
web_paths: Union[str, Sequence[str]],
api_key: str,
timeout: Optional[str] = None,
verify_ssl: bool = True,
trust_env: bool = False,
requests_per_second: Optional[float] = None,
continue_on_failure: bool = True,
):
if not api_key or not api_key.strip():
raise ValueError('Exa web loader requires an EXA_API_KEY')
self.web_paths = [web_paths] if isinstance(web_paths, str) else list(web_paths)
self.api_key = api_key
try:
request_timeout = float(timeout)
except (TypeError, ValueError):
request_timeout = 60
self.timeout = request_timeout if math.isfinite(request_timeout) and request_timeout > 0 else 60
self.verify_ssl = verify_ssl
self.trust_env = trust_env
self.requests_per_second = requests_per_second
self.last_request_time = None
self.continue_on_failure = continue_on_failure
def lazy_load(self) -> Iterator[Document]:
# Exa's search models import this module for URL validation.
from open_webui.retrieval.web.exa import EXA_API_BASE
loaded = 0
with requests.Session() as session:
session.trust_env = self.trust_env
session.verify = self.verify_ssl
session.headers.update({'Authorization': f'Bearer {self.api_key}'})
for offset in range(0, len(self.web_paths), 100):
urls = self.web_paths[offset : offset + 100]
self._sync_wait_for_rate_limit()
try:
response = session.post(
f'{EXA_API_BASE}/contents',
json={'urls': urls, 'text': True},
timeout=self.timeout,
allow_redirects=False,
)
if response.status_code in (401, 402, 403):
raise PermissionError(
f'Exa web loader authentication or billing failed (HTTP {response.status_code})'
)
response.raise_for_status()
data = response.json()
if not isinstance(data, dict) or not isinstance(data.get('results'), list):
raise ValueError('Invalid Exa Contents response')
if data.get('statuses') is not None and not isinstance(data['statuses'], list):
raise ValueError('Invalid Exa Contents statuses')
except PermissionError:
raise
except (requests.RequestException, ValueError) as e:
# Do not log provider bodies or exception messages, which can contain credentials.
log.warning('Exa web loader batch failed (%s)', type(e).__name__)
if not self.continue_on_failure:
raise ValueError('Exa web loader request failed') from None
continue
failed_ids = {
status.get('id')
for status in (data.get('statuses') or [])
if isinstance(status, dict)
and isinstance(status.get('id'), str)
and status.get('status') != 'success'
}
documents = {}
for result in data['results']:
if not isinstance(result, dict):
continue
source = next((result.get(key) for key in ('id', 'url') if result.get(key) in urls), None)
content = result.get('text')
if source is None or source in failed_ids or not isinstance(content, str):
continue
if not content.strip():
continue
metadata = {'source': source}
if isinstance(result.get('title'), str):
metadata['title'] = result['title']
documents[source] = Document(page_content=content, metadata=metadata)
missing = len(set(urls) - documents.keys())
if missing:
log.warning('Exa web loader could not load %s URL(s)', missing)
if not self.continue_on_failure:
raise ValueError(f'Exa web loader could not load {missing} URL(s)')
for url in urls:
if url in documents:
loaded += 1
yield documents[url]
if not loaded:
raise ValueError('Exa web loader could not load any page content')
async def alazy_load(self) -> AsyncIterator[Document]:
docs = await run_in_threadpool(lambda: list(self.lazy_load()))
for doc in docs:
yield doc
class URLProcessingMixin:
async def _verify_ssl_cert(self, url: str) -> bool:
"""Verify SSL certificate for a URL."""
@ -1056,10 +1150,7 @@ class SafeWebBaseLoader(BaseLoader):
def get_web_loader(
urls: Union[str, Sequence[str]],
verify_ssl: bool = True,
requests_per_second: int = 2,
trust_env: bool = False,
loader_config: Optional[dict] = None,
config: dict,
):
# Check if the URLs are valid
safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)
@ -1068,22 +1159,20 @@ def get_web_loader(
log.warning(f'All provided URLs were blocked or invalid: {urls}')
raise ValueError(ERROR_MESSAGES.INVALID_URL)
loader_config = loader_config or {}
def cfg(key):
# Preserve the web loaders' fallback for legacy null settings.
value = config.get(key)
return DEFAULT_CONFIG[key] if value is None else value
def cfg(key, env_value):
# Admin-saved DB value wins; env constant covers keys never saved.
value = loader_config.get(key)
return env_value if value is None else value
engine = cfg('web_loader_engine', WEB_LOADER_ENGINE)
web_loader_timeout = cfg('web_loader_timeout', WEB_LOADER_TIMEOUT)
engine = cfg('web.loader.engine')
web_loader_timeout = cfg('web.loader.timeout')
web_loader_args = {
'web_paths': safe_urls,
'verify_ssl': verify_ssl,
'requests_per_second': requests_per_second,
'verify_ssl': config['web.loader.ssl_verification'],
'requests_per_second': config['web.loader.concurrent_requests'],
'continue_on_failure': True,
'trust_env': trust_env,
'trust_env': config['web.search.trust_env'],
}
WebLoaderClass = None
@ -1106,16 +1195,16 @@ def get_web_loader(
if engine == 'playwright':
WebLoaderClass = SafePlaywrightURLLoader
web_loader_args['playwright_timeout'] = cfg('playwright_timeout', PLAYWRIGHT_TIMEOUT)
playwright_ws_url = cfg('playwright_ws_url', PLAYWRIGHT_WS_URL)
web_loader_args['playwright_timeout'] = cfg('web.loader.playwright_timeout')
playwright_ws_url = cfg('web.loader.playwright_ws_url')
if playwright_ws_url:
web_loader_args['playwright_ws_url'] = playwright_ws_url
if engine == 'firecrawl':
WebLoaderClass = SafeFireCrawlLoader
web_loader_args['api_key'] = cfg('firecrawl_api_key', FIRECRAWL_API_KEY)
web_loader_args['api_url'] = cfg('firecrawl_api_url', FIRECRAWL_API_BASE_URL)
firecrawl_timeout = cfg('firecrawl_timeout', FIRECRAWL_TIMEOUT)
web_loader_args['api_key'] = cfg('web.loader.firecrawl_api_key')
web_loader_args['api_url'] = cfg('web.loader.firecrawl_api_url')
firecrawl_timeout = cfg('web.loader.firecrawl_timeout')
if firecrawl_timeout:
try:
web_loader_args['timeout'] = int(firecrawl_timeout)
@ -1124,14 +1213,19 @@ def get_web_loader(
if engine == 'tavily':
WebLoaderClass = SafeTavilyLoader
web_loader_args['api_key'] = cfg('tavily_api_key', TAVILY_API_KEY)
web_loader_args['extract_depth'] = cfg('tavily_extract_depth', TAVILY_EXTRACT_DEPTH)
web_loader_args['api_key'] = cfg('web.search.tavily_api_key')
web_loader_args['extract_depth'] = cfg('web.search.tavily_extract_depth')
if engine == 'exa':
WebLoaderClass = SafeExaLoader
web_loader_args['api_key'] = cfg('web.search.exa_api_key')
web_loader_args['timeout'] = web_loader_timeout
if engine == 'microsoft_web_iq':
WebLoaderClass = SafeMicrosoftWebIQLoader
web_loader_args['api_base_url'] = cfg('microsoft_web_iq_api_base_url', MICROSOFT_WEB_IQ_API_BASE_URL)
web_loader_args['api_key'] = cfg('microsoft_web_iq_api_key', MICROSOFT_WEB_IQ_API_KEY)
web_loader_args['language'] = cfg('microsoft_web_iq_language', MICROSOFT_WEB_IQ_LANGUAGE)
web_loader_args['api_base_url'] = cfg('web.search.microsoft_web_iq_api_base_url')
web_loader_args['api_key'] = cfg('web.search.microsoft_web_iq_api_key')
web_loader_args['language'] = cfg('web.search.microsoft_web_iq_language')
if web_loader_timeout:
try:
web_loader_args['timeout'] = int(web_loader_timeout)
@ -1140,8 +1234,8 @@ def get_web_loader(
if engine == 'external':
WebLoaderClass = ExternalWebLoader
web_loader_args['external_url'] = cfg('external_web_loader_url', EXTERNAL_WEB_LOADER_URL)
web_loader_args['external_api_key'] = cfg('external_web_loader_api_key', EXTERNAL_WEB_LOADER_API_KEY)
web_loader_args['external_url'] = cfg('web.loader.external_web_loader_url')
web_loader_args['external_api_key'] = cfg('web.loader.external_web_loader_api_key')
if WebLoaderClass:
web_loader = WebLoaderClass(**web_loader_args)
@ -1156,5 +1250,5 @@ def get_web_loader(
else:
raise ValueError(
f'Invalid WEB_LOADER_ENGINE: {engine}. '
"Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'external', or 'microsoft_web_iq'."
"Please set it to 'safe_web', 'playwright', 'firecrawl', 'tavily', 'exa', 'external', or 'microsoft_web_iq'."
)

1637
backend/open_webui/routers/retrieval.py
View file

File diff suppressed because it is too large Load diff

27
src/lib/components/admin/Settings/WebSearch.svelte
View file

@ -50,7 +50,14 @@
'linkup',
'openserp'
];
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'microsoft_web_iq', 'external'];
let webLoaderEngines = [
'playwright',
'firecrawl',
'tavily',
'exa',
'microsoft_web_iq',
'external'
];
let webConfig: any = null;
const inputClass =
@ -1258,7 +1265,23 @@
</SettingsSelect>
</AdminSettingRow>
{#if webConfig.WEB_LOADER_ENGINE === '' || webConfig.WEB_LOADER_ENGINE === 'safe_web'}
{#if webConfig.WEB_LOADER_ENGINE === 'exa'}
{#if webConfig.WEB_SEARCH_ENGINE === 'exa'}
<p class="text-xs text-gray-500 dark:text-gray-400">
{$i18n.t('Exa search and web loading share the API key configured above.')}
</p>
{:else}
<AdminSettingField label={$i18n.t('settings.admin.web.exaApiKey.label')}>
<SensitiveInput
variant="settings"
placeholder={$i18n.t('Enter Exa API Key')}
bind:value={webConfig.EXA_API_KEY}
/>
</AdminSettingField>
{/if}
{/if}
{#if ['', 'safe_web', 'exa'].includes(webConfig.WEB_LOADER_ENGINE)}
<AdminSettingField
label={$i18n.t('settings.admin.web.timeout.label')}
description={$i18n.t('settings.admin.web.timeout.description')}