diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 72a71021f9..6f4ec8e0c7 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1171,6 +1171,12 @@ FIRECRAWL_API_BASE_URL = os.getenv('FIRECRAWL_API_BASE_URL', 'https://api.firecr FIRECRAWL_TIMEOUT = os.getenv('FIRECRAWL_TIMEOUT', '') +CRW_API_KEY = os.getenv('CRW_API_KEY', '') + +CRW_API_BASE_URL = os.getenv('CRW_API_BASE_URL', 'https://fastcrw.com/api') + +CRW_TIMEOUT = os.getenv('CRW_TIMEOUT', '') + EXTERNAL_WEB_SEARCH_URL = os.getenv('EXTERNAL_WEB_SEARCH_URL', '') EXTERNAL_WEB_SEARCH_API_KEY = os.getenv('EXTERNAL_WEB_SEARCH_API_KEY', '') diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 9aacdfeef6..ea24907935 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -53,6 +53,24 @@ from open_webui.config import ( ENABLE_ONEDRIVE_PERSONAL, # OpenAI ENV, + EVALUATION_ARENA_MODELS, + EXA_API_KEY, + EXTERNAL_DOCUMENT_LOADER_API_KEY, + EXTERNAL_DOCUMENT_LOADER_URL, + EXTERNAL_WEB_LOADER_API_KEY, + EXTERNAL_WEB_LOADER_URL, + EXTERNAL_WEB_SEARCH_API_KEY, + EXTERNAL_WEB_SEARCH_URL, + FILE_IMAGE_COMPRESSION_HEIGHT, + FILE_IMAGE_COMPRESSION_WIDTH, + FIRECRAWL_API_BASE_URL, + FIRECRAWL_API_KEY, + FIRECRAWL_TIMEOUT, + CRW_API_BASE_URL, + CRW_API_KEY, + CRW_TIMEOUT, + FOLDER_MAX_FILE_COUNT, + FOLLOW_UP_GENERATION_PROMPT_TEMPLATE, FRONTEND_BUILD_DIR, GOOGLE_DRIVE_API_KEY, GOOGLE_DRIVE_CLIENT_ID, @@ -509,6 +527,179 @@ app.state.BASE_MODELS = [] +app.state.config.TOP_K = RAG_TOP_K +app.state.config.TOP_K_RERANKER = RAG_TOP_K_RERANKER +app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD +app.state.config.HYBRID_BM25_WEIGHT = RAG_HYBRID_BM25_WEIGHT + + +app.state.config.ALLOWED_FILE_EXTENSIONS = RAG_ALLOWED_FILE_EXTENSIONS +app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE +app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT +app.state.config.FILE_IMAGE_COMPRESSION_WIDTH = FILE_IMAGE_COMPRESSION_WIDTH +app.state.config.FILE_IMAGE_COMPRESSION_HEIGHT = FILE_IMAGE_COMPRESSION_HEIGHT + + +app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT +app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL +app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH +app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS +app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION = ENABLE_WEB_LOADER_SSL_VERIFICATION + +app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE +app.state.config.DATALAB_MARKER_API_KEY = DATALAB_MARKER_API_KEY +app.state.config.DATALAB_MARKER_API_BASE_URL = DATALAB_MARKER_API_BASE_URL +app.state.config.DATALAB_MARKER_ADDITIONAL_CONFIG = DATALAB_MARKER_ADDITIONAL_CONFIG +app.state.config.DATALAB_MARKER_SKIP_CACHE = DATALAB_MARKER_SKIP_CACHE +app.state.config.DATALAB_MARKER_FORCE_OCR = DATALAB_MARKER_FORCE_OCR +app.state.config.DATALAB_MARKER_PAGINATE = DATALAB_MARKER_PAGINATE +app.state.config.DATALAB_MARKER_STRIP_EXISTING_OCR = DATALAB_MARKER_STRIP_EXISTING_OCR +app.state.config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION = DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION +app.state.config.DATALAB_MARKER_FORMAT_LINES = DATALAB_MARKER_FORMAT_LINES +app.state.config.DATALAB_MARKER_USE_LLM = DATALAB_MARKER_USE_LLM +app.state.config.DATALAB_MARKER_OUTPUT_FORMAT = DATALAB_MARKER_OUTPUT_FORMAT +app.state.config.EXTERNAL_DOCUMENT_LOADER_URL = EXTERNAL_DOCUMENT_LOADER_URL +app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY = EXTERNAL_DOCUMENT_LOADER_API_KEY +app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL +app.state.config.DOCLING_SERVER_URL = DOCLING_SERVER_URL +app.state.config.DOCLING_API_KEY = DOCLING_API_KEY +app.state.config.DOCLING_PARAMS = DOCLING_PARAMS +app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = DOCUMENT_INTELLIGENCE_ENDPOINT +app.state.config.DOCUMENT_INTELLIGENCE_KEY = DOCUMENT_INTELLIGENCE_KEY +app.state.config.DOCUMENT_INTELLIGENCE_MODEL = DOCUMENT_INTELLIGENCE_MODEL +app.state.config.MISTRAL_OCR_API_BASE_URL = MISTRAL_OCR_API_BASE_URL +app.state.config.MISTRAL_OCR_API_KEY = MISTRAL_OCR_API_KEY +app.state.config.PADDLEOCR_VL_BASE_URL = PADDLEOCR_VL_BASE_URL +app.state.config.PADDLEOCR_VL_TOKEN = PADDLEOCR_VL_TOKEN +app.state.config.MINERU_API_MODE = MINERU_API_MODE +app.state.config.MINERU_API_URL = MINERU_API_URL +app.state.config.MINERU_API_KEY = MINERU_API_KEY +app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT +app.state.config.MINERU_PARAMS = MINERU_PARAMS +app.state.config.MINERU_FILE_EXTENSIONS = MINERU_FILE_EXTENSIONS + +app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER +app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER + +app.state.config.TIKTOKEN_ENCODING_NAME = TIKTOKEN_ENCODING_NAME + +app.state.config.CHUNK_SIZE = CHUNK_SIZE +app.state.config.CHUNK_MIN_SIZE_TARGET = CHUNK_MIN_SIZE_TARGET +app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP + + +app.state.config.RAG_EMBEDDING_ENGINE = RAG_EMBEDDING_ENGINE +app.state.config.RAG_EMBEDDING_MODEL = RAG_EMBEDDING_MODEL +app.state.config.RAG_EMBEDDING_BATCH_SIZE = RAG_EMBEDDING_BATCH_SIZE +app.state.config.ENABLE_ASYNC_EMBEDDING = ENABLE_ASYNC_EMBEDDING +app.state.config.RAG_EMBEDDING_CONCURRENT_REQUESTS = RAG_EMBEDDING_CONCURRENT_REQUESTS + +app.state.config.RAG_RERANKING_ENGINE = RAG_RERANKING_ENGINE +app.state.config.RAG_RERANKING_MODEL = RAG_RERANKING_MODEL +app.state.config.RAG_EXTERNAL_RERANKER_URL = RAG_EXTERNAL_RERANKER_URL +app.state.config.RAG_EXTERNAL_RERANKER_API_KEY = RAG_EXTERNAL_RERANKER_API_KEY +app.state.config.RAG_EXTERNAL_RERANKER_TIMEOUT = RAG_EXTERNAL_RERANKER_TIMEOUT +app.state.config.RAG_RERANKING_BATCH_SIZE = RAG_RERANKING_BATCH_SIZE + +app.state.config.RAG_TEMPLATE = RAG_TEMPLATE + +app.state.config.RAG_OPENAI_API_BASE_URL = RAG_OPENAI_API_BASE_URL +app.state.config.RAG_OPENAI_API_KEY = RAG_OPENAI_API_KEY + +app.state.config.RAG_AZURE_OPENAI_BASE_URL = RAG_AZURE_OPENAI_BASE_URL +app.state.config.RAG_AZURE_OPENAI_API_KEY = RAG_AZURE_OPENAI_API_KEY +app.state.config.RAG_AZURE_OPENAI_API_VERSION = RAG_AZURE_OPENAI_API_VERSION + +app.state.config.RAG_OLLAMA_BASE_URL = RAG_OLLAMA_BASE_URL +app.state.config.RAG_OLLAMA_API_KEY = RAG_OLLAMA_API_KEY + +app.state.config.PDF_EXTRACT_IMAGES = PDF_EXTRACT_IMAGES +app.state.config.PDF_LOADER_MODE = PDF_LOADER_MODE + +app.state.config.YOUTUBE_LOADER_LANGUAGE = YOUTUBE_LOADER_LANGUAGE +app.state.config.YOUTUBE_LOADER_PROXY_URL = YOUTUBE_LOADER_PROXY_URL + + +app.state.config.ENABLE_WEB_SEARCH = ENABLE_WEB_SEARCH +app.state.config.WEB_SEARCH_ENGINE = WEB_SEARCH_ENGINE +app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST = WEB_SEARCH_DOMAIN_FILTER_LIST +app.state.config.WEB_SEARCH_RESULT_COUNT = WEB_SEARCH_RESULT_COUNT +app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = WEB_SEARCH_CONCURRENT_REQUESTS +app.state.config.WEB_FETCH_MAX_CONTENT_LENGTH = WEB_FETCH_MAX_CONTENT_LENGTH + +app.state.config.WEB_LOADER_ENGINE = WEB_LOADER_ENGINE +app.state.config.WEB_LOADER_CONCURRENT_REQUESTS = WEB_LOADER_CONCURRENT_REQUESTS +app.state.config.WEB_LOADER_TIMEOUT = WEB_LOADER_TIMEOUT + +app.state.config.WEB_SEARCH_TRUST_ENV = WEB_SEARCH_TRUST_ENV +app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL +app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER = BYPASS_WEB_SEARCH_WEB_LOADER + +app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION +app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION + +app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = OLLAMA_CLOUD_WEB_SEARCH_API_KEY +app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL +app.state.config.SEARXNG_LANGUAGE = SEARXNG_LANGUAGE +app.state.config.YACY_QUERY_URL = YACY_QUERY_URL +app.state.config.YACY_USERNAME = YACY_USERNAME +app.state.config.YACY_PASSWORD = YACY_PASSWORD +app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY +app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID +app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY +app.state.config.BRAVE_SEARCH_CONTEXT_TOKENS = BRAVE_SEARCH_CONTEXT_TOKENS +app.state.config.KAGI_SEARCH_API_KEY = KAGI_SEARCH_API_KEY +app.state.config.MOJEEK_SEARCH_API_KEY = MOJEEK_SEARCH_API_KEY +app.state.config.BOCHA_SEARCH_API_KEY = BOCHA_SEARCH_API_KEY +app.state.config.SERPSTACK_API_KEY = SERPSTACK_API_KEY +app.state.config.SERPSTACK_HTTPS = SERPSTACK_HTTPS +app.state.config.SERPER_API_KEY = SERPER_API_KEY +app.state.config.SERPLY_API_KEY = SERPLY_API_KEY +app.state.config.DDGS_BACKEND = DDGS_BACKEND +app.state.config.TAVILY_API_KEY = TAVILY_API_KEY +app.state.config.SEARCHAPI_API_KEY = SEARCHAPI_API_KEY +app.state.config.SEARCHAPI_ENGINE = SEARCHAPI_ENGINE +app.state.config.SERPAPI_API_KEY = SERPAPI_API_KEY +app.state.config.SERPAPI_ENGINE = SERPAPI_ENGINE +app.state.config.JINA_API_KEY = JINA_API_KEY +app.state.config.JINA_API_BASE_URL = JINA_API_BASE_URL +app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT +app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY +app.state.config.EXA_API_KEY = EXA_API_KEY +app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY +app.state.config.PERPLEXITY_MODEL = PERPLEXITY_MODEL +app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE = PERPLEXITY_SEARCH_CONTEXT_USAGE +app.state.config.PERPLEXITY_SEARCH_API_URL = PERPLEXITY_SEARCH_API_URL +app.state.config.SOUGOU_API_SID = SOUGOU_API_SID +app.state.config.SOUGOU_API_SK = SOUGOU_API_SK +app.state.config.EXTERNAL_WEB_SEARCH_URL = EXTERNAL_WEB_SEARCH_URL +app.state.config.EXTERNAL_WEB_SEARCH_API_KEY = EXTERNAL_WEB_SEARCH_API_KEY +app.state.config.EXTERNAL_WEB_LOADER_URL = EXTERNAL_WEB_LOADER_URL +app.state.config.EXTERNAL_WEB_LOADER_API_KEY = EXTERNAL_WEB_LOADER_API_KEY +app.state.config.YANDEX_WEB_SEARCH_URL = YANDEX_WEB_SEARCH_URL +app.state.config.YANDEX_WEB_SEARCH_API_KEY = YANDEX_WEB_SEARCH_API_KEY +app.state.config.YANDEX_WEB_SEARCH_CONFIG = YANDEX_WEB_SEARCH_CONFIG +app.state.config.YOUCOM_API_KEY = YOUCOM_API_KEY +app.state.config.LINKUP_API_KEY = LINKUP_API_KEY +app.state.config.LINKUP_SEARCH_PARAMS = LINKUP_SEARCH_PARAMS + + +app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL +app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT +app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL +app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY +app.state.config.FIRECRAWL_TIMEOUT = FIRECRAWL_TIMEOUT +app.state.config.CRW_API_BASE_URL = CRW_API_BASE_URL +app.state.config.CRW_API_KEY = CRW_API_KEY +app.state.config.CRW_TIMEOUT = CRW_TIMEOUT +app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH + +app.state.EMBEDDING_FUNCTION = None +app.state.RERANKING_FUNCTION = None +app.state.ef = None +app.state.rf = None + +app.state.YOUTUBE_LOADER_TRANSLATION = None diff --git a/backend/open_webui/retrieval/web/crw.py b/backend/open_webui/retrieval/web/crw.py new file mode 100644 index 0000000000..d4909deaf7 --- /dev/null +++ b/backend/open_webui/retrieval/web/crw.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING, Any + +import requests +from langchain_core.documents import Document + +if TYPE_CHECKING: + from open_webui.retrieval.web.main import SearchResult + +log = logging.getLogger(__name__) + +# fastCRW is a Firecrawl-compatible web scraper shipped as a single Rust binary; +# self-host (AGPL open core) or use the managed cloud at https://fastcrw.com. +# Because the REST API is Firecrawl-compatible, this provider mirrors the +# Firecrawl provider closely and simply targets fastCRW's base URL and routes. +DEFAULT_CRW_API_BASE_URL = 'https://fastcrw.com/api' +CRW_RETRY_STATUS_CODES = {429, 500, 502, 503, 504} +CRW_MAX_RETRIES = 2 + + +def build_crw_url(base_url: str | None, path: str) -> str: + base_url = (base_url or DEFAULT_CRW_API_BASE_URL).rstrip('/') + path = path.lstrip('/') + + if base_url.endswith('/v1'): + return f'{base_url}/{path}' + + return f'{base_url}/v1/{path}' + + +def build_crw_headers(api_key: str | None) -> dict[str, str]: + headers = {'Content-Type': 'application/json'} + # Self-hosted fastCRW runs keyless by default; only send auth when a key is set. + if api_key: + headers['Authorization'] = f'Bearer {api_key}' + return headers + + +def get_crw_timeout_seconds(timeout: Any) -> float | None: + if timeout in (None, ''): + return None + + try: + timeout = float(timeout) + except (TypeError, ValueError): + return None + + return timeout if timeout > 0 else None + + +def get_crw_scrape_timeout_ms(timeout: Any) -> int | None: + timeout_seconds = get_crw_timeout_seconds(timeout) + if timeout_seconds is None: + return None + + return min(300000, max(1000, int(timeout_seconds * 1000))) + + +def get_crw_client_timeout_seconds(timeout: Any, fallback: float = 60) -> float: + # Keep the local HTTP timeout slightly above fastCRW's scrape timeout. + return (get_crw_timeout_seconds(timeout) or fallback) + 10 + + +def get_crw_retry_delay(headers: Any, attempt: int) -> float: + retry_after = headers.get('Retry-After') if headers else None + if retry_after: + try: + return min(10.0, max(0.0, float(retry_after))) + except (TypeError, ValueError): + pass + + return min(8.0, float(2**attempt)) + + +def request_crw_json( + method: str, + url: str, + *, + headers: dict[str, str], + json: dict[str, Any] | None = None, + timeout: float | None = None, + verify: bool = True, +) -> dict[str, Any]: + last_error = None + + for attempt in range(CRW_MAX_RETRIES + 1): + try: + response = requests.request( + method, + url, + headers=headers, + json=json, + timeout=timeout, + verify=verify, + ) + + if response.status_code in CRW_RETRY_STATUS_CODES and attempt < CRW_MAX_RETRIES: + delay = get_crw_retry_delay(response.headers, attempt) + log.warning( + 'fastCRW %s %s returned HTTP %s; retrying in %.1fs', + method, + url, + response.status_code, + delay, + ) + time.sleep(delay) + continue + + response.raise_for_status() + return response.json() + except (requests.ConnectionError, requests.Timeout) as e: + last_error = e + if attempt >= CRW_MAX_RETRIES: + break + + delay = get_crw_retry_delay(None, attempt) + log.warning('fastCRW %s %s failed; retrying in %.1fs: %s', method, url, delay, e) + time.sleep(delay) + + if last_error: + raise last_error + + raise RuntimeError(f'fastCRW {method} {url} failed without a response') + + +def get_crw_result_url(result: dict[str, Any]) -> str: + metadata = result.get('metadata') or {} + return ( + result.get('url') + or result.get('link') + or metadata.get('url') + or metadata.get('sourceURL') + or metadata.get('source_url') + or '' + ) + + +def scrape_crw_url( + crw_url: str, + crw_api_key: str, + url: str, + *, + verify_ssl: bool = True, + timeout: Any = None, + params: dict[str, Any] | None = None, +) -> Document | None: + payload = { + 'url': url, + 'formats': ['markdown'], + 'onlyMainContent': True, + **(params or {}), + } + scrape_timeout_ms = get_crw_scrape_timeout_ms(timeout) + if scrape_timeout_ms is not None: + payload['timeout'] = scrape_timeout_ms + + response = request_crw_json( + 'POST', + build_crw_url(crw_url, 'scrape'), + headers=build_crw_headers(crw_api_key), + json=payload, + timeout=get_crw_client_timeout_seconds(timeout), + verify=verify_ssl, + ) + data = response.get('data') or {} + content = data.get('markdown') or '' + if not isinstance(content, str) or not content.strip(): + return None + + metadata = data.get('metadata') or {} + document_metadata = {'source': get_crw_result_url(data) or url} + if metadata.get('title'): + document_metadata['title'] = metadata['title'] + if metadata.get('description'): + document_metadata['description'] = metadata['description'] + + return Document(page_content=content, metadata=document_metadata) + + +def search_crw( + crw_url: str, + crw_api_key: str, + query: str, + count: int, + filter_list: list[str] | None = None, +) -> list[SearchResult]: + try: + response = request_crw_json( + 'POST', + build_crw_url(crw_url, 'search'), + headers=build_crw_headers(crw_api_key), + json={ + 'query': query, + 'limit': count, + }, + timeout=count * 3 + 10, + ) + # fastCRW's Firecrawl-compatible /search returns a flat + # `{"data": [{title, url, description}]}` list. + data = response.get('data') or {} + results = data if isinstance(data, list) else (data.get('web') or []) + + if filter_list: + from open_webui.retrieval.web.main import get_filtered_results + + results = get_filtered_results(results, filter_list) + + from open_webui.retrieval.web.main import SearchResult + + search_results = [] + for result in results[:count]: + url = get_crw_result_url(result) + if not url: + continue + + metadata = result.get('metadata') or {} + search_results.append( + SearchResult( + link=url, + title=result.get('title') or metadata.get('title'), + snippet=result.get('description') or result.get('snippet') or metadata.get('description'), + ) + ) + + log.info(f'fastCRW search results: {search_results}') + return search_results + except Exception as e: + log.error(f'Error in fastCRW search: {e}') + return [] diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index eed8156023..f3fa20ba57 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -30,6 +30,9 @@ from langchain_community.document_loaders import PlaywrightURLLoader, WebBaseLoa from langchain_community.document_loaders.base import BaseLoader from langchain_core.documents import Document from open_webui.config import ( + CRW_API_BASE_URL, + CRW_API_KEY, + CRW_TIMEOUT, ENABLE_RAG_LOCAL_WEB_FETCH, EXTERNAL_WEB_LOADER_API_KEY, EXTERNAL_WEB_LOADER_URL, @@ -53,6 +56,7 @@ from open_webui.env import ( ) from open_webui.retrieval.loaders.external_web import ExternalWebLoader from open_webui.retrieval.loaders.tavily import TavilyLoader +from open_webui.retrieval.web.crw import scrape_crw_url from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url from open_webui.utils.misc import is_host_allowed @@ -352,6 +356,73 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): raise e +class SafeCRWLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): + def __init__( + self, + web_paths, + verify_ssl: bool = True, + trust_env: bool = False, + requests_per_second: Optional[float] = None, + continue_on_failure: bool = True, + api_key: Optional[str] = None, + api_url: Optional[str] = None, + timeout: Optional[int] = None, + mode: Literal['crawl', 'scrape', 'map'] = 'scrape', + proxy: Optional[Dict[str, str]] = None, + params: Optional[Dict] = None, + ): + proxy_server = proxy.get('server') if proxy else None + if trust_env and not proxy_server: + env_proxies = urllib.request.getproxies() + env_proxy_server = env_proxies.get('https') or env_proxies.get('http') + if env_proxy_server: + if proxy: + proxy['server'] = env_proxy_server + else: + proxy = {'server': env_proxy_server} + self.web_paths = web_paths + self.verify_ssl = verify_ssl + self.requests_per_second = requests_per_second + self.last_request_time = None + self.trust_env = trust_env + self.continue_on_failure = continue_on_failure + self.api_key = api_key + self.api_url = (api_url or 'https://fastcrw.com/api').rstrip('/') + self.timeout = timeout + self.mode = mode + self.params = params or {} + + def lazy_load(self) -> Iterator[Document]: + try: + for url in self.web_paths: + doc = scrape_crw_url( + self.api_url, + self.api_key, + url, + verify_ssl=self.verify_ssl, + timeout=self.timeout, + params=self.params, + ) + if doc is not None: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from URLs with fastCRW: {e}') + else: + raise e + + async def alazy_load(self): + try: + docs = await run_in_threadpool(lambda: list(self.lazy_load())) + for doc in docs: + yield doc + except Exception as e: + if self.continue_on_failure: + log.warning(f'Error extracting content from URLs with fastCRW: {e}') + else: + raise e + + class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): def __init__( self, @@ -808,6 +879,16 @@ def get_web_loader( except ValueError: pass + if WEB_LOADER_ENGINE == 'crw': + WebLoaderClass = SafeCRWLoader + web_loader_args['api_key'] = CRW_API_KEY + web_loader_args['api_url'] = CRW_API_BASE_URL + if CRW_TIMEOUT: + try: + web_loader_args['timeout'] = int(CRW_TIMEOUT) + except ValueError: + pass + if WEB_LOADER_ENGINE == 'tavily': WebLoaderClass = SafeTavilyLoader web_loader_args['api_key'] = TAVILY_API_KEY @@ -831,5 +912,5 @@ def get_web_loader( else: raise ValueError( f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. ' - "Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'." + "Please set it to 'safe_web', 'playwright', 'firecrawl', 'crw', or 'tavily'." ) diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index 3269ad0e3a..5ddddb06f8 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -84,6 +84,7 @@ from open_webui.retrieval.web.bing import search_bing from open_webui.retrieval.web.bocha import search_bocha from open_webui.retrieval.web.brave import search_brave from open_webui.retrieval.web.brave_llm_context import search_brave_llm_context +from open_webui.retrieval.web.crw import search_crw from open_webui.retrieval.web.duckduckgo import search_duckduckgo from open_webui.retrieval.web.exa import search_exa from open_webui.retrieval.web.external import search_external @@ -260,6 +261,9 @@ RETRIEVAL_CONFIG_KEYS = { 'CHUNK_OVERLAP': 'rag.chunk_overlap', 'CHUNK_SIZE': 'rag.chunk_size', 'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine', + 'CRW_API_BASE_URL': 'rag.web.loader.crw_api_url', + 'CRW_API_KEY': 'rag.web.loader.crw_api_key', + 'CRW_TIMEOUT': 'rag.web.loader.crw_timeout', 'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config', 'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url', 'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key', @@ -726,6 +730,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): 'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY, 'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL, 'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT, + 'CRW_API_KEY': config.CRW_API_KEY, + 'CRW_API_BASE_URL': config.CRW_API_BASE_URL, + 'CRW_TIMEOUT': config.CRW_TIMEOUT, 'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH, 'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL, 'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY, @@ -797,6 +804,9 @@ class WebConfig(BaseModel): FIRECRAWL_API_KEY: str | None = None FIRECRAWL_API_BASE_URL: str | None = None FIRECRAWL_TIMEOUT: str | None = None + CRW_API_KEY: str | None = None + CRW_API_BASE_URL: str | None = None + CRW_TIMEOUT: str | None = None TAVILY_EXTRACT_DEPTH: str | None = None EXTERNAL_WEB_SEARCH_URL: str | None = None EXTERNAL_WEB_SEARCH_API_KEY: str | None = None @@ -1293,6 +1303,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend config.FIRECRAWL_API_KEY = form_data.web.FIRECRAWL_API_KEY config.FIRECRAWL_API_BASE_URL = form_data.web.FIRECRAWL_API_BASE_URL config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT + config.CRW_API_KEY = form_data.web.CRW_API_KEY + config.CRW_API_BASE_URL = form_data.web.CRW_API_BASE_URL + config.CRW_TIMEOUT = form_data.web.CRW_TIMEOUT config.EXTERNAL_WEB_SEARCH_URL = form_data.web.EXTERNAL_WEB_SEARCH_URL config.EXTERNAL_WEB_SEARCH_API_KEY = form_data.web.EXTERNAL_WEB_SEARCH_API_KEY config.EXTERNAL_WEB_LOADER_URL = form_data.web.EXTERNAL_WEB_LOADER_URL @@ -1427,6 +1440,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend 'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY, 'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL, 'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT, + 'CRW_API_KEY': config.CRW_API_KEY, + 'CRW_API_BASE_URL': config.CRW_API_BASE_URL, + 'CRW_TIMEOUT': config.CRW_TIMEOUT, 'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH, 'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL, 'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY, @@ -2357,6 +2373,15 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li config.WEB_SEARCH_RESULT_COUNT, config.WEB_SEARCH_DOMAIN_FILTER_LIST, ) + elif engine == 'crw': + return await asyncio.to_thread( + search_crw, + config.CRW_API_BASE_URL, + config.CRW_API_KEY, + query, + config.WEB_SEARCH_RESULT_COUNT, + config.WEB_SEARCH_DOMAIN_FILTER_LIST, + ) elif engine == 'external': return await asyncio.to_thread( search_external, diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 4f9929dc60..76ddb4e8c6 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -37,12 +37,13 @@ 'perplexity', 'sougou', 'firecrawl', + 'crw', 'external', 'yandex', 'youcom', 'linkup' ]; - let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external']; + let webLoaderEngines = ['playwright', 'firecrawl', 'crw', 'tavily', 'external']; let webConfig = null; @@ -755,6 +756,55 @@ + {:else if webConfig.WEB_SEARCH_ENGINE === 'crw'} +
+
+
+ {$i18n.t('fastCRW API Base URL')} +
+ +
+
+ +
+
+
+ +
+
+ {$i18n.t('fastCRW API Key')} +
+ + +
+ +
+
+ {$i18n.t('fastCRW Timeout (s)')} +
+ +
+
+ +
+
+
+
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}
@@ -1145,6 +1195,37 @@ />
+ {:else if webConfig.WEB_LOADER_ENGINE === 'crw' && webConfig.WEB_SEARCH_ENGINE !== 'crw'} +
+
+
+ {$i18n.t('fastCRW API Base URL')} +
+ +
+
+ +
+
+
+ +
+
+ {$i18n.t('fastCRW API Key')} +
+ + +
+
{:else if webConfig.WEB_LOADER_ENGINE === 'tavily'}
diff --git a/src/lib/i18n/locales/en-US/translation.json b/src/lib/i18n/locales/en-US/translation.json index b41552679d..f583ef3ecf 100644 --- a/src/lib/i18n/locales/en-US/translation.json +++ b/src/lib/i18n/locales/en-US/translation.json @@ -771,6 +771,9 @@ "Enter Firecrawl API Base URL": "", "Enter Firecrawl API Key": "", "Enter Firecrawl Timeout": "", + "Enter fastCRW API Base URL": "", + "Enter fastCRW API Key": "", + "Enter fastCRW Timeout": "", "Enter folder name": "", "Enter function name filter list (e.g. func1, !func2)": "", "Enter Github Raw URL": "", @@ -959,6 +962,9 @@ "Failed to update settings": "", "Failed to update status": "", "Failed to upload file.": "", + "fastCRW API Base URL": "", + "fastCRW API Key": "", + "fastCRW Timeout (s)": "", "Features": "", "Features Permissions": "", "February": "",