mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-06 02:48:04 +00:00
feat: add fastCRW web search engine (rebased on dev, keyless self-host)
This commit is contained in:
us
parent
0ffc04797f
commit
63d010ed7a
7 changed files with 624 additions and 2 deletions
|
|
@ -1171,6 +1171,12 @@ FIRECRAWL_API_BASE_URL = os.getenv('FIRECRAWL_API_BASE_URL', 'https://api.firecr
|
|||
|
||||
FIRECRAWL_TIMEOUT = os.getenv('FIRECRAWL_TIMEOUT', '')
|
||||
|
||||
CRW_API_KEY = os.getenv('CRW_API_KEY', '')
|
||||
|
||||
CRW_API_BASE_URL = os.getenv('CRW_API_BASE_URL', 'https://fastcrw.com/api')
|
||||
|
||||
CRW_TIMEOUT = os.getenv('CRW_TIMEOUT', '')
|
||||
|
||||
EXTERNAL_WEB_SEARCH_URL = os.getenv('EXTERNAL_WEB_SEARCH_URL', '')
|
||||
|
||||
EXTERNAL_WEB_SEARCH_API_KEY = os.getenv('EXTERNAL_WEB_SEARCH_API_KEY', '')
|
||||
|
|
|
|||
|
|
@ -53,6 +53,24 @@ from open_webui.config import (
|
|||
ENABLE_ONEDRIVE_PERSONAL,
|
||||
# OpenAI
|
||||
ENV,
|
||||
EVALUATION_ARENA_MODELS,
|
||||
EXA_API_KEY,
|
||||
EXTERNAL_DOCUMENT_LOADER_API_KEY,
|
||||
EXTERNAL_DOCUMENT_LOADER_URL,
|
||||
EXTERNAL_WEB_LOADER_API_KEY,
|
||||
EXTERNAL_WEB_LOADER_URL,
|
||||
EXTERNAL_WEB_SEARCH_API_KEY,
|
||||
EXTERNAL_WEB_SEARCH_URL,
|
||||
FILE_IMAGE_COMPRESSION_HEIGHT,
|
||||
FILE_IMAGE_COMPRESSION_WIDTH,
|
||||
FIRECRAWL_API_BASE_URL,
|
||||
FIRECRAWL_API_KEY,
|
||||
FIRECRAWL_TIMEOUT,
|
||||
CRW_API_BASE_URL,
|
||||
CRW_API_KEY,
|
||||
CRW_TIMEOUT,
|
||||
FOLDER_MAX_FILE_COUNT,
|
||||
FOLLOW_UP_GENERATION_PROMPT_TEMPLATE,
|
||||
FRONTEND_BUILD_DIR,
|
||||
GOOGLE_DRIVE_API_KEY,
|
||||
GOOGLE_DRIVE_CLIENT_ID,
|
||||
|
|
@ -509,6 +527,179 @@ app.state.BASE_MODELS = []
|
|||
|
||||
|
||||
|
||||
app.state.config.TOP_K = RAG_TOP_K
|
||||
app.state.config.TOP_K_RERANKER = RAG_TOP_K_RERANKER
|
||||
app.state.config.RELEVANCE_THRESHOLD = RAG_RELEVANCE_THRESHOLD
|
||||
app.state.config.HYBRID_BM25_WEIGHT = RAG_HYBRID_BM25_WEIGHT
|
||||
|
||||
|
||||
app.state.config.ALLOWED_FILE_EXTENSIONS = RAG_ALLOWED_FILE_EXTENSIONS
|
||||
app.state.config.FILE_MAX_SIZE = RAG_FILE_MAX_SIZE
|
||||
app.state.config.FILE_MAX_COUNT = RAG_FILE_MAX_COUNT
|
||||
app.state.config.FILE_IMAGE_COMPRESSION_WIDTH = FILE_IMAGE_COMPRESSION_WIDTH
|
||||
app.state.config.FILE_IMAGE_COMPRESSION_HEIGHT = FILE_IMAGE_COMPRESSION_HEIGHT
|
||||
|
||||
|
||||
app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT
|
||||
app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL
|
||||
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
|
||||
app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS
|
||||
app.state.config.ENABLE_WEB_LOADER_SSL_VERIFICATION = ENABLE_WEB_LOADER_SSL_VERIFICATION
|
||||
|
||||
app.state.config.CONTENT_EXTRACTION_ENGINE = CONTENT_EXTRACTION_ENGINE
|
||||
app.state.config.DATALAB_MARKER_API_KEY = DATALAB_MARKER_API_KEY
|
||||
app.state.config.DATALAB_MARKER_API_BASE_URL = DATALAB_MARKER_API_BASE_URL
|
||||
app.state.config.DATALAB_MARKER_ADDITIONAL_CONFIG = DATALAB_MARKER_ADDITIONAL_CONFIG
|
||||
app.state.config.DATALAB_MARKER_SKIP_CACHE = DATALAB_MARKER_SKIP_CACHE
|
||||
app.state.config.DATALAB_MARKER_FORCE_OCR = DATALAB_MARKER_FORCE_OCR
|
||||
app.state.config.DATALAB_MARKER_PAGINATE = DATALAB_MARKER_PAGINATE
|
||||
app.state.config.DATALAB_MARKER_STRIP_EXISTING_OCR = DATALAB_MARKER_STRIP_EXISTING_OCR
|
||||
app.state.config.DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION = DATALAB_MARKER_DISABLE_IMAGE_EXTRACTION
|
||||
app.state.config.DATALAB_MARKER_FORMAT_LINES = DATALAB_MARKER_FORMAT_LINES
|
||||
app.state.config.DATALAB_MARKER_USE_LLM = DATALAB_MARKER_USE_LLM
|
||||
app.state.config.DATALAB_MARKER_OUTPUT_FORMAT = DATALAB_MARKER_OUTPUT_FORMAT
|
||||
app.state.config.EXTERNAL_DOCUMENT_LOADER_URL = EXTERNAL_DOCUMENT_LOADER_URL
|
||||
app.state.config.EXTERNAL_DOCUMENT_LOADER_API_KEY = EXTERNAL_DOCUMENT_LOADER_API_KEY
|
||||
app.state.config.TIKA_SERVER_URL = TIKA_SERVER_URL
|
||||
app.state.config.DOCLING_SERVER_URL = DOCLING_SERVER_URL
|
||||
app.state.config.DOCLING_API_KEY = DOCLING_API_KEY
|
||||
app.state.config.DOCLING_PARAMS = DOCLING_PARAMS
|
||||
app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT = DOCUMENT_INTELLIGENCE_ENDPOINT
|
||||
app.state.config.DOCUMENT_INTELLIGENCE_KEY = DOCUMENT_INTELLIGENCE_KEY
|
||||
app.state.config.DOCUMENT_INTELLIGENCE_MODEL = DOCUMENT_INTELLIGENCE_MODEL
|
||||
app.state.config.MISTRAL_OCR_API_BASE_URL = MISTRAL_OCR_API_BASE_URL
|
||||
app.state.config.MISTRAL_OCR_API_KEY = MISTRAL_OCR_API_KEY
|
||||
app.state.config.PADDLEOCR_VL_BASE_URL = PADDLEOCR_VL_BASE_URL
|
||||
app.state.config.PADDLEOCR_VL_TOKEN = PADDLEOCR_VL_TOKEN
|
||||
app.state.config.MINERU_API_MODE = MINERU_API_MODE
|
||||
app.state.config.MINERU_API_URL = MINERU_API_URL
|
||||
app.state.config.MINERU_API_KEY = MINERU_API_KEY
|
||||
app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT
|
||||
app.state.config.MINERU_PARAMS = MINERU_PARAMS
|
||||
app.state.config.MINERU_FILE_EXTENSIONS = MINERU_FILE_EXTENSIONS
|
||||
|
||||
app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
|
||||
app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER
|
||||
|
||||
app.state.config.TIKTOKEN_ENCODING_NAME = TIKTOKEN_ENCODING_NAME
|
||||
|
||||
app.state.config.CHUNK_SIZE = CHUNK_SIZE
|
||||
app.state.config.CHUNK_MIN_SIZE_TARGET = CHUNK_MIN_SIZE_TARGET
|
||||
app.state.config.CHUNK_OVERLAP = CHUNK_OVERLAP
|
||||
|
||||
|
||||
app.state.config.RAG_EMBEDDING_ENGINE = RAG_EMBEDDING_ENGINE
|
||||
app.state.config.RAG_EMBEDDING_MODEL = RAG_EMBEDDING_MODEL
|
||||
app.state.config.RAG_EMBEDDING_BATCH_SIZE = RAG_EMBEDDING_BATCH_SIZE
|
||||
app.state.config.ENABLE_ASYNC_EMBEDDING = ENABLE_ASYNC_EMBEDDING
|
||||
app.state.config.RAG_EMBEDDING_CONCURRENT_REQUESTS = RAG_EMBEDDING_CONCURRENT_REQUESTS
|
||||
|
||||
app.state.config.RAG_RERANKING_ENGINE = RAG_RERANKING_ENGINE
|
||||
app.state.config.RAG_RERANKING_MODEL = RAG_RERANKING_MODEL
|
||||
app.state.config.RAG_EXTERNAL_RERANKER_URL = RAG_EXTERNAL_RERANKER_URL
|
||||
app.state.config.RAG_EXTERNAL_RERANKER_API_KEY = RAG_EXTERNAL_RERANKER_API_KEY
|
||||
app.state.config.RAG_EXTERNAL_RERANKER_TIMEOUT = RAG_EXTERNAL_RERANKER_TIMEOUT
|
||||
app.state.config.RAG_RERANKING_BATCH_SIZE = RAG_RERANKING_BATCH_SIZE
|
||||
|
||||
app.state.config.RAG_TEMPLATE = RAG_TEMPLATE
|
||||
|
||||
app.state.config.RAG_OPENAI_API_BASE_URL = RAG_OPENAI_API_BASE_URL
|
||||
app.state.config.RAG_OPENAI_API_KEY = RAG_OPENAI_API_KEY
|
||||
|
||||
app.state.config.RAG_AZURE_OPENAI_BASE_URL = RAG_AZURE_OPENAI_BASE_URL
|
||||
app.state.config.RAG_AZURE_OPENAI_API_KEY = RAG_AZURE_OPENAI_API_KEY
|
||||
app.state.config.RAG_AZURE_OPENAI_API_VERSION = RAG_AZURE_OPENAI_API_VERSION
|
||||
|
||||
app.state.config.RAG_OLLAMA_BASE_URL = RAG_OLLAMA_BASE_URL
|
||||
app.state.config.RAG_OLLAMA_API_KEY = RAG_OLLAMA_API_KEY
|
||||
|
||||
app.state.config.PDF_EXTRACT_IMAGES = PDF_EXTRACT_IMAGES
|
||||
app.state.config.PDF_LOADER_MODE = PDF_LOADER_MODE
|
||||
|
||||
app.state.config.YOUTUBE_LOADER_LANGUAGE = YOUTUBE_LOADER_LANGUAGE
|
||||
app.state.config.YOUTUBE_LOADER_PROXY_URL = YOUTUBE_LOADER_PROXY_URL
|
||||
|
||||
|
||||
app.state.config.ENABLE_WEB_SEARCH = ENABLE_WEB_SEARCH
|
||||
app.state.config.WEB_SEARCH_ENGINE = WEB_SEARCH_ENGINE
|
||||
app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST = WEB_SEARCH_DOMAIN_FILTER_LIST
|
||||
app.state.config.WEB_SEARCH_RESULT_COUNT = WEB_SEARCH_RESULT_COUNT
|
||||
app.state.config.WEB_SEARCH_CONCURRENT_REQUESTS = WEB_SEARCH_CONCURRENT_REQUESTS
|
||||
app.state.config.WEB_FETCH_MAX_CONTENT_LENGTH = WEB_FETCH_MAX_CONTENT_LENGTH
|
||||
|
||||
app.state.config.WEB_LOADER_ENGINE = WEB_LOADER_ENGINE
|
||||
app.state.config.WEB_LOADER_CONCURRENT_REQUESTS = WEB_LOADER_CONCURRENT_REQUESTS
|
||||
app.state.config.WEB_LOADER_TIMEOUT = WEB_LOADER_TIMEOUT
|
||||
|
||||
app.state.config.WEB_SEARCH_TRUST_ENV = WEB_SEARCH_TRUST_ENV
|
||||
app.state.config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL = BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL
|
||||
app.state.config.BYPASS_WEB_SEARCH_WEB_LOADER = BYPASS_WEB_SEARCH_WEB_LOADER
|
||||
|
||||
app.state.config.ENABLE_GOOGLE_DRIVE_INTEGRATION = ENABLE_GOOGLE_DRIVE_INTEGRATION
|
||||
app.state.config.ENABLE_ONEDRIVE_INTEGRATION = ENABLE_ONEDRIVE_INTEGRATION
|
||||
|
||||
app.state.config.OLLAMA_CLOUD_WEB_SEARCH_API_KEY = OLLAMA_CLOUD_WEB_SEARCH_API_KEY
|
||||
app.state.config.SEARXNG_QUERY_URL = SEARXNG_QUERY_URL
|
||||
app.state.config.SEARXNG_LANGUAGE = SEARXNG_LANGUAGE
|
||||
app.state.config.YACY_QUERY_URL = YACY_QUERY_URL
|
||||
app.state.config.YACY_USERNAME = YACY_USERNAME
|
||||
app.state.config.YACY_PASSWORD = YACY_PASSWORD
|
||||
app.state.config.GOOGLE_PSE_API_KEY = GOOGLE_PSE_API_KEY
|
||||
app.state.config.GOOGLE_PSE_ENGINE_ID = GOOGLE_PSE_ENGINE_ID
|
||||
app.state.config.BRAVE_SEARCH_API_KEY = BRAVE_SEARCH_API_KEY
|
||||
app.state.config.BRAVE_SEARCH_CONTEXT_TOKENS = BRAVE_SEARCH_CONTEXT_TOKENS
|
||||
app.state.config.KAGI_SEARCH_API_KEY = KAGI_SEARCH_API_KEY
|
||||
app.state.config.MOJEEK_SEARCH_API_KEY = MOJEEK_SEARCH_API_KEY
|
||||
app.state.config.BOCHA_SEARCH_API_KEY = BOCHA_SEARCH_API_KEY
|
||||
app.state.config.SERPSTACK_API_KEY = SERPSTACK_API_KEY
|
||||
app.state.config.SERPSTACK_HTTPS = SERPSTACK_HTTPS
|
||||
app.state.config.SERPER_API_KEY = SERPER_API_KEY
|
||||
app.state.config.SERPLY_API_KEY = SERPLY_API_KEY
|
||||
app.state.config.DDGS_BACKEND = DDGS_BACKEND
|
||||
app.state.config.TAVILY_API_KEY = TAVILY_API_KEY
|
||||
app.state.config.SEARCHAPI_API_KEY = SEARCHAPI_API_KEY
|
||||
app.state.config.SEARCHAPI_ENGINE = SEARCHAPI_ENGINE
|
||||
app.state.config.SERPAPI_API_KEY = SERPAPI_API_KEY
|
||||
app.state.config.SERPAPI_ENGINE = SERPAPI_ENGINE
|
||||
app.state.config.JINA_API_KEY = JINA_API_KEY
|
||||
app.state.config.JINA_API_BASE_URL = JINA_API_BASE_URL
|
||||
app.state.config.BING_SEARCH_V7_ENDPOINT = BING_SEARCH_V7_ENDPOINT
|
||||
app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY = BING_SEARCH_V7_SUBSCRIPTION_KEY
|
||||
app.state.config.EXA_API_KEY = EXA_API_KEY
|
||||
app.state.config.PERPLEXITY_API_KEY = PERPLEXITY_API_KEY
|
||||
app.state.config.PERPLEXITY_MODEL = PERPLEXITY_MODEL
|
||||
app.state.config.PERPLEXITY_SEARCH_CONTEXT_USAGE = PERPLEXITY_SEARCH_CONTEXT_USAGE
|
||||
app.state.config.PERPLEXITY_SEARCH_API_URL = PERPLEXITY_SEARCH_API_URL
|
||||
app.state.config.SOUGOU_API_SID = SOUGOU_API_SID
|
||||
app.state.config.SOUGOU_API_SK = SOUGOU_API_SK
|
||||
app.state.config.EXTERNAL_WEB_SEARCH_URL = EXTERNAL_WEB_SEARCH_URL
|
||||
app.state.config.EXTERNAL_WEB_SEARCH_API_KEY = EXTERNAL_WEB_SEARCH_API_KEY
|
||||
app.state.config.EXTERNAL_WEB_LOADER_URL = EXTERNAL_WEB_LOADER_URL
|
||||
app.state.config.EXTERNAL_WEB_LOADER_API_KEY = EXTERNAL_WEB_LOADER_API_KEY
|
||||
app.state.config.YANDEX_WEB_SEARCH_URL = YANDEX_WEB_SEARCH_URL
|
||||
app.state.config.YANDEX_WEB_SEARCH_API_KEY = YANDEX_WEB_SEARCH_API_KEY
|
||||
app.state.config.YANDEX_WEB_SEARCH_CONFIG = YANDEX_WEB_SEARCH_CONFIG
|
||||
app.state.config.YOUCOM_API_KEY = YOUCOM_API_KEY
|
||||
app.state.config.LINKUP_API_KEY = LINKUP_API_KEY
|
||||
app.state.config.LINKUP_SEARCH_PARAMS = LINKUP_SEARCH_PARAMS
|
||||
|
||||
|
||||
app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL
|
||||
app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT
|
||||
app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL
|
||||
app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY
|
||||
app.state.config.FIRECRAWL_TIMEOUT = FIRECRAWL_TIMEOUT
|
||||
app.state.config.CRW_API_BASE_URL = CRW_API_BASE_URL
|
||||
app.state.config.CRW_API_KEY = CRW_API_KEY
|
||||
app.state.config.CRW_TIMEOUT = CRW_TIMEOUT
|
||||
app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH
|
||||
|
||||
app.state.EMBEDDING_FUNCTION = None
|
||||
app.state.RERANKING_FUNCTION = None
|
||||
app.state.ef = None
|
||||
app.state.rf = None
|
||||
|
||||
app.state.YOUTUBE_LOADER_TRANSLATION = None
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
232
backend/open_webui/retrieval/web/crw.py
Normal file
232
backend/open_webui/retrieval/web/crw.py
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import requests
|
||||
from langchain_core.documents import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from open_webui.retrieval.web.main import SearchResult
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# fastCRW is a Firecrawl-compatible web scraper shipped as a single Rust binary;
|
||||
# self-host (AGPL open core) or use the managed cloud at https://fastcrw.com.
|
||||
# Because the REST API is Firecrawl-compatible, this provider mirrors the
|
||||
# Firecrawl provider closely and simply targets fastCRW's base URL and routes.
|
||||
DEFAULT_CRW_API_BASE_URL = 'https://fastcrw.com/api'
|
||||
CRW_RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
CRW_MAX_RETRIES = 2
|
||||
|
||||
|
||||
def build_crw_url(base_url: str | None, path: str) -> str:
|
||||
base_url = (base_url or DEFAULT_CRW_API_BASE_URL).rstrip('/')
|
||||
path = path.lstrip('/')
|
||||
|
||||
if base_url.endswith('/v1'):
|
||||
return f'{base_url}/{path}'
|
||||
|
||||
return f'{base_url}/v1/{path}'
|
||||
|
||||
|
||||
def build_crw_headers(api_key: str | None) -> dict[str, str]:
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
# Self-hosted fastCRW runs keyless by default; only send auth when a key is set.
|
||||
if api_key:
|
||||
headers['Authorization'] = f'Bearer {api_key}'
|
||||
return headers
|
||||
|
||||
|
||||
def get_crw_timeout_seconds(timeout: Any) -> float | None:
|
||||
if timeout in (None, ''):
|
||||
return None
|
||||
|
||||
try:
|
||||
timeout = float(timeout)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
return timeout if timeout > 0 else None
|
||||
|
||||
|
||||
def get_crw_scrape_timeout_ms(timeout: Any) -> int | None:
|
||||
timeout_seconds = get_crw_timeout_seconds(timeout)
|
||||
if timeout_seconds is None:
|
||||
return None
|
||||
|
||||
return min(300000, max(1000, int(timeout_seconds * 1000)))
|
||||
|
||||
|
||||
def get_crw_client_timeout_seconds(timeout: Any, fallback: float = 60) -> float:
|
||||
# Keep the local HTTP timeout slightly above fastCRW's scrape timeout.
|
||||
return (get_crw_timeout_seconds(timeout) or fallback) + 10
|
||||
|
||||
|
||||
def get_crw_retry_delay(headers: Any, attempt: int) -> float:
|
||||
retry_after = headers.get('Retry-After') if headers else None
|
||||
if retry_after:
|
||||
try:
|
||||
return min(10.0, max(0.0, float(retry_after)))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
return min(8.0, float(2**attempt))
|
||||
|
||||
|
||||
def request_crw_json(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: dict[str, str],
|
||||
json: dict[str, Any] | None = None,
|
||||
timeout: float | None = None,
|
||||
verify: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
last_error = None
|
||||
|
||||
for attempt in range(CRW_MAX_RETRIES + 1):
|
||||
try:
|
||||
response = requests.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
json=json,
|
||||
timeout=timeout,
|
||||
verify=verify,
|
||||
)
|
||||
|
||||
if response.status_code in CRW_RETRY_STATUS_CODES and attempt < CRW_MAX_RETRIES:
|
||||
delay = get_crw_retry_delay(response.headers, attempt)
|
||||
log.warning(
|
||||
'fastCRW %s %s returned HTTP %s; retrying in %.1fs',
|
||||
method,
|
||||
url,
|
||||
response.status_code,
|
||||
delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
last_error = e
|
||||
if attempt >= CRW_MAX_RETRIES:
|
||||
break
|
||||
|
||||
delay = get_crw_retry_delay(None, attempt)
|
||||
log.warning('fastCRW %s %s failed; retrying in %.1fs: %s', method, url, delay, e)
|
||||
time.sleep(delay)
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
|
||||
raise RuntimeError(f'fastCRW {method} {url} failed without a response')
|
||||
|
||||
|
||||
def get_crw_result_url(result: dict[str, Any]) -> str:
|
||||
metadata = result.get('metadata') or {}
|
||||
return (
|
||||
result.get('url')
|
||||
or result.get('link')
|
||||
or metadata.get('url')
|
||||
or metadata.get('sourceURL')
|
||||
or metadata.get('source_url')
|
||||
or ''
|
||||
)
|
||||
|
||||
|
||||
def scrape_crw_url(
|
||||
crw_url: str,
|
||||
crw_api_key: str,
|
||||
url: str,
|
||||
*,
|
||||
verify_ssl: bool = True,
|
||||
timeout: Any = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> Document | None:
|
||||
payload = {
|
||||
'url': url,
|
||||
'formats': ['markdown'],
|
||||
'onlyMainContent': True,
|
||||
**(params or {}),
|
||||
}
|
||||
scrape_timeout_ms = get_crw_scrape_timeout_ms(timeout)
|
||||
if scrape_timeout_ms is not None:
|
||||
payload['timeout'] = scrape_timeout_ms
|
||||
|
||||
response = request_crw_json(
|
||||
'POST',
|
||||
build_crw_url(crw_url, 'scrape'),
|
||||
headers=build_crw_headers(crw_api_key),
|
||||
json=payload,
|
||||
timeout=get_crw_client_timeout_seconds(timeout),
|
||||
verify=verify_ssl,
|
||||
)
|
||||
data = response.get('data') or {}
|
||||
content = data.get('markdown') or ''
|
||||
if not isinstance(content, str) or not content.strip():
|
||||
return None
|
||||
|
||||
metadata = data.get('metadata') or {}
|
||||
document_metadata = {'source': get_crw_result_url(data) or url}
|
||||
if metadata.get('title'):
|
||||
document_metadata['title'] = metadata['title']
|
||||
if metadata.get('description'):
|
||||
document_metadata['description'] = metadata['description']
|
||||
|
||||
return Document(page_content=content, metadata=document_metadata)
|
||||
|
||||
|
||||
def search_crw(
|
||||
crw_url: str,
|
||||
crw_api_key: str,
|
||||
query: str,
|
||||
count: int,
|
||||
filter_list: list[str] | None = None,
|
||||
) -> list[SearchResult]:
|
||||
try:
|
||||
response = request_crw_json(
|
||||
'POST',
|
||||
build_crw_url(crw_url, 'search'),
|
||||
headers=build_crw_headers(crw_api_key),
|
||||
json={
|
||||
'query': query,
|
||||
'limit': count,
|
||||
},
|
||||
timeout=count * 3 + 10,
|
||||
)
|
||||
# fastCRW's Firecrawl-compatible /search returns a flat
|
||||
# `{"data": [{title, url, description}]}` list.
|
||||
data = response.get('data') or {}
|
||||
results = data if isinstance(data, list) else (data.get('web') or [])
|
||||
|
||||
if filter_list:
|
||||
from open_webui.retrieval.web.main import get_filtered_results
|
||||
|
||||
results = get_filtered_results(results, filter_list)
|
||||
|
||||
from open_webui.retrieval.web.main import SearchResult
|
||||
|
||||
search_results = []
|
||||
for result in results[:count]:
|
||||
url = get_crw_result_url(result)
|
||||
if not url:
|
||||
continue
|
||||
|
||||
metadata = result.get('metadata') or {}
|
||||
search_results.append(
|
||||
SearchResult(
|
||||
link=url,
|
||||
title=result.get('title') or metadata.get('title'),
|
||||
snippet=result.get('description') or result.get('snippet') or metadata.get('description'),
|
||||
)
|
||||
)
|
||||
|
||||
log.info(f'fastCRW search results: {search_results}')
|
||||
return search_results
|
||||
except Exception as e:
|
||||
log.error(f'Error in fastCRW search: {e}')
|
||||
return []
|
||||
|
|
@ -30,6 +30,9 @@ from langchain_community.document_loaders import PlaywrightURLLoader, WebBaseLoa
|
|||
from langchain_community.document_loaders.base import BaseLoader
|
||||
from langchain_core.documents import Document
|
||||
from open_webui.config import (
|
||||
CRW_API_BASE_URL,
|
||||
CRW_API_KEY,
|
||||
CRW_TIMEOUT,
|
||||
ENABLE_RAG_LOCAL_WEB_FETCH,
|
||||
EXTERNAL_WEB_LOADER_API_KEY,
|
||||
EXTERNAL_WEB_LOADER_URL,
|
||||
|
|
@ -53,6 +56,7 @@ from open_webui.env import (
|
|||
)
|
||||
from open_webui.retrieval.loaders.external_web import ExternalWebLoader
|
||||
from open_webui.retrieval.loaders.tavily import TavilyLoader
|
||||
from open_webui.retrieval.web.crw import scrape_crw_url
|
||||
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
|
||||
from open_webui.utils.misc import is_host_allowed
|
||||
|
||||
|
|
@ -352,6 +356,73 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||
raise e
|
||||
|
||||
|
||||
class SafeCRWLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
web_paths,
|
||||
verify_ssl: bool = True,
|
||||
trust_env: bool = False,
|
||||
requests_per_second: Optional[float] = None,
|
||||
continue_on_failure: bool = True,
|
||||
api_key: Optional[str] = None,
|
||||
api_url: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
mode: Literal['crawl', 'scrape', 'map'] = 'scrape',
|
||||
proxy: Optional[Dict[str, str]] = None,
|
||||
params: Optional[Dict] = None,
|
||||
):
|
||||
proxy_server = proxy.get('server') if proxy else None
|
||||
if trust_env and not proxy_server:
|
||||
env_proxies = urllib.request.getproxies()
|
||||
env_proxy_server = env_proxies.get('https') or env_proxies.get('http')
|
||||
if env_proxy_server:
|
||||
if proxy:
|
||||
proxy['server'] = env_proxy_server
|
||||
else:
|
||||
proxy = {'server': env_proxy_server}
|
||||
self.web_paths = web_paths
|
||||
self.verify_ssl = verify_ssl
|
||||
self.requests_per_second = requests_per_second
|
||||
self.last_request_time = None
|
||||
self.trust_env = trust_env
|
||||
self.continue_on_failure = continue_on_failure
|
||||
self.api_key = api_key
|
||||
self.api_url = (api_url or 'https://fastcrw.com/api').rstrip('/')
|
||||
self.timeout = timeout
|
||||
self.mode = mode
|
||||
self.params = params or {}
|
||||
|
||||
def lazy_load(self) -> Iterator[Document]:
|
||||
try:
|
||||
for url in self.web_paths:
|
||||
doc = scrape_crw_url(
|
||||
self.api_url,
|
||||
self.api_key,
|
||||
url,
|
||||
verify_ssl=self.verify_ssl,
|
||||
timeout=self.timeout,
|
||||
params=self.params,
|
||||
)
|
||||
if doc is not None:
|
||||
yield doc
|
||||
except Exception as e:
|
||||
if self.continue_on_failure:
|
||||
log.warning(f'Error extracting content from URLs with fastCRW: {e}')
|
||||
else:
|
||||
raise e
|
||||
|
||||
async def alazy_load(self):
|
||||
try:
|
||||
docs = await run_in_threadpool(lambda: list(self.lazy_load()))
|
||||
for doc in docs:
|
||||
yield doc
|
||||
except Exception as e:
|
||||
if self.continue_on_failure:
|
||||
log.warning(f'Error extracting content from URLs with fastCRW: {e}')
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -808,6 +879,16 @@ def get_web_loader(
|
|||
except ValueError:
|
||||
pass
|
||||
|
||||
if WEB_LOADER_ENGINE == 'crw':
|
||||
WebLoaderClass = SafeCRWLoader
|
||||
web_loader_args['api_key'] = CRW_API_KEY
|
||||
web_loader_args['api_url'] = CRW_API_BASE_URL
|
||||
if CRW_TIMEOUT:
|
||||
try:
|
||||
web_loader_args['timeout'] = int(CRW_TIMEOUT)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if WEB_LOADER_ENGINE == 'tavily':
|
||||
WebLoaderClass = SafeTavilyLoader
|
||||
web_loader_args['api_key'] = TAVILY_API_KEY
|
||||
|
|
@ -831,5 +912,5 @@ def get_web_loader(
|
|||
else:
|
||||
raise ValueError(
|
||||
f'Invalid WEB_LOADER_ENGINE: {WEB_LOADER_ENGINE}. '
|
||||
"Please set it to 'safe_web', 'playwright', 'firecrawl', or 'tavily'."
|
||||
"Please set it to 'safe_web', 'playwright', 'firecrawl', 'crw', or 'tavily'."
|
||||
)
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ from open_webui.retrieval.web.bing import search_bing
|
|||
from open_webui.retrieval.web.bocha import search_bocha
|
||||
from open_webui.retrieval.web.brave import search_brave
|
||||
from open_webui.retrieval.web.brave_llm_context import search_brave_llm_context
|
||||
from open_webui.retrieval.web.crw import search_crw
|
||||
from open_webui.retrieval.web.duckduckgo import search_duckduckgo
|
||||
from open_webui.retrieval.web.exa import search_exa
|
||||
from open_webui.retrieval.web.external import search_external
|
||||
|
|
@ -260,6 +261,9 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||
'CHUNK_OVERLAP': 'rag.chunk_overlap',
|
||||
'CHUNK_SIZE': 'rag.chunk_size',
|
||||
'CONTENT_EXTRACTION_ENGINE': 'rag.content_extraction_engine',
|
||||
'CRW_API_BASE_URL': 'rag.web.loader.crw_api_url',
|
||||
'CRW_API_KEY': 'rag.web.loader.crw_api_key',
|
||||
'CRW_TIMEOUT': 'rag.web.loader.crw_timeout',
|
||||
'DATALAB_MARKER_ADDITIONAL_CONFIG': 'rag.datalab_marker_additional_config',
|
||||
'DATALAB_MARKER_API_BASE_URL': 'rag.datalab_marker_api_base_url',
|
||||
'DATALAB_MARKER_API_KEY': 'rag.datalab_marker_api_key',
|
||||
|
|
@ -726,6 +730,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||
'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY,
|
||||
'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL,
|
||||
'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT,
|
||||
'CRW_API_KEY': config.CRW_API_KEY,
|
||||
'CRW_API_BASE_URL': config.CRW_API_BASE_URL,
|
||||
'CRW_TIMEOUT': config.CRW_TIMEOUT,
|
||||
'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH,
|
||||
'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL,
|
||||
'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY,
|
||||
|
|
@ -797,6 +804,9 @@ class WebConfig(BaseModel):
|
|||
FIRECRAWL_API_KEY: str | None = None
|
||||
FIRECRAWL_API_BASE_URL: str | None = None
|
||||
FIRECRAWL_TIMEOUT: str | None = None
|
||||
CRW_API_KEY: str | None = None
|
||||
CRW_API_BASE_URL: str | None = None
|
||||
CRW_TIMEOUT: str | None = None
|
||||
TAVILY_EXTRACT_DEPTH: str | None = None
|
||||
EXTERNAL_WEB_SEARCH_URL: str | None = None
|
||||
EXTERNAL_WEB_SEARCH_API_KEY: str | None = None
|
||||
|
|
@ -1293,6 +1303,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||
config.FIRECRAWL_API_KEY = form_data.web.FIRECRAWL_API_KEY
|
||||
config.FIRECRAWL_API_BASE_URL = form_data.web.FIRECRAWL_API_BASE_URL
|
||||
config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT
|
||||
config.CRW_API_KEY = form_data.web.CRW_API_KEY
|
||||
config.CRW_API_BASE_URL = form_data.web.CRW_API_BASE_URL
|
||||
config.CRW_TIMEOUT = form_data.web.CRW_TIMEOUT
|
||||
config.EXTERNAL_WEB_SEARCH_URL = form_data.web.EXTERNAL_WEB_SEARCH_URL
|
||||
config.EXTERNAL_WEB_SEARCH_API_KEY = form_data.web.EXTERNAL_WEB_SEARCH_API_KEY
|
||||
config.EXTERNAL_WEB_LOADER_URL = form_data.web.EXTERNAL_WEB_LOADER_URL
|
||||
|
|
@ -1427,6 +1440,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||
'FIRECRAWL_API_KEY': config.FIRECRAWL_API_KEY,
|
||||
'FIRECRAWL_API_BASE_URL': config.FIRECRAWL_API_BASE_URL,
|
||||
'FIRECRAWL_TIMEOUT': config.FIRECRAWL_TIMEOUT,
|
||||
'CRW_API_KEY': config.CRW_API_KEY,
|
||||
'CRW_API_BASE_URL': config.CRW_API_BASE_URL,
|
||||
'CRW_TIMEOUT': config.CRW_TIMEOUT,
|
||||
'TAVILY_EXTRACT_DEPTH': config.TAVILY_EXTRACT_DEPTH,
|
||||
'EXTERNAL_WEB_SEARCH_URL': config.EXTERNAL_WEB_SEARCH_URL,
|
||||
'EXTERNAL_WEB_SEARCH_API_KEY': config.EXTERNAL_WEB_SEARCH_API_KEY,
|
||||
|
|
@ -2357,6 +2373,15 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li
|
|||
config.WEB_SEARCH_RESULT_COUNT,
|
||||
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||
)
|
||||
elif engine == 'crw':
|
||||
return await asyncio.to_thread(
|
||||
search_crw,
|
||||
config.CRW_API_BASE_URL,
|
||||
config.CRW_API_KEY,
|
||||
query,
|
||||
config.WEB_SEARCH_RESULT_COUNT,
|
||||
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||
)
|
||||
elif engine == 'external':
|
||||
return await asyncio.to_thread(
|
||||
search_external,
|
||||
|
|
|
|||
|
|
@ -37,12 +37,13 @@
|
|||
'perplexity',
|
||||
'sougou',
|
||||
'firecrawl',
|
||||
'crw',
|
||||
'external',
|
||||
'yandex',
|
||||
'youcom',
|
||||
'linkup'
|
||||
];
|
||||
let webLoaderEngines = ['playwright', 'firecrawl', 'tavily', 'external'];
|
||||
let webLoaderEngines = ['playwright', 'firecrawl', 'crw', 'tavily', 'external'];
|
||||
|
||||
let webConfig = null;
|
||||
|
||||
|
|
@ -755,6 +756,55 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'crw'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
<div class=" self-center text-xs font-medium mb-1">
|
||||
{$i18n.t('fastCRW API Base URL')}
|
||||
</div>
|
||||
|
||||
<div class="flex w-full">
|
||||
<div class="flex-1">
|
||||
<input
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
type="text"
|
||||
placeholder={$i18n.t('Enter fastCRW API Base URL')}
|
||||
bind:value={webConfig.CRW_API_BASE_URL}
|
||||
autocomplete="off"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mt-2">
|
||||
<div class=" self-center text-xs font-medium mb-1">
|
||||
{$i18n.t('fastCRW API Key')}
|
||||
</div>
|
||||
|
||||
<SensitiveInput
|
||||
placeholder={$i18n.t('Enter fastCRW API Key')}
|
||||
bind:value={webConfig.CRW_API_KEY}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="mt-2">
|
||||
<div class=" self-center text-xs font-medium mb-1">
|
||||
{$i18n.t('fastCRW Timeout (s)')}
|
||||
</div>
|
||||
|
||||
<div class="flex w-full">
|
||||
<div class="flex-1">
|
||||
<input
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
type="number"
|
||||
placeholder={$i18n.t('Enter fastCRW Timeout')}
|
||||
bind:value={webConfig.CRW_TIMEOUT}
|
||||
autocomplete="off"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
|
|
@ -1145,6 +1195,37 @@
|
|||
/>
|
||||
</div>
|
||||
</div>
|
||||
{:else if webConfig.WEB_LOADER_ENGINE === 'crw' && webConfig.WEB_SEARCH_ENGINE !== 'crw'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
<div class=" self-center text-xs font-medium mb-1">
|
||||
{$i18n.t('fastCRW API Base URL')}
|
||||
</div>
|
||||
|
||||
<div class="flex w-full">
|
||||
<div class="flex-1">
|
||||
<input
|
||||
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||
type="text"
|
||||
placeholder={$i18n.t('Enter fastCRW API Base URL')}
|
||||
bind:value={webConfig.CRW_API_BASE_URL}
|
||||
autocomplete="off"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mt-2">
|
||||
<div class=" self-center text-xs font-medium mb-1">
|
||||
{$i18n.t('fastCRW API Key')}
|
||||
</div>
|
||||
|
||||
<SensitiveInput
|
||||
placeholder={$i18n.t('Enter fastCRW API Key')}
|
||||
bind:value={webConfig.CRW_API_KEY}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{:else if webConfig.WEB_LOADER_ENGINE === 'tavily'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -771,6 +771,9 @@
|
|||
"Enter Firecrawl API Base URL": "",
|
||||
"Enter Firecrawl API Key": "",
|
||||
"Enter Firecrawl Timeout": "",
|
||||
"Enter fastCRW API Base URL": "",
|
||||
"Enter fastCRW API Key": "",
|
||||
"Enter fastCRW Timeout": "",
|
||||
"Enter folder name": "",
|
||||
"Enter function name filter list (e.g. func1, !func2)": "",
|
||||
"Enter Github Raw URL": "",
|
||||
|
|
@ -959,6 +962,9 @@
|
|||
"Failed to update settings": "",
|
||||
"Failed to update status": "",
|
||||
"Failed to upload file.": "",
|
||||
"fastCRW API Base URL": "",
|
||||
"fastCRW API Key": "",
|
||||
"fastCRW Timeout (s)": "",
|
||||
"Features": "",
|
||||
"Features Permissions": "",
|
||||
"February": "",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue