mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-02 02:12:25 +00:00
feat: add Staan as a native web search provider (#30138)
European deployments that need EU data residency have no hosted web search option out of the box: every plug-and-play provider shipped today is US-based, and the only sovereign alternative is self-hosting SearXNG, which means running and maintaining that infrastructure yourself. Staan (staan.ai) is a European search API with EU data residency, so adding it gives those deployments a drop-in choice. It is configured like any other provider, through the admin UI or STAAN_API_KEY, STAAN_MARKET and STAAN_MAX_SNIPPETS. Market sets the region and language of the results and defaults to en-us. Max snippets asks Staan to fetch each result page and return semantically scored chunks of it, which get merged into that result's snippet so retrieval has more to work with; leaving it at 0 uses the plain search endpoint. Wired the same way as Tavily and Exa, with domain filtering going through the shared get_filtered_results. Requested in #26006.
This commit is contained in:
Classic298
GitHub
parent
655337f7bf
commit
9ee810ad09
5 changed files with 149 additions and 0 deletions
|
|
@ -1298,6 +1298,12 @@ TAVILY_API_KEY = os.getenv('TAVILY_API_KEY', '')
|
|||
|
||||
TAVILY_EXTRACT_DEPTH = os.getenv('TAVILY_EXTRACT_DEPTH', 'basic')
|
||||
|
||||
STAAN_API_KEY = os.getenv('STAAN_API_KEY', '')
|
||||
|
||||
STAAN_MARKET = os.getenv('STAAN_MARKET', 'en-us')
|
||||
|
||||
STAAN_MAX_SNIPPETS = int(os.getenv('STAAN_MAX_SNIPPETS', '0'))
|
||||
|
||||
PLAYWRIGHT_WS_URL = os.getenv('PLAYWRIGHT_WS_URL', '')
|
||||
|
||||
PLAYWRIGHT_TIMEOUT = int(os.getenv('PLAYWRIGHT_TIMEOUT', '10000'))
|
||||
|
|
@ -3018,6 +3024,9 @@ DEFAULT_CONFIG = {
|
|||
'web.search.sougou_api_sk': SOUGOU_API_SK,
|
||||
'web.search.tavily_api_key': TAVILY_API_KEY,
|
||||
'web.search.tavily_extract_depth': TAVILY_EXTRACT_DEPTH,
|
||||
'web.search.staan_api_key': STAAN_API_KEY,
|
||||
'web.search.staan_market': STAAN_MARKET,
|
||||
'web.search.staan_max_snippets': STAAN_MAX_SNIPPETS,
|
||||
'web.loader.playwright_ws_url': PLAYWRIGHT_WS_URL,
|
||||
'web.loader.playwright_timeout': PLAYWRIGHT_TIMEOUT,
|
||||
'web.loader.firecrawl_api_key': FIRECRAWL_API_KEY,
|
||||
|
|
|
|||
60
backend/open_webui/retrieval/web/staan.py
Normal file
60
backend/open_webui/retrieval/web/staan.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import requests
|
||||
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
|
||||
|
||||
|
||||
def search_staan(
|
||||
api_key: str,
|
||||
query: str,
|
||||
count: int,
|
||||
filter_list: list[str] | None = None,
|
||||
market: str | None = None,
|
||||
max_snippets: int | None = None,
|
||||
) -> list[SearchResult]:
|
||||
"""Search using Staan's Web Search API and return the results as a list of SearchResult objects.
|
||||
|
||||
Args:
|
||||
api_key (str): A Staan API key
|
||||
query (str): The query to search for
|
||||
count (int): The maximum number of results to return
|
||||
filter_list (list[str] | None): The domains to allow or block
|
||||
market (str | None): The market to search in, e.g. 'en-us'
|
||||
max_snippets (int | None): The maximum extra snippets to request per result
|
||||
|
||||
Returns:
|
||||
A list of SearchResult objects.
|
||||
"""
|
||||
url = 'https://api.staan.ai/v2/search/web'
|
||||
headers = {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': f'Bearer {api_key}',
|
||||
}
|
||||
params = {'q': query, 'market': market, 'count': count}
|
||||
|
||||
if max_snippets:
|
||||
params['extra_snippets'] = 'true'
|
||||
params['max_snippets'] = max_snippets
|
||||
|
||||
response = requests.get(url, headers=headers, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
results = response.json().get('web', {}).get('results', [])
|
||||
if filter_list:
|
||||
results = get_filtered_results(results, filter_list)
|
||||
|
||||
return [
|
||||
SearchResult(
|
||||
link=result.get('url', ''),
|
||||
title=result.get('title'),
|
||||
snippet=_build_snippet(result),
|
||||
)
|
||||
for result in results[:count]
|
||||
]
|
||||
|
||||
|
||||
def _build_snippet(result: dict) -> str:
|
||||
"""Combine the snippet and the extra snippets list into a single string."""
|
||||
parts = [result.get('snippet')]
|
||||
parts.extend(extra.get('chunk') for extra in result.get('extra_snippets', []))
|
||||
return '\n\n'.join(part for part in parts if part)
|
||||
|
|
@ -115,6 +115,7 @@ from open_webui.retrieval.web.serphouse import search_serphouse
|
|||
from open_webui.retrieval.web.serply import search_serply
|
||||
from open_webui.retrieval.web.serpstack import search_serpstack
|
||||
from open_webui.retrieval.web.sougou import search_sougou
|
||||
from open_webui.retrieval.web.staan import search_staan
|
||||
from open_webui.retrieval.web.tavily import search_tavily
|
||||
from open_webui.retrieval.web.utils import get_web_loader
|
||||
from open_webui.retrieval.web.yacy import search_yacy
|
||||
|
|
@ -380,6 +381,9 @@ RETRIEVAL_CONFIG_KEYS = {
|
|||
'SERPSTACK_HTTPS': 'web.search.serpstack_https',
|
||||
'SOUGOU_API_SID': 'web.search.sougou_api_sid',
|
||||
'SOUGOU_API_SK': 'web.search.sougou_api_sk',
|
||||
'STAAN_API_KEY': 'web.search.staan_api_key',
|
||||
'STAAN_MARKET': 'web.search.staan_market',
|
||||
'STAAN_MAX_SNIPPETS': 'web.search.staan_max_snippets',
|
||||
'TAVILY_API_KEY': 'web.search.tavily_api_key',
|
||||
'TAVILY_EXTRACT_DEPTH': 'web.search.tavily_extract_depth',
|
||||
'TEXT_SPLITTER': 'rag.text_splitter',
|
||||
|
|
@ -737,6 +741,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||
'SERPLY_API_KEY': config.SERPLY_API_KEY,
|
||||
'DDGS_BACKEND': config.DDGS_BACKEND,
|
||||
'TAVILY_API_KEY': config.TAVILY_API_KEY,
|
||||
'STAAN_API_KEY': config.STAAN_API_KEY,
|
||||
'STAAN_MARKET': config.STAAN_MARKET,
|
||||
'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS,
|
||||
'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY,
|
||||
'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE,
|
||||
'SERPAPI_API_KEY': config.SERPAPI_API_KEY,
|
||||
|
|
@ -817,6 +824,9 @@ class WebConfig(BaseModel):
|
|||
SERPLY_API_KEY: str | None = None
|
||||
DDGS_BACKEND: str | None = None
|
||||
TAVILY_API_KEY: str | None = None
|
||||
STAAN_API_KEY: str | None = None
|
||||
STAAN_MARKET: str | None = None
|
||||
STAAN_MAX_SNIPPETS: int | None = None
|
||||
SEARCHAPI_API_KEY: str | None = None
|
||||
SEARCHAPI_ENGINE: str | None = None
|
||||
SERPAPI_API_KEY: str | None = None
|
||||
|
|
@ -1337,6 +1347,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||
config.SERPLY_API_KEY = form_data.web.SERPLY_API_KEY
|
||||
config.DDGS_BACKEND = form_data.web.DDGS_BACKEND
|
||||
config.TAVILY_API_KEY = form_data.web.TAVILY_API_KEY
|
||||
config.STAAN_API_KEY = form_data.web.STAAN_API_KEY
|
||||
config.STAAN_MARKET = form_data.web.STAAN_MARKET
|
||||
config.STAAN_MAX_SNIPPETS = form_data.web.STAAN_MAX_SNIPPETS
|
||||
config.SEARCHAPI_API_KEY = form_data.web.SEARCHAPI_API_KEY
|
||||
config.SEARCHAPI_ENGINE = form_data.web.SEARCHAPI_ENGINE
|
||||
config.SERPAPI_API_KEY = form_data.web.SERPAPI_API_KEY
|
||||
|
|
@ -1490,6 +1503,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
|
|||
'SERPHOUSE_DOMAIN': config.SERPHOUSE_DOMAIN,
|
||||
'SERPLY_API_KEY': config.SERPLY_API_KEY,
|
||||
'TAVILY_API_KEY': config.TAVILY_API_KEY,
|
||||
'STAAN_API_KEY': config.STAAN_API_KEY,
|
||||
'STAAN_MARKET': config.STAAN_MARKET,
|
||||
'STAAN_MAX_SNIPPETS': config.STAAN_MAX_SNIPPETS,
|
||||
'SEARCHAPI_API_KEY': config.SEARCHAPI_API_KEY,
|
||||
'SEARCHAPI_ENGINE': config.SEARCHAPI_ENGINE,
|
||||
'SERPAPI_API_KEY': config.SERPAPI_API_KEY,
|
||||
|
|
@ -2689,6 +2705,19 @@ async def search_web(request: Request, engine: str, query: str, user=None) -> li
|
|||
)
|
||||
else:
|
||||
raise Exception('No TAVILY_API_KEY found in environment variables')
|
||||
elif engine == 'staan':
|
||||
if config.STAAN_API_KEY:
|
||||
return await asyncio.to_thread(
|
||||
search_staan,
|
||||
config.STAAN_API_KEY,
|
||||
query,
|
||||
config.WEB_SEARCH_RESULT_COUNT,
|
||||
config.WEB_SEARCH_DOMAIN_FILTER_LIST,
|
||||
market=config.STAAN_MARKET,
|
||||
max_snippets=config.STAAN_MAX_SNIPPETS,
|
||||
)
|
||||
else:
|
||||
raise Exception('No STAAN_API_KEY found in environment variables')
|
||||
elif engine == 'exa':
|
||||
if config.EXA_API_KEY:
|
||||
return await asyncio.to_thread(
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
'serpapi',
|
||||
'duckduckgo',
|
||||
'tavily',
|
||||
'staan',
|
||||
'jina',
|
||||
'bing',
|
||||
'exa',
|
||||
|
|
@ -578,6 +579,50 @@
|
|||
/>
|
||||
</div>
|
||||
</div>
|
||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'staan'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
<div class=" self-center text-xs text-gray-600 dark:text-gray-400 mb-1">
|
||||
{$i18n.t('Staan API Key')}
|
||||
</div>
|
||||
|
||||
<SensitiveInput
|
||||
variant="settings"
|
||||
placeholder={$i18n.t('Enter Staan API Key')}
|
||||
bind:value={webConfig.STAAN_API_KEY}
|
||||
/>
|
||||
</div>
|
||||
<AdminSettingField
|
||||
className="mt-1.5"
|
||||
label={$i18n.t('Market')}
|
||||
forId="staan-market"
|
||||
description={$i18n.t('Region and language of the search results, e.g. en-us.')}
|
||||
>
|
||||
<input
|
||||
id="staan-market"
|
||||
class={inputClass}
|
||||
placeholder="en-us"
|
||||
bind:value={webConfig.STAAN_MARKET}
|
||||
/>
|
||||
</AdminSettingField>
|
||||
<AdminSettingField
|
||||
className="mt-1.5"
|
||||
label={$i18n.t('Max Snippets')}
|
||||
forId="staan-max-snippets"
|
||||
description={$i18n.t(
|
||||
'Maximum extra snippets returned per result. Set to 0 to disable them.'
|
||||
)}
|
||||
>
|
||||
<input
|
||||
id="staan-max-snippets"
|
||||
class={inputClass}
|
||||
type="number"
|
||||
min="0"
|
||||
step="1"
|
||||
bind:value={webConfig.STAAN_MAX_SNIPPETS}
|
||||
/>
|
||||
</AdminSettingField>
|
||||
</div>
|
||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'searchapi'}
|
||||
<div class="mb-2.5 flex w-full flex-col">
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -1115,6 +1115,7 @@
|
|||
"Enter skill instructions in markdown...": "",
|
||||
"Enter Sougou Search API sID": "",
|
||||
"Enter Sougou Search API SK": "",
|
||||
"Enter Staan API Key": "",
|
||||
"Enter stop sequence": "",
|
||||
"Enter system prompt": "",
|
||||
"Enter system prompt here": "",
|
||||
|
|
@ -1717,16 +1718,19 @@
|
|||
"Mark as unread": "",
|
||||
"Markdown": "",
|
||||
"Markdown Header Text Splitter": "",
|
||||
"Market": "",
|
||||
"Max background": "",
|
||||
"Max concurrent": "",
|
||||
"Max iterations": "",
|
||||
"Max output": "",
|
||||
"Max Snippets": "",
|
||||
"Max Speakers": "",
|
||||
"Max tokens to retrieve (1024-32768, default 8192)": "",
|
||||
"Max Upload Count": "",
|
||||
"Max Upload Size": "",
|
||||
"Maximum characters to return from fetched URLs. Leave empty for no limit.": "",
|
||||
"Maximum concurrent embedding requests. Set to 0 for unlimited.": "",
|
||||
"Maximum extra snippets returned per result. Set to 0 to disable them.": "",
|
||||
"Maximum file size in MB. Leave empty for unlimited.": "",
|
||||
"Maximum number of files allowed per folder.": "",
|
||||
"Maximum number of files per folder is {{max}}.": "",
|
||||
|
|
@ -2303,6 +2307,7 @@
|
|||
"Regenerate": "",
|
||||
"Regenerate Menu": "",
|
||||
"Regenerate Response": "",
|
||||
"Region and language of the search results, e.g. en-us.": "",
|
||||
"Register Again": "",
|
||||
"Register Client": "",
|
||||
"Registered": "",
|
||||
|
|
@ -2685,6 +2690,7 @@
|
|||
"Speech-to-Text Language": "",
|
||||
"Split documents by markdown headers before character or token splitting.": "",
|
||||
"Sponsor on GitHub": "",
|
||||
"Staan API Key": "",
|
||||
"Start a new conversation": "",
|
||||
"Start new chats as temporary unless changed.": "",
|
||||
"Start of the channel": "",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue