mirror of
https://github.com/open-webui/open-webui.git
synced 2026-09-13 23:14:11 +00:00
fix: add bot-detection hardening for DuckDuckGo search and URL fetching
duckduckgo.py: - Pass browser-mimicking headers (User-Agent, Accept, Sec-Fetch-*) to DDGS() — missing Sec-Fetch-* headers are a primary bot-detection trigger - Add module-level cooldown (_cooldown_until) after RatelimitException; retrying immediately extends the IP block on DuckDuckGo's side - Bail out early while cooldown is active to avoid hitting the network utils.py (SafeWebBaseLoader._fetch): - Merge _BROWSER_FETCH_HEADERS into every aiohttp request so content sites do not soft-block the scraper as a headless bot - Detect non-standard rate-limit codes: HTTP 202 (DuckDuckGo soft-limit) and 503 (Cloudflare/WAF soft-block) are raised as ClientResponseError instead of being silently treated as success - Add _is_bot_challenge() to detect CAPTCHA / WAF challenge pages returned with HTTP 200 (Cloudflare returns a challenge page in a 200 body) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
imoes
parent
0c68d269a8
commit
5612f68a5c
2 changed files with 111 additions and 6 deletions
|
|
@ -1,5 +1,6 @@
|
|||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
|
||||
|
|
@ -8,6 +9,43 @@ from ddgs.exceptions import RatelimitException
|
|||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Cooldown durations (seconds). Retrying immediately after a rate-limit or
|
||||
# bot-detection extends the IP block — back off meaningfully instead.
|
||||
_COOLDOWN_RATELIMIT_S = 30
|
||||
_COOLDOWN_BOT_S = 60
|
||||
|
||||
_cooldown_until: float = 0.0 # monotonic timestamp; 0 = no active cooldown
|
||||
|
||||
|
||||
def _activate_cooldown(seconds: float) -> None:
|
||||
global _cooldown_until
|
||||
_cooldown_until = max(_cooldown_until, time.monotonic() + seconds)
|
||||
|
||||
|
||||
def _cooldown_remaining() -> float:
|
||||
return max(0.0, _cooldown_until - time.monotonic())
|
||||
|
||||
|
||||
# Mimic a real browser navigation request. DuckDuckGo (and many other
|
||||
# services) inspect Sec-Fetch-* headers to distinguish browser traffic from
|
||||
# bots — missing these headers is one of the most common detection triggers.
|
||||
_BROWSER_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,image/apng,*/*;q=0.8"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
# Referer tells DuckDuckGo the request originates from its own HTML frontend
|
||||
"Referer": "https://html.duckduckgo.com/",
|
||||
}
|
||||
|
||||
|
||||
def search_duckduckgo(
|
||||
query: str,
|
||||
|
|
@ -26,6 +64,14 @@ def search_duckduckgo(
|
|||
Returns:
|
||||
list[SearchResult]: A list of search results
|
||||
"""
|
||||
# Bail out early during a cooldown — hitting the network while blocked
|
||||
# resets the ban timer on DuckDuckGo's side.
|
||||
remaining = _cooldown_remaining()
|
||||
if remaining > 0:
|
||||
raise RuntimeError(
|
||||
f"DuckDuckGo rate-limit cooldown active — retry in {int(remaining) + 1}s"
|
||||
)
|
||||
|
||||
proxy = (
|
||||
os.environ.get("https_proxy")
|
||||
or os.environ.get("HTTPS_PROXY")
|
||||
|
|
@ -38,21 +84,22 @@ def search_duckduckgo(
|
|||
os.environ.setdefault("HTTPS_PROXY", proxy)
|
||||
os.environ.setdefault("HTTP_PROXY", proxy)
|
||||
|
||||
# Use the DDGS context manager to create a DDGS object
|
||||
search_results = []
|
||||
with DDGS(proxy=proxy) as ddgs:
|
||||
with DDGS(proxy=proxy, headers=_BROWSER_HEADERS) as ddgs:
|
||||
if concurrent_requests:
|
||||
ddgs.threads = concurrent_requests
|
||||
|
||||
# Use the ddgs.text() method to perform the search
|
||||
try:
|
||||
search_results = ddgs.text(query, safesearch='moderate', max_results=count, backend=backend)
|
||||
except RatelimitException as e:
|
||||
# Activate cooldown before logging — prevents any concurrent call
|
||||
# from slipping through while we handle the exception.
|
||||
_activate_cooldown(_COOLDOWN_RATELIMIT_S)
|
||||
log.error(f'RatelimitException: {e}')
|
||||
|
||||
if filter_list:
|
||||
search_results = get_filtered_results(search_results, filter_list)
|
||||
|
||||
# Return the list of search results
|
||||
return [
|
||||
SearchResult(
|
||||
link=result['href'],
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import urllib.parse
|
||||
|
|
@ -490,6 +491,42 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
|
|||
await browser.close()
|
||||
|
||||
|
||||
# Browser headers for URL fetching. Many news and content sites check
|
||||
# Sec-Fetch-* and User-Agent to filter out non-browser HTTP clients; sending
|
||||
# these headers avoids most soft bot-detection blocks without spoofing cookies.
|
||||
_BROWSER_FETCH_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,image/apng,*/*;q=0.8"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Site": "none",
|
||||
}
|
||||
|
||||
# Compiled once at import time for efficiency
|
||||
_BOT_CHALLENGE_RE = re.compile(
|
||||
r"g-recaptcha|are you a human|id=['\"]challenge-form['\"]"
|
||||
r"|name=['\"]challenge['\"]|cf-challenge|cf_chl_captcha"
|
||||
r"|__cf_chl_jschl_tk__|DDoS protection by",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_bot_challenge(html: str) -> bool:
|
||||
"""Return True if the page looks like a CAPTCHA or WAF challenge.
|
||||
|
||||
Services like Cloudflare return HTTP 200 with a challenge page instead of
|
||||
a real error code — callers must inspect the body to detect this case.
|
||||
"""
|
||||
return bool(_BOT_CHALLENGE_RE.search(html))
|
||||
|
||||
|
||||
class SafeWebBaseLoader(WebBaseLoader):
|
||||
"""WebBaseLoader with enhanced error handling for URLs."""
|
||||
|
||||
|
|
@ -511,8 +548,10 @@ class SafeWebBaseLoader(WebBaseLoader):
|
|||
async with aiohttp.ClientSession(trust_env=effective_trust_env) as session:
|
||||
for i in range(retries):
|
||||
try:
|
||||
# Merge browser headers first so caller-supplied headers
|
||||
# (e.g. auth tokens from self.session.headers) take precedence
|
||||
kwargs: Dict = dict(
|
||||
headers=self.session.headers,
|
||||
headers={**_BROWSER_FETCH_HEADERS, **self.session.headers},
|
||||
cookies=self.session.cookies.get_dict(),
|
||||
)
|
||||
if not self.session.verify:
|
||||
|
|
@ -523,9 +562,28 @@ class SafeWebBaseLoader(WebBaseLoader):
|
|||
**(self.requests_kwargs | kwargs),
|
||||
allow_redirects=False,
|
||||
) as response:
|
||||
# 202 is used by some services as a soft rate-limit signal;
|
||||
# 503 is a WAF / Cloudflare soft-block — treat like 429
|
||||
if response.status in (202, 503):
|
||||
raise aiohttp.ClientResponseError(
|
||||
response.request_info,
|
||||
response.history,
|
||||
status=response.status,
|
||||
message=f"Bot-detection / rate-limit (HTTP {response.status})",
|
||||
)
|
||||
if self.raise_for_status:
|
||||
response.raise_for_status()
|
||||
return await response.text()
|
||||
html = await response.text()
|
||||
# Cloudflare and other WAFs return HTTP 200 with a challenge
|
||||
# page instead of a real error code — detect by body content
|
||||
if _is_bot_challenge(html):
|
||||
raise aiohttp.ClientResponseError(
|
||||
response.request_info,
|
||||
response.history,
|
||||
status=200,
|
||||
message="Bot-detection challenge page in response body",
|
||||
)
|
||||
return html
|
||||
except aiohttp.ClientConnectionError as e:
|
||||
if i == retries - 1:
|
||||
raise
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue