From 5612f68a5c76c4d16efd7a057c4426ba03fea52d Mon Sep 17 00:00:00 2001 From: imoes Date: Wed, 22 Apr 2026 14:56:55 +0200 Subject: [PATCH] fix: add bot-detection hardening for DuckDuckGo search and URL fetching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit duckduckgo.py: - Pass browser-mimicking headers (User-Agent, Accept, Sec-Fetch-*) to DDGS() — missing Sec-Fetch-* headers are a primary bot-detection trigger - Add module-level cooldown (_cooldown_until) after RatelimitException; retrying immediately extends the IP block on DuckDuckGo's side - Bail out early while cooldown is active to avoid hitting the network utils.py (SafeWebBaseLoader._fetch): - Merge _BROWSER_FETCH_HEADERS into every aiohttp request so content sites do not soft-block the scraper as a headless bot - Detect non-standard rate-limit codes: HTTP 202 (DuckDuckGo soft-limit) and 503 (Cloudflare/WAF soft-block) are raised as ClientResponseError instead of being silently treated as success - Add _is_bot_challenge() to detect CAPTCHA / WAF challenge pages returned with HTTP 200 (Cloudflare returns a challenge page in a 200 body) Co-Authored-By: Claude Sonnet 4.6 --- .../open_webui/retrieval/web/duckduckgo.py | 55 ++++++++++++++-- backend/open_webui/retrieval/web/utils.py | 62 ++++++++++++++++++- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/backend/open_webui/retrieval/web/duckduckgo.py b/backend/open_webui/retrieval/web/duckduckgo.py index fddd12a195..91b9707dc6 100644 --- a/backend/open_webui/retrieval/web/duckduckgo.py +++ b/backend/open_webui/retrieval/web/duckduckgo.py @@ -1,5 +1,6 @@ import logging import os +import time from typing import Optional from open_webui.retrieval.web.main import SearchResult, get_filtered_results @@ -8,6 +9,43 @@ from ddgs.exceptions import RatelimitException log = logging.getLogger(__name__) +# Cooldown durations (seconds). Retrying immediately after a rate-limit or +# bot-detection extends the IP block — back off meaningfully instead. +_COOLDOWN_RATELIMIT_S = 30 +_COOLDOWN_BOT_S = 60 + +_cooldown_until: float = 0.0 # monotonic timestamp; 0 = no active cooldown + + +def _activate_cooldown(seconds: float) -> None: + global _cooldown_until + _cooldown_until = max(_cooldown_until, time.monotonic() + seconds) + + +def _cooldown_remaining() -> float: + return max(0.0, _cooldown_until - time.monotonic()) + + +# Mimic a real browser navigation request. DuckDuckGo (and many other +# services) inspect Sec-Fetch-* headers to distinguish browser traffic from +# bots — missing these headers is one of the most common detection triggers. +_BROWSER_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + ), + "Accept": ( + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8" + ), + "Accept-Language": "en-US,en;q=0.9", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Site": "same-origin", + # Referer tells DuckDuckGo the request originates from its own HTML frontend + "Referer": "https://html.duckduckgo.com/", +} + def search_duckduckgo( query: str, @@ -26,6 +64,14 @@ def search_duckduckgo( Returns: list[SearchResult]: A list of search results """ + # Bail out early during a cooldown — hitting the network while blocked + # resets the ban timer on DuckDuckGo's side. + remaining = _cooldown_remaining() + if remaining > 0: + raise RuntimeError( + f"DuckDuckGo rate-limit cooldown active — retry in {int(remaining) + 1}s" + ) + proxy = ( os.environ.get("https_proxy") or os.environ.get("HTTPS_PROXY") @@ -38,21 +84,22 @@ def search_duckduckgo( os.environ.setdefault("HTTPS_PROXY", proxy) os.environ.setdefault("HTTP_PROXY", proxy) - # Use the DDGS context manager to create a DDGS object search_results = [] - with DDGS(proxy=proxy) as ddgs: + with DDGS(proxy=proxy, headers=_BROWSER_HEADERS) as ddgs: if concurrent_requests: ddgs.threads = concurrent_requests - # Use the ddgs.text() method to perform the search try: search_results = ddgs.text(query, safesearch='moderate', max_results=count, backend=backend) except RatelimitException as e: + # Activate cooldown before logging — prevents any concurrent call + # from slipping through while we handle the exception. + _activate_cooldown(_COOLDOWN_RATELIMIT_S) log.error(f'RatelimitException: {e}') + if filter_list: search_results = get_filtered_results(search_results, filter_list) - # Return the list of search results return [ SearchResult( link=result['href'], diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 0eac711eea..7f5ddc985e 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -2,6 +2,7 @@ import asyncio import ipaddress import logging import os +import re import socket import ssl import urllib.parse @@ -490,6 +491,42 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing await browser.close() +# Browser headers for URL fetching. Many news and content sites check +# Sec-Fetch-* and User-Agent to filter out non-browser HTTP clients; sending +# these headers avoids most soft bot-detection blocks without spoofing cookies. +_BROWSER_FETCH_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + ), + "Accept": ( + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8" + ), + "Accept-Language": "en-US,en;q=0.9", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Site": "none", +} + +# Compiled once at import time for efficiency +_BOT_CHALLENGE_RE = re.compile( + r"g-recaptcha|are you a human|id=['\"]challenge-form['\"]" + r"|name=['\"]challenge['\"]|cf-challenge|cf_chl_captcha" + r"|__cf_chl_jschl_tk__|DDoS protection by", + re.IGNORECASE, +) + + +def _is_bot_challenge(html: str) -> bool: + """Return True if the page looks like a CAPTCHA or WAF challenge. + + Services like Cloudflare return HTTP 200 with a challenge page instead of + a real error code — callers must inspect the body to detect this case. + """ + return bool(_BOT_CHALLENGE_RE.search(html)) + + class SafeWebBaseLoader(WebBaseLoader): """WebBaseLoader with enhanced error handling for URLs.""" @@ -511,8 +548,10 @@ class SafeWebBaseLoader(WebBaseLoader): async with aiohttp.ClientSession(trust_env=effective_trust_env) as session: for i in range(retries): try: + # Merge browser headers first so caller-supplied headers + # (e.g. auth tokens from self.session.headers) take precedence kwargs: Dict = dict( - headers=self.session.headers, + headers={**_BROWSER_FETCH_HEADERS, **self.session.headers}, cookies=self.session.cookies.get_dict(), ) if not self.session.verify: @@ -523,9 +562,28 @@ class SafeWebBaseLoader(WebBaseLoader): **(self.requests_kwargs | kwargs), allow_redirects=False, ) as response: + # 202 is used by some services as a soft rate-limit signal; + # 503 is a WAF / Cloudflare soft-block — treat like 429 + if response.status in (202, 503): + raise aiohttp.ClientResponseError( + response.request_info, + response.history, + status=response.status, + message=f"Bot-detection / rate-limit (HTTP {response.status})", + ) if self.raise_for_status: response.raise_for_status() - return await response.text() + html = await response.text() + # Cloudflare and other WAFs return HTTP 200 with a challenge + # page instead of a real error code — detect by body content + if _is_bot_challenge(html): + raise aiohttp.ClientResponseError( + response.request_info, + response.history, + status=200, + message="Bot-detection challenge page in response body", + ) + return html except aiohttp.ClientConnectionError as e: if i == retries - 1: raise