From 4b610a3e1c37059152b5f3a70cf51c1b77fcd2d8 Mon Sep 17 00:00:00 2001 From: imoes Date: Thu, 16 Apr 2026 19:51:14 +0200 Subject: [PATCH] fix: honour HTTP proxy env vars for DuckDuckGo search and URL fetching Two issues prevented web search from working behind a corporate HTTP proxy: 1. duckduckgo.py: DDGS() was instantiated without a proxy argument, so all requests to DuckDuckGo bypassed the https_proxy/http_proxy env vars. Fix: read https_proxy (falling back to http_proxy) and pass it to DDGS(). 2. utils.py: SafeWebBaseLoader._fetch created aiohttp.ClientSession with trust_env hardcoded to False. PersistentConfig can store False in its database even when WEB_SEARCH_TRUST_ENV=true is set in the environment, causing the DB value to silently override the env var on container restart. Fix: compute effective_trust_env = self.trust_env OR (proxy env var present), so the aiohttp session always uses the proxy when https_proxy/http_proxy are set, regardless of the stored config value. Also change all trust_env parameter defaults from False to True. Co-Authored-By: Claude Sonnet 4.6 --- backend/open_webui/retrieval/web/duckduckgo.py | 4 +++- backend/open_webui/retrieval/web/utils.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/open_webui/retrieval/web/duckduckgo.py b/backend/open_webui/retrieval/web/duckduckgo.py index da1c3f77ec..ea3b09022f 100644 --- a/backend/open_webui/retrieval/web/duckduckgo.py +++ b/backend/open_webui/retrieval/web/duckduckgo.py @@ -1,4 +1,5 @@ import logging +import os from typing import Optional from open_webui.retrieval.web.main import SearchResult, get_filtered_results @@ -25,9 +26,10 @@ def search_duckduckgo( Returns: list[SearchResult]: A list of search results """ + proxy = os.environ.get("https_proxy") or os.environ.get("http_proxy") # Use the DDGS context manager to create a DDGS object search_results = [] - with DDGS() as ddgs: + with DDGS(proxy=proxy) as ddgs: if concurrent_requests: ddgs.threads = concurrent_requests diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index cfe0f71b85..0eac711eea 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -1,6 +1,7 @@ import asyncio import ipaddress import logging +import os import socket import ssl import urllib.parse @@ -182,7 +183,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): self, web_paths, verify_ssl: bool = True, - trust_env: bool = False, + trust_env: bool = True, requests_per_second: Optional[float] = None, continue_on_failure: bool = True, api_key: Optional[str] = None, @@ -272,7 +273,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): continue_on_failure: bool = True, requests_per_second: Optional[float] = None, verify_ssl: bool = True, - trust_env: bool = False, + trust_env: bool = True, proxy: Optional[Dict[str, str]] = None, ): """Initialize SafeTavilyLoader with rate limiting and SSL verification support. @@ -394,7 +395,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing self, web_paths: List[str], verify_ssl: bool = True, - trust_env: bool = False, + trust_env: bool = True, requests_per_second: Optional[float] = None, continue_on_failure: bool = True, headless: bool = True, @@ -492,7 +493,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing class SafeWebBaseLoader(WebBaseLoader): """WebBaseLoader with enhanced error handling for URLs.""" - def __init__(self, trust_env: bool = False, *args, **kwargs): + def __init__(self, trust_env: bool = True, *args, **kwargs): """Initialize SafeWebBaseLoader Args: trust_env (bool, optional): set to True if using proxy to make web requests, for example @@ -502,7 +503,12 @@ class SafeWebBaseLoader(WebBaseLoader): self.trust_env = trust_env async def _fetch(self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5) -> str: - async with aiohttp.ClientSession(trust_env=self.trust_env) as session: + # honour trust_env, but also auto-enable when proxy env vars are present + # so that PersistentConfig DB value of False cannot silently bypass the proxy + effective_trust_env = self.trust_env or bool( + os.environ.get("https_proxy") or os.environ.get("http_proxy") + ) + async with aiohttp.ClientSession(trust_env=effective_trust_env) as session: for i in range(retries): try: kwargs: Dict = dict( @@ -587,7 +593,7 @@ def get_web_loader( urls: Union[str, Sequence[str]], verify_ssl: bool = True, requests_per_second: int = 2, - trust_env: bool = False, + trust_env: bool = True, ): # Check if the URLs are valid safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)