scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-09-14 23:22:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: honour HTTP proxy env vars for DuckDuckGo search and URL fetching

Browse source 
Two issues prevented web search from working behind a corporate HTTP proxy:

1. duckduckgo.py: DDGS() was instantiated without a proxy argument, so all
   requests to DuckDuckGo bypassed the https_proxy/http_proxy env vars.
   Fix: read https_proxy (falling back to http_proxy) and pass it to DDGS().

2. utils.py: SafeWebBaseLoader._fetch created aiohttp.ClientSession with
   trust_env hardcoded to False. PersistentConfig can store False in its
   database even when WEB_SEARCH_TRUST_ENV=true is set in the environment,
   causing the DB value to silently override the env var on container restart.
   Fix: compute effective_trust_env = self.trust_env OR (proxy env var present),
   so the aiohttp session always uses the proxy when https_proxy/http_proxy
   are set, regardless of the stored config value.
   Also change all trust_env parameter defaults from False to True.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
imoes 2026-04-16 19:51:14 +02:00
parent 70a6a24f14
commit 4b610a3e1c
2 changed files with 15 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
backend/open_webui/retrieval/web/duckduckgo.py
View file

@ -1,4 +1,5 @@
import logging
import os
from typing import Optional
from open_webui.retrieval.web.main import SearchResult, get_filtered_results
@ -25,9 +26,10 @@ def search_duckduckgo(
Returns:
list[SearchResult]: A list of search results
"""
proxy = os.environ.get("https_proxy") or os.environ.get("http_proxy")
# Use the DDGS context manager to create a DDGS object
search_results = []
with DDGS() as ddgs:
with DDGS(proxy=proxy) as ddgs:
if concurrent_requests:
ddgs.threads = concurrent_requests

18
backend/open_webui/retrieval/web/utils.py
View file

@ -1,6 +1,7 @@
import asyncio
import ipaddress
import logging
import os
import socket
import ssl
import urllib.parse
@ -182,7 +183,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
self,
web_paths,
verify_ssl: bool = True,
trust_env: bool = False,
trust_env: bool = True,
requests_per_second: Optional[float] = None,
continue_on_failure: bool = True,
api_key: Optional[str] = None,
@ -272,7 +273,7 @@ class SafeTavilyLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
continue_on_failure: bool = True,
requests_per_second: Optional[float] = None,
verify_ssl: bool = True,
trust_env: bool = False,
trust_env: bool = True,
proxy: Optional[Dict[str, str]] = None,
):
"""Initialize SafeTavilyLoader with rate limiting and SSL verification support.
@ -394,7 +395,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
self,
web_paths: List[str],
verify_ssl: bool = True,
trust_env: bool = False,
trust_env: bool = True,
requests_per_second: Optional[float] = None,
continue_on_failure: bool = True,
headless: bool = True,
@ -492,7 +493,7 @@ class SafePlaywrightURLLoader(PlaywrightURLLoader, RateLimitMixin, URLProcessing
class SafeWebBaseLoader(WebBaseLoader):
"""WebBaseLoader with enhanced error handling for URLs."""
def __init__(self, trust_env: bool = False, *args, **kwargs):
def __init__(self, trust_env: bool = True, *args, **kwargs):
"""Initialize SafeWebBaseLoader
Args:
trust_env (bool, optional): set to True if using proxy to make web requests, for example
@ -502,7 +503,12 @@ class SafeWebBaseLoader(WebBaseLoader):
self.trust_env = trust_env
async def _fetch(self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5) -> str:
async with aiohttp.ClientSession(trust_env=self.trust_env) as session:
# honour trust_env, but also auto-enable when proxy env vars are present
# so that PersistentConfig DB value of False cannot silently bypass the proxy
effective_trust_env = self.trust_env or bool(
os.environ.get("https_proxy") or os.environ.get("http_proxy")
)
async with aiohttp.ClientSession(trust_env=effective_trust_env) as session:
for i in range(retries):
try:
kwargs: Dict = dict(
@ -587,7 +593,7 @@ def get_web_loader(
urls: Union[str, Sequence[str]],
verify_ssl: bool = True,
requests_per_second: int = 2,
trust_env: bool = False,
trust_env: bool = True,
):
# Check if the URLs are valid
safe_urls = safe_validate_urls([urls] if isinstance(urls, str) else urls)