diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index e5a61d7ce5..8f5c080298 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -236,7 +236,7 @@ async def get_content_from_url(request, url: str, *, config: dict | None = None) def _get_content_from_url_sync(request, url: str, config: dict): - from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session + from open_webui.retrieval.web.utils import WEB_LOADER_HEADERS, validate_url, get_ssrf_safe_requests_session # Validate URL before making any request (blocks private IPs, non-HTTP, filter list) validate_url(url) @@ -261,6 +261,9 @@ def _get_content_from_url_sync(request, url: str, config: dict): try: # Probe through the connect-time SSRF guard; bare requests.get re-resolves (DNS-rebinding gap). session = get_ssrf_safe_requests_session() + # Match the web loader's request, or sites that block unknown clients make a file look like a web page. + session.headers = WEB_LOADER_HEADERS + session.verify = config['web.loader.ssl_verification'] response = session.get(url, stream=True, timeout=30, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS) response.raise_for_status() content_type = response.headers.get('Content-Type', '') diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index e124d4328c..814bf0d886 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -1031,6 +1031,17 @@ class SafePlaywrightURLLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): raise e +WEB_LOADER_HEADERS = { + 'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://www.google.com/', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', +} + + class SafeWebBaseLoader(BaseLoader): """Fetch pages with connect-time address checks and bounded concurrency.""" @@ -1059,15 +1070,7 @@ class SafeWebBaseLoader(BaseLoader): # Preserve the synchronous loader's environment-proxy behavior. self.session = get_ssrf_safe_requests_session() self.session.verify = verify_ssl - self.session.headers = { - 'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Referer': 'https://www.google.com/', - 'DNT': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - } + self.session.headers = WEB_LOADER_HEADERS async def fetch_all(self, urls): semaphore = asyncio.Semaphore(self.requests_per_second)