From a5942da0a264b4e97a5d6a18adea7d4fcbfc275a Mon Sep 17 00:00:00 2001
From: Classic298 <27028174+Classic298@users.noreply.github.com>
Date: Fri, 16 Oct 2026 12:08:50 +0200
Subject: [PATCH] fix: PDFs from some sites reach the model as raw bytes
 (#32523)

Before Open WebUI reads a URL for the model, it checks whether the URL is a web page or a file. That check went out without the user agent, browser headers and SSL verification setting the web loader uses, so sites that block unknown clients (Wikimedia, for example) refused it. The URL was then treated as a web page and the model got the raw PDF bytes instead of the document text. The check now sends the same request as the web loader, so it gets the same answer and files go to document extraction again.
---
 backend/open_webui/retrieval/utils.py     |  5 ++++-
 backend/open_webui/retrieval/web/utils.py | 21 ++++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py
index e5a61d7ce5..8f5c080298 100644
--- a/backend/open_webui/retrieval/utils.py
+++ b/backend/open_webui/retrieval/utils.py
@@ -236,7 +236,7 @@ async def get_content_from_url(request, url: str, *, config: dict | None = None)
 
 
 def _get_content_from_url_sync(request, url: str, config: dict):
-    from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session
+    from open_webui.retrieval.web.utils import WEB_LOADER_HEADERS, validate_url, get_ssrf_safe_requests_session
 
     # Validate URL before making any request (blocks private IPs, non-HTTP, filter list)
     validate_url(url)
@@ -261,6 +261,9 @@ def _get_content_from_url_sync(request, url: str, config: dict):
     try:
         # Probe through the connect-time SSRF guard; bare requests.get re-resolves (DNS-rebinding gap).
         session = get_ssrf_safe_requests_session()
+        # Match the web loader's request, or sites that block unknown clients make a file look like a web page.
+        session.headers = WEB_LOADER_HEADERS
+        session.verify = config['web.loader.ssl_verification']
         response = session.get(url, stream=True, timeout=30, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS)
         response.raise_for_status()
         content_type = response.headers.get('Content-Type', '')
diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py
index e124d4328c..814bf0d886 100644
--- a/backend/open_webui/retrieval/web/utils.py
+++ b/backend/open_webui/retrieval/web/utils.py
@@ -1031,6 +1031,17 @@ class SafePlaywrightURLLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
                         raise e
 
 
+WEB_LOADER_HEADERS = {
+    'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Referer': 'https://www.google.com/',
+    'DNT': '1',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+}
+
+
 class SafeWebBaseLoader(BaseLoader):
     """Fetch pages with connect-time address checks and bounded concurrency."""
 
@@ -1059,15 +1070,7 @@ class SafeWebBaseLoader(BaseLoader):
         # Preserve the synchronous loader's environment-proxy behavior.
         self.session = get_ssrf_safe_requests_session()
         self.session.verify = verify_ssl
-        self.session.headers = {
-            'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Language': 'en-US,en;q=0.5',
-            'Referer': 'https://www.google.com/',
-            'DNT': '1',
-            'Connection': 'keep-alive',
-            'Upgrade-Insecure-Requests': '1',
-        }
+        self.session.headers = WEB_LOADER_HEADERS
 
     async def fetch_all(self, urls):
         semaphore = asyncio.Semaphore(self.requests_per_second)