scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-10-18 04:43:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: PDFs from some sites reach the model as raw bytes (#32523)

Browse source 
Before Open WebUI reads a URL for the model, it checks whether the URL is a web page or a file. That check went out without the user agent, browser headers and SSL verification setting the web loader uses, so sites that block unknown clients (Wikimedia, for example) refused it. The URL was then treated as a web page and the model got the raw PDF bytes instead of the document text. The check now sends the same request as the web loader, so it gets the same answer and files go to document extraction again.
This commit is contained in:
Classic298 2026-10-16 12:08:50 +02:00 • committed by GitHub
parent d00044cdda
commit a5942da0a2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 16 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
backend/open_webui/retrieval/utils.py
View file

@ -236,7 +236,7 @@ async def get_content_from_url(request, url: str, *, config: dict | None = None)
def _get_content_from_url_sync(request, url: str, config: dict):
from open_webui.retrieval.web.utils import validate_url, get_ssrf_safe_requests_session
from open_webui.retrieval.web.utils import WEB_LOADER_HEADERS, validate_url, get_ssrf_safe_requests_session
# Validate URL before making any request (blocks private IPs, non-HTTP, filter list)
validate_url(url)
@ -261,6 +261,9 @@ def _get_content_from_url_sync(request, url: str, config: dict):
try:
# Probe through the connect-time SSRF guard; bare requests.get re-resolves (DNS-rebinding gap).
session = get_ssrf_safe_requests_session()
# Match the web loader's request, or sites that block unknown clients make a file look like a web page.
session.headers = WEB_LOADER_HEADERS
session.verify = config['web.loader.ssl_verification']
response = session.get(url, stream=True, timeout=30, allow_redirects=AIOHTTP_CLIENT_ALLOW_REDIRECTS)
response.raise_for_status()
content_type = response.headers.get('Content-Type', '')

21
backend/open_webui/retrieval/web/utils.py
View file

@ -1031,6 +1031,17 @@ class SafePlaywrightURLLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
raise e
WEB_LOADER_HEADERS = {
'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': 'https://www.google.com/',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}
class SafeWebBaseLoader(BaseLoader):
"""Fetch pages with connect-time address checks and bounded concurrency."""
@ -1059,15 +1070,7 @@ class SafeWebBaseLoader(BaseLoader):
# Preserve the synchronous loader's environment-proxy behavior.
self.session = get_ssrf_safe_requests_session()
self.session.verify = verify_ssl
self.session.headers = {
'User-Agent': USER_AGENT or 'DefaultLangchainUserAgent',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': 'https://www.google.com/',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
}
self.session.headers = WEB_LOADER_HEADERS
async def fetch_all(self, urls):
semaphore = asyncio.Semaphore(self.requests_per_second)