From 834a2cf109079d4ee2a6fd13aa8fe601b78554fc Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:55:35 +0200 Subject: [PATCH] Match WEB_FETCH_FILTER_LIST on hostnames with label boundaries, not URL suffix (CWE-693) is_string_allowed does endswith() matching and was called with the full URL (retrieval/web/utils.py) against WEB_FETCH_FILTER_LIST, so a blocklisted host with any path (https://blocked.example/x) ended with /x, not the host, and slipped through; the allowlist direction false-rejected legitimate URLs and admitted attacker URLs ending in an allowed string. The same endswith caused label confusion at the hostname call site (retrieval/web/main.py): corp.com matched evilcorp.com, 10.0.0.1 matched 110.0.0.1. Add is_host_allowed(host, ...) matching on DNS label boundaries (host == pattern or host.endswith('.' + pattern)), called with the parsed hostname at both web-fetch call sites. is_string_allowed is left unchanged for the unrelated function-name filters (utils/middleware.py, utils/tools.py). The separate is_global guard (validate_url / _ssrf_safe_new_conn, active when ENABLE_RAG_LOCAL_WEB_FETCH is off) already blocks RFC1918/loopback/link-local, so this restores the admin's intended blocking of specific public hosts. Co-authored-by: addcontent <59762500+addcontent@users.noreply.github.com> Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/open_webui/retrieval/web/main.py | 4 +-- backend/open_webui/retrieval/web/utils.py | 6 ++-- backend/open_webui/utils/misc.py | 38 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py index a55c62c8b5..56f796db58 100644 --- a/backend/open_webui/retrieval/web/main.py +++ b/backend/open_webui/retrieval/web/main.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import validators from open_webui.retrieval.web.utils import resolve_hostname -from open_webui.utils.misc import is_string_allowed +from open_webui.utils.misc import is_host_allowed from pydantic import BaseModel @@ -32,7 +32,7 @@ def get_filtered_results(results, filter_list): except Exception: pass - if is_string_allowed(hostnames, filter_list): + if is_host_allowed(hostnames, filter_list): filtered_results.append(result) continue diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index afa73a9e0e..eae367c09f 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -50,7 +50,7 @@ from open_webui.env import AIOHTTP_CLIENT_ALLOW_REDIRECTS, AIOHTTP_CLIENT_SESSIO from open_webui.retrieval.loaders.external_web import ExternalWebLoader from open_webui.retrieval.loaders.tavily import TavilyLoader from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url -from open_webui.utils.misc import is_string_allowed +from open_webui.utils.misc import is_host_allowed log = logging.getLogger(__name__) @@ -88,7 +88,9 @@ def validate_url(url: Union[str, Sequence[str]]): # Blocklist check using unified filtering logic if WEB_FETCH_FILTER_LIST: - if not is_string_allowed(url, WEB_FETCH_FILTER_LIST): + # Match on the parsed hostname, not the full URL: a path component would + # otherwise let any URL slip past a hostname-based block/allow entry. + if not is_host_allowed(parsed_url.hostname, WEB_FETCH_FILTER_LIST): log.warning(f'URL blocked by filter list: {url}') raise ValueError(ERROR_MESSAGES.INVALID_URL) diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index d5d8078bed..c5c6528d10 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -69,6 +69,44 @@ def is_string_allowed(string: Union[str, Sequence[str]], filter_list: list[str | return True +def _host_matches_pattern(host: str, pattern: str) -> bool: + """Match a hostname against a filter entry on DNS label boundaries. + + `pattern` matches `host` when equal or a parent domain of it, so `corp.com` + matches `api.corp.com` but not `evilcorp.com`, and an IP literal matches only + itself. Avoids the raw-suffix confusion of a plain endswith. + """ + host = (host or '').strip().lower().rstrip('.') + pattern = (pattern or '').strip().lower().rstrip('.') + if not host or not pattern: + return False + return host == pattern or host.endswith('.' + pattern) + + +def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | None] = None) -> bool: + """Allow/block a hostname (or list of hostnames / resolved IPs) against a + WEB_FETCH_FILTER_LIST-style filter, matching on label boundaries. + + Pass a parsed hostname, never a full URL: matching against a URL lets a path + component defeat the filter (e.g. ``https://blocked.example/x`` ends with ``/x``, + not the blocked host). Entries prefixed with ``!`` are blocked; the rest form an allowlist. + """ + if not filter_list: + return True + + allow_list, block_list = get_allow_block_lists(filter_list) + hosts = [host] if isinstance(host, str) else list(host or []) + + if allow_list: + if not any(_host_matches_pattern(h, allowed) for h in hosts for allowed in allow_list): + return False + + if any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list): + return False + + return True + + def get_message_list(messages_map, message_id): """ Reconstructs a list of messages in order up to the specified message_id.