From 94fb0447c3e7c822f9bfd9de00dd1aa2cf3e6dd2 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:44:33 +0200 Subject: [PATCH] refac: match web fetch filter entries that name an address or a range A filter entry that parses as an address or a CIDR range is matched by containment rather than by DNS label suffix, so a range covers the addresses inside it and an address matches however it is spelled. A range entry previously matched nothing at all, silently. The built-in list gains the special-purpose networks that ipaddress.is_global reports as reachable while nothing on them is a legitimate destination, so taking an address out of reach is a WEB_FETCH_FILTER_LIST change rather than a release. Those entries hold whether or not local web fetch is enabled; the private-address rule still follows the toggle. --- backend/open_webui/config.py | 14 ++++++ backend/open_webui/retrieval/web/main.py | 13 ++---- backend/open_webui/retrieval/web/utils.py | 57 +++++++---------------- backend/open_webui/utils/misc.py | 35 +++++++++++--- 4 files changed, 62 insertions(+), 57 deletions(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 75b4b6d284..664c61be01 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1106,12 +1106,26 @@ ENABLE_LOCAL_WEB_FETCH = ( ENABLE_RAG_LOCAL_WEB_FETCH = ENABLE_LOCAL_WEB_FETCH +# Operators extend this through WEB_FETCH_FILTER_LIST. DEFAULT_WEB_FETCH_FILTER_LIST = [ '!169.254.169.254', '!fd00:ec2::254', '!metadata.google.internal', '!metadata.azure.com', '!100.100.100.200', + '!168.63.129.16', # Azure platform channel, reachable from every Azure VM + '!192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526 + '!224.0.0.0/4', # IPv4 multicast + '!::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed + '!64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination + '!100:0:0:1::/64', # dummy prefix, RFC 9780 + '!2001:1::1', # PCP anycast, RFC 7723, answered by the local network's own edge device + '!2001:1::2', # TURN anycast, RFC 8155, likewise + '!2001:20::/28', # ORCHIDv2, RFC 7343, never routed + '!2001:30::/28', # DRIP, RFC 9374, never routed + '!5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain + '!fec0::/10', # IPv6 site-local, deprecated by RFC 3879 + '!ff00::/8', # IPv6 multicast ] web_fetch_filter_list = os.getenv('WEB_FETCH_FILTER_LIST', '') diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py index d8127807cf..be297c11c5 100644 --- a/backend/open_webui/retrieval/web/main.py +++ b/backend/open_webui/retrieval/web/main.py @@ -1,11 +1,10 @@ from __future__ import annotations -import ipaddress from urllib.parse import urlparse import validators from open_webui.retrieval.web.utils import resolve_hostname -from open_webui.utils.misc import get_allow_block_lists, is_host_allowed +from open_webui.utils.misc import as_network, get_allow_block_lists, is_host_allowed from pydantic import BaseModel @@ -14,14 +13,8 @@ def get_filtered_results(results, filter_list): return results allow_list, block_list = get_allow_block_lists(filter_list) - resolve_ips = False - for entry in allow_list + block_list: - try: - ipaddress.ip_address(entry) - except ValueError: - continue - resolve_ips = True - break + # Only worth a lookup when an entry names an address, since a hostname entry matches by name. + resolve_ips = any(as_network(entry) is not None for entry in allow_list + block_list) filtered_results = [] diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index 9bbc87224e..9fe26302b3 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -79,30 +79,8 @@ def resolve_hostname(hostname): return ipv4_addresses, ipv6_addresses -# Blocked despite ipaddress.is_global saying otherwise: none of these is a legitimate fetch target. -_BLOCKED_NETWORKS = tuple( - ipaddress.ip_network(cidr) - for cidr in ( - '168.63.129.16/32', # Azure platform channel, reachable from every Azure VM - '192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526 - '224.0.0.0/4', # IPv4 multicast - '::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed - '64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination - '100:0:0:1::/64', # dummy prefix, RFC 9780 - '5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain - 'fec0::/10', # IPv6 site-local, deprecated by RFC 3879 - 'ff00::/8', # IPv6 multicast - ) -) - - -def _in_allowed_range(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: - return addr.is_global and not any(addr in network for network in _BLOCKED_NETWORKS) - - -def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]: - """The IPv4 addresses an IPv6 address carries inside it: mapped, 6to4, teredo and NAT64.""" - addr = ipaddress.ip_address(ip) +def _embedded_ipv4(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> list[ipaddress.IPv4Address]: + """The IPv4 addresses an IPv6 address carries: mapped, compatible, 6to4, teredo and NAT64.""" if not isinstance(addr, ipaddress.IPv6Address): return [] @@ -115,8 +93,8 @@ def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]: embedded.extend(addr.teredo) b = addr.packed - # Prefixes that put the address in the last four bytes: v4-compatible, NAT64 /96, v4-translated. - if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8, b'\x00' * 8 + b'\xff\xff\x00\x00'): + # Prefixes that put the address in the last four bytes: v4-compatible and NAT64 /96. + if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8): embedded.append(ipaddress.IPv4Address(b[12:])) elif b[:6] == b'\x00\x64\xff\x9b\x00\x01': embedded.append(ipaddress.IPv4Address(bytes((b[6], b[7], b[9], b[10])))) @@ -124,13 +102,6 @@ def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]: return embedded -def _is_fetchable_ip(ip: str) -> bool: - addr = ipaddress.ip_address(ip) - if not _in_allowed_range(addr): - return False - return all(_in_allowed_range(embedded) for embedded in _embedded_ipv4(ip)) - - def _assert_host_allowed(host: str | None) -> None: if WEB_FETCH_FILTER_LIST and not is_host_allowed(host, WEB_FETCH_FILTER_LIST): log.warning(f'Blocked by filter list: {host}') @@ -138,14 +109,20 @@ def _assert_host_allowed(host: str | None) -> None: def _assert_addresses_allowed(addresses: Sequence[str]) -> None: - # An IPv6 address can carry a blocked IPv4 address inside it, so match both spellings. - candidates = [*addresses, *(str(ipv4) for address in addresses for ipv4 in _embedded_ipv4(address))] - if is_host_blocked(candidates, WEB_FETCH_FILTER_LIST): - log.warning(f'Blocked by filter list: {", ".join(candidates)}') + # An IPv6 address can carry a blocked IPv4 address inside it, so judge both spellings. + parsed = [ipaddress.ip_address(address) for address in addresses] + candidates = [*parsed, *(ipv4 for address in parsed for ipv4 in _embedded_ipv4(address))] + + # Block entries only: an allow entry names a host, so judging a resolved address against one + # would reject every allow-listed host. + if is_host_blocked([str(address) for address in candidates], WEB_FETCH_FILTER_LIST): + log.warning(f'Blocked by filter list: {", ".join(str(address) for address in candidates)}') raise ValueError(ERROR_MESSAGES.INVALID_URL) + if not ENABLE_LOCAL_WEB_FETCH: - for address in addresses: - if not _is_fetchable_ip(address): + for address in candidates: + if not address.is_global: + log.warning(f'Blocked non-global address: {address}') raise ValueError(ERROR_MESSAGES.INVALID_URL) @@ -205,7 +182,7 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]: def _ssrf_safe_new_conn(self): - """Resolve DNS, validate all IPs are global, connect to validated IP. + """Resolve DNS, screen every resolved address, connect to one of them. Replaces urllib3's _new_conn so the DNS lookup that feeds the actual TCP connect is the same one we validate — no second resolution, no rebinding diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index ed1b7b0253..5e06ea5624 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -2,12 +2,14 @@ from __future__ import annotations import collections.abc import hashlib +import ipaddress import logging import re import threading import time import uuid from datetime import timedelta +from functools import lru_cache from pathlib import Path from typing import Callable, Optional, Sequence, Union @@ -87,17 +89,39 @@ def is_string_allowed(string: Union[str, Sequence[str]], filter_list: list[str | return True +@lru_cache(maxsize=512) +def as_network(pattern: str) -> ipaddress.IPv4Network | ipaddress.IPv6Network | None: + """A filter entry read as an address range, or None when the entry names a host instead. + + Surrounding whitespace and a trailing dot are stripped here rather than by each caller, + since ip_network rejects both and the callers do not normalise the same way. + """ + try: + return ipaddress.ip_network((pattern or '').strip().lower().rstrip('.'), strict=False) + except ValueError: + return None + + def _host_matches_pattern(host: str, pattern: str) -> bool: """Match a hostname against a filter entry on DNS label boundaries. `pattern` matches `host` when equal or a parent domain of it, so `corp.com` - matches `api.corp.com` but not `evilcorp.com`, and an IP literal matches only - itself. Avoids the raw-suffix confusion of a plain endswith. + matches `api.corp.com` but not `evilcorp.com`. Avoids the raw-suffix confusion + of a plain endswith. + + An entry that names an address or a CIDR range is matched by containment instead, so + `10.0.0.0/8` covers `10.1.2.3` and an address matches any spelling of itself in its own family. """ host = (host or '').strip().lower().rstrip('.') pattern = (pattern or '').strip().lower().rstrip('.') if not host or not pattern: return False + network = as_network(pattern) + if network is not None: + try: + return ipaddress.ip_address(host) in network + except ValueError: + return False # a hostname is never inside an address range return host == pattern or host.endswith('.' + pattern) @@ -108,6 +132,7 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non Pass a parsed hostname, never a full URL: matching against a URL lets a path component defeat the filter (e.g. ``https://blocked.example/x`` ends with ``/x``, not the blocked host). Entries prefixed with ``!`` are blocked; the rest form an allowlist. + An entry naming an address or a CIDR range is matched by containment instead. """ if not filter_list: return True @@ -123,11 +148,7 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non def is_host_blocked(host: Union[str, Sequence[str]], filter_list: list[str | None] = None) -> bool: - """Whether a host or resolved address matches a block entry, ignoring any allow entries. - - For addresses, where an allow entry cannot apply: it names a host, and the address at hand - may belong to a forward proxy rather than to the host the request is actually for. - """ + """Whether a host or resolved address matches a block entry, ignoring any allow entries.""" _, block_list = get_allow_block_lists(filter_list) hosts = [host] if isinstance(host, str) else list(host or []) return any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list)