diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index f8868940d8..acd3d83bf8 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1111,12 +1111,26 @@ ENABLE_LOCAL_WEB_FETCH = ( ENABLE_RAG_LOCAL_WEB_FETCH = ENABLE_LOCAL_WEB_FETCH +# Operators extend this through WEB_FETCH_FILTER_LIST. DEFAULT_WEB_FETCH_FILTER_LIST = [ '!169.254.169.254', '!fd00:ec2::254', '!metadata.google.internal', '!metadata.azure.com', '!100.100.100.200', + '!168.63.129.16', # Azure platform channel, reachable from every Azure VM + '!192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526 + '!224.0.0.0/4', # IPv4 multicast + '!::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed + '!64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination + '!100:0:0:1::/64', # dummy prefix, RFC 9780 + '!2001:1::1', # PCP anycast, RFC 7723, answered by the local network's own edge device + '!2001:1::2', # TURN anycast, RFC 8155, likewise + '!2001:20::/28', # ORCHIDv2, RFC 7343, never routed + '!2001:30::/28', # DRIP, RFC 9374, never routed + '!5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain + '!fec0::/10', # IPv6 site-local, deprecated by RFC 3879 + '!ff00::/8', # IPv6 multicast ] web_fetch_filter_list = os.getenv('WEB_FETCH_FILTER_LIST', '') diff --git a/backend/open_webui/retrieval/web/main.py b/backend/open_webui/retrieval/web/main.py index d8127807cf..be297c11c5 100644 --- a/backend/open_webui/retrieval/web/main.py +++ b/backend/open_webui/retrieval/web/main.py @@ -1,11 +1,10 @@ from __future__ import annotations -import ipaddress from urllib.parse import urlparse import validators from open_webui.retrieval.web.utils import resolve_hostname -from open_webui.utils.misc import get_allow_block_lists, is_host_allowed +from open_webui.utils.misc import as_network, get_allow_block_lists, is_host_allowed from pydantic import BaseModel @@ -14,14 +13,8 @@ def get_filtered_results(results, filter_list): return results allow_list, block_list = get_allow_block_lists(filter_list) - resolve_ips = False - for entry in allow_list + block_list: - try: - ipaddress.ip_address(entry) - except ValueError: - continue - resolve_ips = True - break + # Only worth a lookup when an entry names an address, since a hostname entry matches by name. + resolve_ips = any(as_network(entry) is not None for entry in allow_list + block_list) filtered_results = [] diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index ace34b062c..9fe26302b3 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -23,7 +23,6 @@ from typing import ( ) import aiohttp -import aiohttp.resolver import certifi import requests import urllib3.connection @@ -64,7 +63,7 @@ from open_webui.retrieval.loaders.external_web import ExternalWebLoader from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader from open_webui.retrieval.loaders.tavily import TavilyLoader from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url -from open_webui.utils.misc import is_host_allowed +from open_webui.utils.misc import is_host_allowed, is_host_blocked log = logging.getLogger(__name__) @@ -80,12 +79,10 @@ def resolve_hostname(hostname): return ipv4_addresses, ipv6_addresses -def _is_global_addr(ip: str) -> bool: - addr = ipaddress.ip_address(ip) - if not addr.is_global: - return False +def _embedded_ipv4(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> list[ipaddress.IPv4Address]: + """The IPv4 addresses an IPv6 address carries: mapped, compatible, 6to4, teredo and NAT64.""" if not isinstance(addr, ipaddress.IPv6Address): - return True + return [] embedded = [] if addr.ipv4_mapped: @@ -96,16 +93,37 @@ def _is_global_addr(ip: str) -> bool: embedded.extend(addr.teredo) b = addr.packed - if b[:12] == b'\x00' * 12: - embedded.append(ipaddress.IPv4Address(b[12:])) - elif b[:12] == b'\x00\x64\xff\x9b' + b'\x00' * 8: + # Prefixes that put the address in the last four bytes: v4-compatible and NAT64 /96. + if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8): embedded.append(ipaddress.IPv4Address(b[12:])) elif b[:6] == b'\x00\x64\xff\x9b\x00\x01': - if b[8] != 0: - return False embedded.append(ipaddress.IPv4Address(bytes((b[6], b[7], b[9], b[10])))) - return all(ip.is_global for ip in embedded) + return embedded + + +def _assert_host_allowed(host: str | None) -> None: + if WEB_FETCH_FILTER_LIST and not is_host_allowed(host, WEB_FETCH_FILTER_LIST): + log.warning(f'Blocked by filter list: {host}') + raise ValueError(ERROR_MESSAGES.INVALID_URL) + + +def _assert_addresses_allowed(addresses: Sequence[str]) -> None: + # An IPv6 address can carry a blocked IPv4 address inside it, so judge both spellings. + parsed = [ipaddress.ip_address(address) for address in addresses] + candidates = [*parsed, *(ipv4 for address in parsed for ipv4 in _embedded_ipv4(address))] + + # Block entries only: an allow entry names a host, so judging a resolved address against one + # would reject every allow-listed host. + if is_host_blocked([str(address) for address in candidates], WEB_FETCH_FILTER_LIST): + log.warning(f'Blocked by filter list: {", ".join(str(address) for address in candidates)}') + raise ValueError(ERROR_MESSAGES.INVALID_URL) + + if not ENABLE_LOCAL_WEB_FETCH: + for address in candidates: + if not address.is_global: + log.warning(f'Blocked non-global address: {address}') + raise ValueError(ERROR_MESSAGES.INVALID_URL) def validate_url(url: Union[str, Sequence[str]]): @@ -128,24 +146,22 @@ def validate_url(url: Union[str, Sequence[str]]): log.warning(f'Blocked non-HTTP(S) protocol: {parsed_url.scheme} in URL: {url}') raise ValueError(ERROR_MESSAGES.INVALID_URL) - # Blocklist check using unified filtering logic - if WEB_FETCH_FILTER_LIST: - # Match on the parsed hostname, not the full URL: a path component would - # otherwise let any URL slip past a hostname-based block/allow entry. - if not is_host_allowed(parsed_url.hostname, WEB_FETCH_FILTER_LIST): - log.warning(f'URL blocked by filter list: {url}') - raise ValueError(ERROR_MESSAGES.INVALID_URL) + # Match on the parsed hostname, not the full URL: a path component would + # otherwise let any URL slip past a hostname-based block/allow entry. + _assert_host_allowed(parsed_url.hostname) - if not ENABLE_LOCAL_WEB_FETCH: - # Local web fetch is disabled, filter out URLs that resolve to non-global IP addresses. - parsed_url = urllib.parse.urlparse(url) - # Get IPv4 and IPv6 addresses + try: ipv4_addresses, ipv6_addresses = resolve_hostname(parsed_url.hostname) - # Check if any of the resolved addresses are private - # DNS rebinding is mitigated at the connection layer; see _SSRFSafeResolver / _SSRFSafeAdapter - for ip in ipv4_addresses + ipv6_addresses: - if not _is_global_addr(ip): - raise ValueError(ERROR_MESSAGES.INVALID_URL) + except (socket.gaierror, UnicodeError) as e: + # With local fetch on, a proxied deployment can carry names only the proxy resolves. + if not ENABLE_LOCAL_WEB_FETCH: + log.warning(f'Could not resolve host {parsed_url.hostname}: {e}') + raise ValueError(ERROR_MESSAGES.INVALID_URL) from None + ipv4_addresses, ipv6_addresses = [], [] + + # A hostname match alone lets a DNS record point at a blocked address. + # DNS rebinding is mitigated at the connection layer; see _SSRFSafeConnector / _SSRFSafeAdapter + _assert_addresses_allowed(ipv4_addresses + ipv6_addresses) return True elif isinstance(url, Sequence): return all(validate_url(u) for u in url) @@ -166,7 +182,7 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]: def _ssrf_safe_new_conn(self): - """Resolve DNS, validate all IPs are global, connect to validated IP. + """Resolve DNS, screen every resolved address, connect to one of them. Replaces urllib3's _new_conn so the DNS lookup that feeds the actual TCP connect is the same one we validate — no second resolution, no rebinding @@ -177,10 +193,7 @@ def _ssrf_safe_new_conn(self): infos = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) if not infos: raise OSError(f'getaddrinfo for {host!r} returned empty list') - if not ENABLE_LOCAL_WEB_FETCH: - for _, _, _, _, sa in infos: - if not _is_global_addr(sa[0]): - raise ValueError(ERROR_MESSAGES.INVALID_URL) + _assert_addresses_allowed([sa[0] for _, _, _, _, sa in infos]) err = None for fam, typ, proto, _, sa in infos: sock = None @@ -223,7 +236,7 @@ class _SafeHTTPSPool(urllib3.connectionpool.HTTPSConnectionPool): class _SSRFSafeAdapter(HTTPAdapter): - """requests transport adapter that validates resolved IPs at connect time.""" + """requests adapter that rejects filter-listed request targets and non-global IPs at connect time.""" def init_poolmanager(self, *args, **kwargs): super().init_poolmanager(*args, **kwargs) @@ -232,21 +245,29 @@ class _SSRFSafeAdapter(HTTPAdapter): 'https': _SafeHTTPSPool, } + def send(self, request, *args, **kwargs): + # Per request, not per connection: the connection layer sees the proxy. + _assert_host_allowed(urllib.parse.urlparse(request.url).hostname) + return super().send(request, *args, **kwargs) -class _SSRFSafeResolver(aiohttp.resolver.DefaultResolver): - """aiohttp resolver that rejects non-global IPs unless local fetch is on.""" - async def resolve(self, host, port=0, family=socket.AF_INET): - results = await super().resolve(host, port, family) - if not ENABLE_LOCAL_WEB_FETCH: - for entry in results: - if not _is_global_addr(entry['host']): - raise ValueError(ERROR_MESSAGES.INVALID_URL) +class _SSRFSafeConnector(aiohttp.TCPConnector): + """Rejects filter-listed request targets, and non-global IPs on each new connection.""" + + async def connect(self, req, traces, timeout): + # Per request, not per connection: _resolve_host sees the proxy and pooled reuse skips it. + _assert_host_allowed(req.url.host) + return await super().connect(req, traces, timeout) + + async def _resolve_host(self, host, port, traces=None): + # aiohttp answers IP-literal hosts itself without consulting a resolver. + results = await super()._resolve_host(host, port, traces=traces) + _assert_addresses_allowed([entry['host'] for entry in results]) return results def get_ssrf_safe_session(trust_env: bool = True, store_cookies: bool = True) -> aiohttp.ClientSession: - """A one-off aiohttp session that re-validates the connect-time IP via _SSRFSafeResolver, + """A one-off aiohttp session that re-validates every connection via _SSRFSafeConnector, defeating DNS rebinding. Use for validate_url-gated fetches of user-supplied URLs that must not use the shared (rebinding-vulnerable) pool. Use as a context manager so it is closed: ``async with get_ssrf_safe_session() as session: ...``. @@ -255,7 +276,7 @@ def get_ssrf_safe_session(trust_env: bool = True, store_cookies: bool = True) -> IP check, because the proxy resolves the hostname instead. """ return aiohttp.ClientSession( - connector=aiohttp.TCPConnector(resolver=_SSRFSafeResolver()), + connector=_SSRFSafeConnector(), timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT), trust_env=trust_env, cookie_jar=None if store_cookies else aiohttp.DummyCookieJar(), @@ -912,7 +933,7 @@ class SafeWebBaseLoader(WebBaseLoader): self.session.mount('https://', _SSRFSafeAdapter()) async def _fetch(self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5) -> str: - connector = aiohttp.TCPConnector(resolver=_SSRFSafeResolver()) + connector = _SSRFSafeConnector() async with aiohttp.ClientSession(trust_env=self.trust_env, connector=connector) as session: for i in range(retries): try: diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index f8d0f3cd66..96fedc3cd9 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -2,12 +2,14 @@ from __future__ import annotations import collections.abc import hashlib +import ipaddress import logging import re import threading import time import uuid from datetime import timedelta +from functools import lru_cache from pathlib import Path from typing import Callable, Optional, Sequence, Union @@ -87,17 +89,39 @@ def is_string_allowed(string: Union[str, Sequence[str]], filter_list: list[str | return True +@lru_cache(maxsize=512) +def as_network(pattern: str) -> ipaddress.IPv4Network | ipaddress.IPv6Network | None: + """A filter entry read as an address range, or None when the entry names a host instead. + + Surrounding whitespace and a trailing dot are stripped here rather than by each caller, + since ip_network rejects both and the callers do not normalise the same way. + """ + try: + return ipaddress.ip_network((pattern or '').strip().lower().rstrip('.'), strict=False) + except ValueError: + return None + + def _host_matches_pattern(host: str, pattern: str) -> bool: """Match a hostname against a filter entry on DNS label boundaries. `pattern` matches `host` when equal or a parent domain of it, so `corp.com` - matches `api.corp.com` but not `evilcorp.com`, and an IP literal matches only - itself. Avoids the raw-suffix confusion of a plain endswith. + matches `api.corp.com` but not `evilcorp.com`. Avoids the raw-suffix confusion + of a plain endswith. + + An entry that names an address or a CIDR range is matched by containment instead, so + `10.0.0.0/8` covers `10.1.2.3` and an address matches any spelling of itself in its own family. """ host = (host or '').strip().lower().rstrip('.') pattern = (pattern or '').strip().lower().rstrip('.') if not host or not pattern: return False + network = as_network(pattern) + if network is not None: + try: + return ipaddress.ip_address(host) in network + except ValueError: + return False # a hostname is never inside an address range return host == pattern or host.endswith('.' + pattern) @@ -108,21 +132,26 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non Pass a parsed hostname, never a full URL: matching against a URL lets a path component defeat the filter (e.g. ``https://blocked.example/x`` ends with ``/x``, not the blocked host). Entries prefixed with ``!`` are blocked; the rest form an allowlist. + An entry naming an address or a CIDR range is matched by containment instead. """ if not filter_list: return True - allow_list, block_list = get_allow_block_lists(filter_list) + allow_list, _ = get_allow_block_lists(filter_list) hosts = [host] if isinstance(host, str) else list(host or []) if allow_list: if not any(_host_matches_pattern(h, allowed) for h in hosts for allowed in allow_list): return False - if any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list): - return False + return not is_host_blocked(hosts, filter_list) - return True + +def is_host_blocked(host: Union[str, Sequence[str]], filter_list: list[str | None] = None) -> bool: + """Whether a host or resolved address matches a block entry, ignoring any allow entries.""" + _, block_list = get_allow_block_lists(filter_list) + hosts = [host] if isinstance(host, str) else list(host or []) + return any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list) def get_message_list(messages_map, message_id):