mirror of
https://github.com/open-webui/open-webui.git
synced 2026-08-28 05:27:35 +00:00
refac: consolidate the web fetch address checks onto the request path (#27823)
* fix: apply the SSRF checks to redirect targets on every web fetch path Two guards protect server-side fetches: a private-IP check and the operator's `WEB_FETCH_FILTER_LIST`. Neither reached a redirect hop on the aiohttp paths, and the filter list never reached one on the requests paths either. aiohttp answers IP-literal hosts itself without consulting a resolver, so `_SSRFSafeResolver` was never invoked for a hop such as `http://169.254.169.254/` and the private-IP check simply did not run. With redirect following enabled, a submitted public URL that redirects to an IP literal reached loopback, RFC1918 and cloud-metadata addresses, and the response body was returned to the caller. The filter list was consulted only in `validate_url`, on the originally submitted URL, so a redirect to a filter-listed host was fetched without it ever being applied. `_SSRFSafeResolver` is replaced by `_SSRFSafeConnector`, which hooks `_resolve_host` so the IP check also covers the IP-literal shortcut and both DNS cache paths. The filter list moves to a per-request hook on each transport, `connect()` for aiohttp and `send()` for the requests adapter, because those see the request destination: at the connection layer a proxied request presents the proxy's host, and a pooled connection skips resolution entirely. This covers every hop, including redirects, on all five aiohttp call sites and both requests sessions. The Playwright loader already validated each hop and is unchanged. Both gaps required `AIOHTTP_CLIENT_ALLOW_REDIRECTS=true`, which is not the default. Two behaviour changes for operators. The filter list now applies to redirect targets rather than only to submitted URLs. Under a forward proxy it is evaluated against the request destination instead of the proxy, which also fixes allowlist entries rejecting every fetch in proxied deployments. * refac: match the web fetch filter list against resolved addresses The filter list is now evaluated against the hostname together with the addresses it resolves to, at URL validation and on each connection, on both transports. An IPv6 address is also matched by the IPv4 address it carries. * refac: screen outbound fetch addresses against reserved ranges ipaddress misses `ipaddress.is_global` was the only test behind the web-fetch address check, and it answers a narrower question than "may we fetch this". Several special-purpose ranges are globally routable by registry while nothing on them is a legitimate destination, so they passed. Classification now screens those ranges on top of `is_global`, and applies the same screen to the IPv4 address embedded in an IPv6 transition encoding rather than only to the literal. All three checkpoints share the predicate, so they all inherit it. The range list is the exact complement of what CPython's `ipaddress` already models, checked entry by entry against both IANA special-purpose registries. Prefixes IANA marks globally reachable are deliberately left out, so no real destination changes behaviour. Verified against 31 addresses covering every entry, their transition-encoded forms, and public controls in both families: 31/31 expected after, 18/31 before. * refac: match web fetch filter entries that name an address or a range A filter entry that parses as an address or a CIDR range is matched by containment rather than by DNS label suffix, so a range covers the addresses inside it and an address matches however it is spelled. A range entry previously matched nothing at all, silently. The built-in list gains the special-purpose networks that ipaddress.is_global reports as reachable while nothing on them is a legitimate destination, so taking an address out of reach is a WEB_FETCH_FILTER_LIST change rather than a release. Those entries hold whether or not local web fetch is enabled; the private-address rule still follows the toggle.
This commit is contained in:
Classic298
GitHub
parent
ca4e07a40b
commit
e3e4bd87df
4 changed files with 119 additions and 62 deletions
|
|
@ -1111,12 +1111,26 @@ ENABLE_LOCAL_WEB_FETCH = (
|
|||
ENABLE_RAG_LOCAL_WEB_FETCH = ENABLE_LOCAL_WEB_FETCH
|
||||
|
||||
|
||||
# Operators extend this through WEB_FETCH_FILTER_LIST.
|
||||
DEFAULT_WEB_FETCH_FILTER_LIST = [
|
||||
'!169.254.169.254',
|
||||
'!fd00:ec2::254',
|
||||
'!metadata.google.internal',
|
||||
'!metadata.azure.com',
|
||||
'!100.100.100.200',
|
||||
'!168.63.129.16', # Azure platform channel, reachable from every Azure VM
|
||||
'!192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526
|
||||
'!224.0.0.0/4', # IPv4 multicast
|
||||
'!::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed
|
||||
'!64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination
|
||||
'!100:0:0:1::/64', # dummy prefix, RFC 9780
|
||||
'!2001:1::1', # PCP anycast, RFC 7723, answered by the local network's own edge device
|
||||
'!2001:1::2', # TURN anycast, RFC 8155, likewise
|
||||
'!2001:20::/28', # ORCHIDv2, RFC 7343, never routed
|
||||
'!2001:30::/28', # DRIP, RFC 9374, never routed
|
||||
'!5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain
|
||||
'!fec0::/10', # IPv6 site-local, deprecated by RFC 3879
|
||||
'!ff00::/8', # IPv6 multicast
|
||||
]
|
||||
|
||||
web_fetch_filter_list = os.getenv('WEB_FETCH_FILTER_LIST', '')
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import validators
|
||||
from open_webui.retrieval.web.utils import resolve_hostname
|
||||
from open_webui.utils.misc import get_allow_block_lists, is_host_allowed
|
||||
from open_webui.utils.misc import as_network, get_allow_block_lists, is_host_allowed
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
|
|
@ -14,14 +13,8 @@ def get_filtered_results(results, filter_list):
|
|||
return results
|
||||
|
||||
allow_list, block_list = get_allow_block_lists(filter_list)
|
||||
resolve_ips = False
|
||||
for entry in allow_list + block_list:
|
||||
try:
|
||||
ipaddress.ip_address(entry)
|
||||
except ValueError:
|
||||
continue
|
||||
resolve_ips = True
|
||||
break
|
||||
# Only worth a lookup when an entry names an address, since a hostname entry matches by name.
|
||||
resolve_ips = any(as_network(entry) is not None for entry in allow_list + block_list)
|
||||
|
||||
filtered_results = []
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ from typing import (
|
|||
)
|
||||
|
||||
import aiohttp
|
||||
import aiohttp.resolver
|
||||
import certifi
|
||||
import requests
|
||||
import urllib3.connection
|
||||
|
|
@ -64,7 +63,7 @@ from open_webui.retrieval.loaders.external_web import ExternalWebLoader
|
|||
from open_webui.retrieval.loaders.microsoft_web_iq import MicrosoftWebIQLoader
|
||||
from open_webui.retrieval.loaders.tavily import TavilyLoader
|
||||
from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
|
||||
from open_webui.utils.misc import is_host_allowed
|
||||
from open_webui.utils.misc import is_host_allowed, is_host_blocked
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -80,12 +79,10 @@ def resolve_hostname(hostname):
|
|||
return ipv4_addresses, ipv6_addresses
|
||||
|
||||
|
||||
def _is_global_addr(ip: str) -> bool:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
if not addr.is_global:
|
||||
return False
|
||||
def _embedded_ipv4(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> list[ipaddress.IPv4Address]:
|
||||
"""The IPv4 addresses an IPv6 address carries: mapped, compatible, 6to4, teredo and NAT64."""
|
||||
if not isinstance(addr, ipaddress.IPv6Address):
|
||||
return True
|
||||
return []
|
||||
|
||||
embedded = []
|
||||
if addr.ipv4_mapped:
|
||||
|
|
@ -96,16 +93,37 @@ def _is_global_addr(ip: str) -> bool:
|
|||
embedded.extend(addr.teredo)
|
||||
|
||||
b = addr.packed
|
||||
if b[:12] == b'\x00' * 12:
|
||||
embedded.append(ipaddress.IPv4Address(b[12:]))
|
||||
elif b[:12] == b'\x00\x64\xff\x9b' + b'\x00' * 8:
|
||||
# Prefixes that put the address in the last four bytes: v4-compatible and NAT64 /96.
|
||||
if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8):
|
||||
embedded.append(ipaddress.IPv4Address(b[12:]))
|
||||
elif b[:6] == b'\x00\x64\xff\x9b\x00\x01':
|
||||
if b[8] != 0:
|
||||
return False
|
||||
embedded.append(ipaddress.IPv4Address(bytes((b[6], b[7], b[9], b[10]))))
|
||||
|
||||
return all(ip.is_global for ip in embedded)
|
||||
return embedded
|
||||
|
||||
|
||||
def _assert_host_allowed(host: str | None) -> None:
|
||||
if WEB_FETCH_FILTER_LIST and not is_host_allowed(host, WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'Blocked by filter list: {host}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
|
||||
def _assert_addresses_allowed(addresses: Sequence[str]) -> None:
|
||||
# An IPv6 address can carry a blocked IPv4 address inside it, so judge both spellings.
|
||||
parsed = [ipaddress.ip_address(address) for address in addresses]
|
||||
candidates = [*parsed, *(ipv4 for address in parsed for ipv4 in _embedded_ipv4(address))]
|
||||
|
||||
# Block entries only: an allow entry names a host, so judging a resolved address against one
|
||||
# would reject every allow-listed host.
|
||||
if is_host_blocked([str(address) for address in candidates], WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'Blocked by filter list: {", ".join(str(address) for address in candidates)}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
for address in candidates:
|
||||
if not address.is_global:
|
||||
log.warning(f'Blocked non-global address: {address}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
|
||||
def validate_url(url: Union[str, Sequence[str]]):
|
||||
|
|
@ -128,24 +146,22 @@ def validate_url(url: Union[str, Sequence[str]]):
|
|||
log.warning(f'Blocked non-HTTP(S) protocol: {parsed_url.scheme} in URL: {url}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
# Blocklist check using unified filtering logic
|
||||
if WEB_FETCH_FILTER_LIST:
|
||||
# Match on the parsed hostname, not the full URL: a path component would
|
||||
# otherwise let any URL slip past a hostname-based block/allow entry.
|
||||
if not is_host_allowed(parsed_url.hostname, WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'URL blocked by filter list: {url}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
# Match on the parsed hostname, not the full URL: a path component would
|
||||
# otherwise let any URL slip past a hostname-based block/allow entry.
|
||||
_assert_host_allowed(parsed_url.hostname)
|
||||
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
# Local web fetch is disabled, filter out URLs that resolve to non-global IP addresses.
|
||||
parsed_url = urllib.parse.urlparse(url)
|
||||
# Get IPv4 and IPv6 addresses
|
||||
try:
|
||||
ipv4_addresses, ipv6_addresses = resolve_hostname(parsed_url.hostname)
|
||||
# Check if any of the resolved addresses are private
|
||||
# DNS rebinding is mitigated at the connection layer; see _SSRFSafeResolver / _SSRFSafeAdapter
|
||||
for ip in ipv4_addresses + ipv6_addresses:
|
||||
if not _is_global_addr(ip):
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
except (socket.gaierror, UnicodeError) as e:
|
||||
# With local fetch on, a proxied deployment can carry names only the proxy resolves.
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
log.warning(f'Could not resolve host {parsed_url.hostname}: {e}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL) from None
|
||||
ipv4_addresses, ipv6_addresses = [], []
|
||||
|
||||
# A hostname match alone lets a DNS record point at a blocked address.
|
||||
# DNS rebinding is mitigated at the connection layer; see _SSRFSafeConnector / _SSRFSafeAdapter
|
||||
_assert_addresses_allowed(ipv4_addresses + ipv6_addresses)
|
||||
return True
|
||||
elif isinstance(url, Sequence):
|
||||
return all(validate_url(u) for u in url)
|
||||
|
|
@ -166,7 +182,7 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
|
|||
|
||||
|
||||
def _ssrf_safe_new_conn(self):
|
||||
"""Resolve DNS, validate all IPs are global, connect to validated IP.
|
||||
"""Resolve DNS, screen every resolved address, connect to one of them.
|
||||
|
||||
Replaces urllib3's _new_conn so the DNS lookup that feeds the actual TCP
|
||||
connect is the same one we validate — no second resolution, no rebinding
|
||||
|
|
@ -177,10 +193,7 @@ def _ssrf_safe_new_conn(self):
|
|||
infos = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
|
||||
if not infos:
|
||||
raise OSError(f'getaddrinfo for {host!r} returned empty list')
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
for _, _, _, _, sa in infos:
|
||||
if not _is_global_addr(sa[0]):
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
_assert_addresses_allowed([sa[0] for _, _, _, _, sa in infos])
|
||||
err = None
|
||||
for fam, typ, proto, _, sa in infos:
|
||||
sock = None
|
||||
|
|
@ -223,7 +236,7 @@ class _SafeHTTPSPool(urllib3.connectionpool.HTTPSConnectionPool):
|
|||
|
||||
|
||||
class _SSRFSafeAdapter(HTTPAdapter):
|
||||
"""requests transport adapter that validates resolved IPs at connect time."""
|
||||
"""requests adapter that rejects filter-listed request targets and non-global IPs at connect time."""
|
||||
|
||||
def init_poolmanager(self, *args, **kwargs):
|
||||
super().init_poolmanager(*args, **kwargs)
|
||||
|
|
@ -232,21 +245,29 @@ class _SSRFSafeAdapter(HTTPAdapter):
|
|||
'https': _SafeHTTPSPool,
|
||||
}
|
||||
|
||||
def send(self, request, *args, **kwargs):
|
||||
# Per request, not per connection: the connection layer sees the proxy.
|
||||
_assert_host_allowed(urllib.parse.urlparse(request.url).hostname)
|
||||
return super().send(request, *args, **kwargs)
|
||||
|
||||
class _SSRFSafeResolver(aiohttp.resolver.DefaultResolver):
|
||||
"""aiohttp resolver that rejects non-global IPs unless local fetch is on."""
|
||||
|
||||
async def resolve(self, host, port=0, family=socket.AF_INET):
|
||||
results = await super().resolve(host, port, family)
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
for entry in results:
|
||||
if not _is_global_addr(entry['host']):
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
class _SSRFSafeConnector(aiohttp.TCPConnector):
|
||||
"""Rejects filter-listed request targets, and non-global IPs on each new connection."""
|
||||
|
||||
async def connect(self, req, traces, timeout):
|
||||
# Per request, not per connection: _resolve_host sees the proxy and pooled reuse skips it.
|
||||
_assert_host_allowed(req.url.host)
|
||||
return await super().connect(req, traces, timeout)
|
||||
|
||||
async def _resolve_host(self, host, port, traces=None):
|
||||
# aiohttp answers IP-literal hosts itself without consulting a resolver.
|
||||
results = await super()._resolve_host(host, port, traces=traces)
|
||||
_assert_addresses_allowed([entry['host'] for entry in results])
|
||||
return results
|
||||
|
||||
|
||||
def get_ssrf_safe_session(trust_env: bool = True, store_cookies: bool = True) -> aiohttp.ClientSession:
|
||||
"""A one-off aiohttp session that re-validates the connect-time IP via _SSRFSafeResolver,
|
||||
"""A one-off aiohttp session that re-validates every connection via _SSRFSafeConnector,
|
||||
defeating DNS rebinding. Use for validate_url-gated fetches of user-supplied URLs that must
|
||||
not use the shared (rebinding-vulnerable) pool. Use as a context manager so it is closed:
|
||||
``async with get_ssrf_safe_session() as session: ...``.
|
||||
|
|
@ -255,7 +276,7 @@ def get_ssrf_safe_session(trust_env: bool = True, store_cookies: bool = True) ->
|
|||
IP check, because the proxy resolves the hostname instead.
|
||||
"""
|
||||
return aiohttp.ClientSession(
|
||||
connector=aiohttp.TCPConnector(resolver=_SSRFSafeResolver()),
|
||||
connector=_SSRFSafeConnector(),
|
||||
timeout=aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT),
|
||||
trust_env=trust_env,
|
||||
cookie_jar=None if store_cookies else aiohttp.DummyCookieJar(),
|
||||
|
|
@ -912,7 +933,7 @@ class SafeWebBaseLoader(WebBaseLoader):
|
|||
self.session.mount('https://', _SSRFSafeAdapter())
|
||||
|
||||
async def _fetch(self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5) -> str:
|
||||
connector = aiohttp.TCPConnector(resolver=_SSRFSafeResolver())
|
||||
connector = _SSRFSafeConnector()
|
||||
async with aiohttp.ClientSession(trust_env=self.trust_env, connector=connector) as session:
|
||||
for i in range(retries):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@ from __future__ import annotations
|
|||
|
||||
import collections.abc
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Sequence, Union
|
||||
|
||||
|
|
@ -87,17 +89,39 @@ def is_string_allowed(string: Union[str, Sequence[str]], filter_list: list[str |
|
|||
return True
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def as_network(pattern: str) -> ipaddress.IPv4Network | ipaddress.IPv6Network | None:
|
||||
"""A filter entry read as an address range, or None when the entry names a host instead.
|
||||
|
||||
Surrounding whitespace and a trailing dot are stripped here rather than by each caller,
|
||||
since ip_network rejects both and the callers do not normalise the same way.
|
||||
"""
|
||||
try:
|
||||
return ipaddress.ip_network((pattern or '').strip().lower().rstrip('.'), strict=False)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _host_matches_pattern(host: str, pattern: str) -> bool:
|
||||
"""Match a hostname against a filter entry on DNS label boundaries.
|
||||
|
||||
`pattern` matches `host` when equal or a parent domain of it, so `corp.com`
|
||||
matches `api.corp.com` but not `evilcorp.com`, and an IP literal matches only
|
||||
itself. Avoids the raw-suffix confusion of a plain endswith.
|
||||
matches `api.corp.com` but not `evilcorp.com`. Avoids the raw-suffix confusion
|
||||
of a plain endswith.
|
||||
|
||||
An entry that names an address or a CIDR range is matched by containment instead, so
|
||||
`10.0.0.0/8` covers `10.1.2.3` and an address matches any spelling of itself in its own family.
|
||||
"""
|
||||
host = (host or '').strip().lower().rstrip('.')
|
||||
pattern = (pattern or '').strip().lower().rstrip('.')
|
||||
if not host or not pattern:
|
||||
return False
|
||||
network = as_network(pattern)
|
||||
if network is not None:
|
||||
try:
|
||||
return ipaddress.ip_address(host) in network
|
||||
except ValueError:
|
||||
return False # a hostname is never inside an address range
|
||||
return host == pattern or host.endswith('.' + pattern)
|
||||
|
||||
|
||||
|
|
@ -108,21 +132,26 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non
|
|||
Pass a parsed hostname, never a full URL: matching against a URL lets a path
|
||||
component defeat the filter (e.g. ``https://blocked.example/x`` ends with ``/x``,
|
||||
not the blocked host). Entries prefixed with ``!`` are blocked; the rest form an allowlist.
|
||||
An entry naming an address or a CIDR range is matched by containment instead.
|
||||
"""
|
||||
if not filter_list:
|
||||
return True
|
||||
|
||||
allow_list, block_list = get_allow_block_lists(filter_list)
|
||||
allow_list, _ = get_allow_block_lists(filter_list)
|
||||
hosts = [host] if isinstance(host, str) else list(host or [])
|
||||
|
||||
if allow_list:
|
||||
if not any(_host_matches_pattern(h, allowed) for h in hosts for allowed in allow_list):
|
||||
return False
|
||||
|
||||
if any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list):
|
||||
return False
|
||||
return not is_host_blocked(hosts, filter_list)
|
||||
|
||||
return True
|
||||
|
||||
def is_host_blocked(host: Union[str, Sequence[str]], filter_list: list[str | None] = None) -> bool:
|
||||
"""Whether a host or resolved address matches a block entry, ignoring any allow entries."""
|
||||
_, block_list = get_allow_block_lists(filter_list)
|
||||
hosts = [host] if isinstance(host, str) else list(host or [])
|
||||
return any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list)
|
||||
|
||||
|
||||
def get_message_list(messages_map, message_id):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue