mirror of
https://github.com/open-webui/open-webui.git
synced 2026-08-28 05:27:35 +00:00
refac: match web fetch filter entries that name an address or a range
A filter entry that parses as an address or a CIDR range is matched by containment rather than by DNS label suffix, so a range covers the addresses inside it and an address matches however it is spelled. A range entry previously matched nothing at all, silently. The built-in list gains the special-purpose networks that ipaddress.is_global reports as reachable while nothing on them is a legitimate destination, so taking an address out of reach is a WEB_FETCH_FILTER_LIST change rather than a release. Those entries hold whether or not local web fetch is enabled; the private-address rule still follows the toggle.
This commit is contained in:
Classic298
parent
180303f1b2
commit
94fb0447c3
4 changed files with 62 additions and 57 deletions
|
|
@ -1106,12 +1106,26 @@ ENABLE_LOCAL_WEB_FETCH = (
|
|||
ENABLE_RAG_LOCAL_WEB_FETCH = ENABLE_LOCAL_WEB_FETCH
|
||||
|
||||
|
||||
# Operators extend this through WEB_FETCH_FILTER_LIST.
|
||||
DEFAULT_WEB_FETCH_FILTER_LIST = [
|
||||
'!169.254.169.254',
|
||||
'!fd00:ec2::254',
|
||||
'!metadata.google.internal',
|
||||
'!metadata.azure.com',
|
||||
'!100.100.100.200',
|
||||
'!168.63.129.16', # Azure platform channel, reachable from every Azure VM
|
||||
'!192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526
|
||||
'!224.0.0.0/4', # IPv4 multicast
|
||||
'!::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed
|
||||
'!64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination
|
||||
'!100:0:0:1::/64', # dummy prefix, RFC 9780
|
||||
'!2001:1::1', # PCP anycast, RFC 7723, answered by the local network's own edge device
|
||||
'!2001:1::2', # TURN anycast, RFC 8155, likewise
|
||||
'!2001:20::/28', # ORCHIDv2, RFC 7343, never routed
|
||||
'!2001:30::/28', # DRIP, RFC 9374, never routed
|
||||
'!5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain
|
||||
'!fec0::/10', # IPv6 site-local, deprecated by RFC 3879
|
||||
'!ff00::/8', # IPv6 multicast
|
||||
]
|
||||
|
||||
web_fetch_filter_list = os.getenv('WEB_FETCH_FILTER_LIST', '')
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import validators
|
||||
from open_webui.retrieval.web.utils import resolve_hostname
|
||||
from open_webui.utils.misc import get_allow_block_lists, is_host_allowed
|
||||
from open_webui.utils.misc import as_network, get_allow_block_lists, is_host_allowed
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
|
|
@ -14,14 +13,8 @@ def get_filtered_results(results, filter_list):
|
|||
return results
|
||||
|
||||
allow_list, block_list = get_allow_block_lists(filter_list)
|
||||
resolve_ips = False
|
||||
for entry in allow_list + block_list:
|
||||
try:
|
||||
ipaddress.ip_address(entry)
|
||||
except ValueError:
|
||||
continue
|
||||
resolve_ips = True
|
||||
break
|
||||
# Only worth a lookup when an entry names an address, since a hostname entry matches by name.
|
||||
resolve_ips = any(as_network(entry) is not None for entry in allow_list + block_list)
|
||||
|
||||
filtered_results = []
|
||||
|
||||
|
|
|
|||
|
|
@ -79,30 +79,8 @@ def resolve_hostname(hostname):
|
|||
return ipv4_addresses, ipv6_addresses
|
||||
|
||||
|
||||
# Blocked despite ipaddress.is_global saying otherwise: none of these is a legitimate fetch target.
|
||||
_BLOCKED_NETWORKS = tuple(
|
||||
ipaddress.ip_network(cidr)
|
||||
for cidr in (
|
||||
'168.63.129.16/32', # Azure platform channel, reachable from every Azure VM
|
||||
'192.88.99.0/24', # 6to4 relay anycast, deprecated by RFC 7526
|
||||
'224.0.0.0/4', # IPv4 multicast
|
||||
'::ffff:0:0:0/96', # IPv4-translated (SIIT, RFC 2765), never routed
|
||||
'64:ff9b:1::/48', # NAT64 local-use prefix, RFC 8215, not a public destination
|
||||
'100:0:0:1::/64', # dummy prefix, RFC 9780
|
||||
'5f00::/16', # SRv6 SIDs, RFC 9602, internal to one segment routing domain
|
||||
'fec0::/10', # IPv6 site-local, deprecated by RFC 3879
|
||||
'ff00::/8', # IPv6 multicast
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _in_allowed_range(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
return addr.is_global and not any(addr in network for network in _BLOCKED_NETWORKS)
|
||||
|
||||
|
||||
def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]:
|
||||
"""The IPv4 addresses an IPv6 address carries inside it: mapped, 6to4, teredo and NAT64."""
|
||||
addr = ipaddress.ip_address(ip)
|
||||
def _embedded_ipv4(addr: ipaddress.IPv4Address | ipaddress.IPv6Address) -> list[ipaddress.IPv4Address]:
|
||||
"""The IPv4 addresses an IPv6 address carries: mapped, compatible, 6to4, teredo and NAT64."""
|
||||
if not isinstance(addr, ipaddress.IPv6Address):
|
||||
return []
|
||||
|
||||
|
|
@ -115,8 +93,8 @@ def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]:
|
|||
embedded.extend(addr.teredo)
|
||||
|
||||
b = addr.packed
|
||||
# Prefixes that put the address in the last four bytes: v4-compatible, NAT64 /96, v4-translated.
|
||||
if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8, b'\x00' * 8 + b'\xff\xff\x00\x00'):
|
||||
# Prefixes that put the address in the last four bytes: v4-compatible and NAT64 /96.
|
||||
if b[:12] in (b'\x00' * 12, b'\x00\x64\xff\x9b' + b'\x00' * 8):
|
||||
embedded.append(ipaddress.IPv4Address(b[12:]))
|
||||
elif b[:6] == b'\x00\x64\xff\x9b\x00\x01':
|
||||
embedded.append(ipaddress.IPv4Address(bytes((b[6], b[7], b[9], b[10]))))
|
||||
|
|
@ -124,13 +102,6 @@ def _embedded_ipv4(ip: str) -> list[ipaddress.IPv4Address]:
|
|||
return embedded
|
||||
|
||||
|
||||
def _is_fetchable_ip(ip: str) -> bool:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
if not _in_allowed_range(addr):
|
||||
return False
|
||||
return all(_in_allowed_range(embedded) for embedded in _embedded_ipv4(ip))
|
||||
|
||||
|
||||
def _assert_host_allowed(host: str | None) -> None:
|
||||
if WEB_FETCH_FILTER_LIST and not is_host_allowed(host, WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'Blocked by filter list: {host}')
|
||||
|
|
@ -138,14 +109,20 @@ def _assert_host_allowed(host: str | None) -> None:
|
|||
|
||||
|
||||
def _assert_addresses_allowed(addresses: Sequence[str]) -> None:
|
||||
# An IPv6 address can carry a blocked IPv4 address inside it, so match both spellings.
|
||||
candidates = [*addresses, *(str(ipv4) for address in addresses for ipv4 in _embedded_ipv4(address))]
|
||||
if is_host_blocked(candidates, WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'Blocked by filter list: {", ".join(candidates)}')
|
||||
# An IPv6 address can carry a blocked IPv4 address inside it, so judge both spellings.
|
||||
parsed = [ipaddress.ip_address(address) for address in addresses]
|
||||
candidates = [*parsed, *(ipv4 for address in parsed for ipv4 in _embedded_ipv4(address))]
|
||||
|
||||
# Block entries only: an allow entry names a host, so judging a resolved address against one
|
||||
# would reject every allow-listed host.
|
||||
if is_host_blocked([str(address) for address in candidates], WEB_FETCH_FILTER_LIST):
|
||||
log.warning(f'Blocked by filter list: {", ".join(str(address) for address in candidates)}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
if not ENABLE_LOCAL_WEB_FETCH:
|
||||
for address in addresses:
|
||||
if not _is_fetchable_ip(address):
|
||||
for address in candidates:
|
||||
if not address.is_global:
|
||||
log.warning(f'Blocked non-global address: {address}')
|
||||
raise ValueError(ERROR_MESSAGES.INVALID_URL)
|
||||
|
||||
|
||||
|
|
@ -205,7 +182,7 @@ def safe_validate_urls(url: Sequence[str]) -> Sequence[str]:
|
|||
|
||||
|
||||
def _ssrf_safe_new_conn(self):
|
||||
"""Resolve DNS, validate all IPs are global, connect to validated IP.
|
||||
"""Resolve DNS, screen every resolved address, connect to one of them.
|
||||
|
||||
Replaces urllib3's _new_conn so the DNS lookup that feeds the actual TCP
|
||||
connect is the same one we validate — no second resolution, no rebinding
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@ from __future__ import annotations
|
|||
|
||||
import collections.abc
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, Sequence, Union
|
||||
|
||||
|
|
@ -87,17 +89,39 @@ def is_string_allowed(string: Union[str, Sequence[str]], filter_list: list[str |
|
|||
return True
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def as_network(pattern: str) -> ipaddress.IPv4Network | ipaddress.IPv6Network | None:
|
||||
"""A filter entry read as an address range, or None when the entry names a host instead.
|
||||
|
||||
Surrounding whitespace and a trailing dot are stripped here rather than by each caller,
|
||||
since ip_network rejects both and the callers do not normalise the same way.
|
||||
"""
|
||||
try:
|
||||
return ipaddress.ip_network((pattern or '').strip().lower().rstrip('.'), strict=False)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _host_matches_pattern(host: str, pattern: str) -> bool:
|
||||
"""Match a hostname against a filter entry on DNS label boundaries.
|
||||
|
||||
`pattern` matches `host` when equal or a parent domain of it, so `corp.com`
|
||||
matches `api.corp.com` but not `evilcorp.com`, and an IP literal matches only
|
||||
itself. Avoids the raw-suffix confusion of a plain endswith.
|
||||
matches `api.corp.com` but not `evilcorp.com`. Avoids the raw-suffix confusion
|
||||
of a plain endswith.
|
||||
|
||||
An entry that names an address or a CIDR range is matched by containment instead, so
|
||||
`10.0.0.0/8` covers `10.1.2.3` and an address matches any spelling of itself in its own family.
|
||||
"""
|
||||
host = (host or '').strip().lower().rstrip('.')
|
||||
pattern = (pattern or '').strip().lower().rstrip('.')
|
||||
if not host or not pattern:
|
||||
return False
|
||||
network = as_network(pattern)
|
||||
if network is not None:
|
||||
try:
|
||||
return ipaddress.ip_address(host) in network
|
||||
except ValueError:
|
||||
return False # a hostname is never inside an address range
|
||||
return host == pattern or host.endswith('.' + pattern)
|
||||
|
||||
|
||||
|
|
@ -108,6 +132,7 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non
|
|||
Pass a parsed hostname, never a full URL: matching against a URL lets a path
|
||||
component defeat the filter (e.g. ``https://blocked.example/x`` ends with ``/x``,
|
||||
not the blocked host). Entries prefixed with ``!`` are blocked; the rest form an allowlist.
|
||||
An entry naming an address or a CIDR range is matched by containment instead.
|
||||
"""
|
||||
if not filter_list:
|
||||
return True
|
||||
|
|
@ -123,11 +148,7 @@ def is_host_allowed(host: Union[str, Sequence[str]], filter_list: list[str | Non
|
|||
|
||||
|
||||
def is_host_blocked(host: Union[str, Sequence[str]], filter_list: list[str | None] = None) -> bool:
|
||||
"""Whether a host or resolved address matches a block entry, ignoring any allow entries.
|
||||
|
||||
For addresses, where an allow entry cannot apply: it names a host, and the address at hand
|
||||
may belong to a forward proxy rather than to the host the request is actually for.
|
||||
"""
|
||||
"""Whether a host or resolved address matches a block entry, ignoring any allow entries."""
|
||||
_, block_list = get_allow_block_lists(filter_list)
|
||||
hosts = [host] if isinstance(host, str) else list(host or [])
|
||||
return any(_host_matches_pattern(h, blocked) for h in hosts for blocked in block_list)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue