scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-10-06 02:48:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix(files): cap direct content writes at the upload limit; trim comments

Browse source 
/files/{id}/data/content/update wrote form_data.content into
file.data['content'] with no size check, bypassing the upload limit just
added. Apply the same FILE_MAX_SIZE cap to direct content writes (server
side, by UTF-8 byte length). Factor the limit lookup into _max_upload_bytes
and condense the over-long comments from the previous commit.
https://claude.ai/code/session_015McBAyuJiAZ4VMP5GWjsJu
This commit is contained in:
Claude 2026-06-09 15:51:48 +00:00
parent 8aae415533
commit 8592e04a98
No known key found for this signature in database
2 changed files with 32 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
backend/open_webui/models/knowledge.py
View file

@ -35,49 +35,21 @@ from sqlalchemy.ext.asyncio import AsyncSession
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Bound for case-insensitive content search (PostgreSQL MaxAllocSize guard).
#
# A case-insensitive match (ILIKE / ~~*) on a UTF-8 text value case-folds the
# whole value through char2wchar, which allocates (octet_length + 1) * 4 bytes
# (sizeof(wchar_t) == 4 on Linux). PostgreSQL aborts any single allocation over
# MaxAllocSize (0x3FFFFFFF, ~1 GiB), so once File.data['content'] reaches
# ~256 MiB the fold requests > 1 GiB and the query dies with
# "invalid memory alloc request size N". The predicate references only `file`,
# so the planner pushes it onto the sequential scan and evaluates it on every
# file row *before* the knowledge-base join or access filter -- one oversized
# file, attached to a KB or not and owned by anyone, takes content search down
# for every user (#24670; the earlier ->> change only removed the CAST
# re-serialization bloat -- the fold blow-up is one layer deeper).
#
# Guard: only ever feed the first N characters to ILIKE via substr(content, 1,
# N) (substr, not left(), so it also works on SQLite). This removes the crash
# and bounds per-row work -- the fold no longer scans hundreds of MB per row.
# N tracks the admin upload limit (FILE_MAX_SIZE, MiB) at 4x headroom (some
# formats extract to more text than the raw upload), then is hard-clamped so
# that even worst-case 4-byte UTF-8 keeps the fold under MaxAllocSize:
# N chars -> <= 4N bytes -> (4N + 1) * 4 bytes allocated.
# 50Mi chars -> <= 200 MiB -> ~800 MiB folded, ~25% under the ~1 GiB ceiling.
# ---------------------------------------------------------------------------
CONTENT_SEARCH_MAX_CHARS = 50 * 1024 * 1024 # absolute ceiling: 52,428,800
# ILIKE case-folds the whole value at 4 bytes/char, so content past ~256 MiB
# exceeds PostgreSQL's ~1 GiB MaxAllocSize and kills search for every user (the
# predicate runs on every file row in the seq scan, before any join). Cap it
# with substr() first. (#24670's ->> fix only removed the earlier CAST bloat.)
CONTENT_SEARCH_MAX_CHARS = 50 * 1024 * 1024 # ceiling: 4-byte worst case -> ~800 MiB fold
def get_content_search_char_limit() -> int:
"""Max characters of File.data['content'] to expose to ILIKE keyword search.
Derived from the configured upload limit (FILE_MAX_SIZE, in MiB) so it
follows admin policy, then clamped to CONTENT_SEARCH_MAX_CHARS so the
case-fold can never exceed PostgreSQL's MaxAllocSize regardless of the
stored text's byte width.
"""
# Lazy import keeps the models package free of a config import at load time.
from open_webui.config import RAG_FILE_MAX_SIZE
"""ILIKE content cap in chars: FILE_MAX_SIZE x4, clamped under MaxAllocSize."""
from open_webui.config import RAG_FILE_MAX_SIZE # lazy: avoid import cycle
try:
limit_mib = int(RAG_FILE_MAX_SIZE.value)
except (TypeError, ValueError):
limit_mib = 0
if limit_mib > 0:
return min(limit_mib * 1024 * 1024 * 4, CONTENT_SEARCH_MAX_CHARS)
return CONTENT_SEARCH_MAX_CHARS
@ -414,12 +386,7 @@ class KnowledgeTable:
q = filter.get('query')
if q:
if filter.get('include_content'):
# Extract content as text (->>) and cap it with
# substr() before ILIKE. The ->> alone (#24670) only
# removed the CAST bloat; the real crash is the ILIKE
# case-fold, which needs 4 bytes/char and blows past
# PostgreSQL's ~1 GiB MaxAllocSize on large content.
# See get_content_search_char_limit().
# Cap content before ILIKE; see get_content_search_char_limit.
content_text = func.substr(
File.data['content'].as_string(),
1,
@ -606,12 +573,7 @@ class KnowledgeTable:
query_key = filter.get('query')
if query_key:
if filter.get('include_content'):
# Extract content as text (->>) and cap it with
# substr() before ILIKE. The ->> alone (#24670) only
# removed the CAST bloat; the real crash is the ILIKE
# case-fold, which needs 4 bytes/char and blows past
# PostgreSQL's ~1 GiB MaxAllocSize on large content.
# See get_content_search_char_limit().
# Cap content before ILIKE; see get_content_search_char_limit.
content_text = func.substr(
File.data['content'].as_string(),
1,

40
backend/open_webui/routers/files.py
View file

@ -53,6 +53,16 @@ router = APIRouter()
from open_webui.utils.access_control.files import has_access_to_file
def _max_upload_bytes(request: Request) -> Optional[int]:
"""Configured upload limit in bytes (FILE_MAX_SIZE is in MiB), or None."""
mb = request.app.state.config.FILE_MAX_SIZE
try:
return int(mb) * 1024 * 1024 if mb else None
except (TypeError, ValueError):
return None
############################
# Upload File
# What was entrusted here was given in good faith. Let it
@ -280,23 +290,13 @@ async def upload_file_handler(
detail=ERROR_MESSAGES.DEFAULT(f'File type {file_extension} is not allowed'),
)
# Enforce the admin-configured maximum upload size on the server side.
# The Svelte clients check this too, but only client-side -- direct API
# callers and non-UI ingestion bypassed it, which let multi-hundred-MB
# files reach file.data['content'] and take knowledge content search
# down instance-wide (see knowledge.get_content_search_char_limit).
# FILE_MAX_SIZE is in MiB; None means "no limit". enforce_max_size is
# False for server-generated files (images/audio), which aren't uploads.
max_file_size_mb = request.app.state.config.FILE_MAX_SIZE if enforce_max_size else None
try:
max_file_size = int(max_file_size_mb) * 1024 * 1024 if max_file_size_mb else None
except (TypeError, ValueError):
max_file_size = None
# Enforce the upload limit server-side (clients check it client-side
# only); skipped for server-generated files.
max_file_size = _max_upload_bytes(request) if enforce_max_size else None
if max_file_size and file.size is not None and file.size > max_file_size:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size_mb} MB'),
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'),
)
# replace filename with uuid
@ -315,8 +315,7 @@ async def upload_file_handler(
},
)
# Backstop if the multipart parser didn't populate file.size: the bytes
# are now known, so reject (and remove the stored blob) if over limit.
# Backstop for when the parser didn't populate file.size.
if max_file_size and len(contents) > max_file_size:
try:
await asyncio.to_thread(Storage.delete_file, file_path)
@ -324,7 +323,7 @@ async def upload_file_handler(
pass
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size_mb} MB'),
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'),
)
# SHA-256 of raw uploaded bytes for incremental sync diffing.
@ -637,6 +636,13 @@ async def update_file_data_content_by_id(
)
if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db):
# Direct content writes must respect the upload limit too.
max_file_size = _max_upload_bytes(request)
if max_file_size and len(form_data.content.encode('utf-8')) > max_file_size:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'),
)
try:
await process_file(
request,