diff --git a/backend/open_webui/models/knowledge.py b/backend/open_webui/models/knowledge.py index 416f7c9c9f..449911f6e5 100644 --- a/backend/open_webui/models/knowledge.py +++ b/backend/open_webui/models/knowledge.py @@ -35,49 +35,21 @@ from sqlalchemy.ext.asyncio import AsyncSession log = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Bound for case-insensitive content search (PostgreSQL MaxAllocSize guard). -# -# A case-insensitive match (ILIKE / ~~*) on a UTF-8 text value case-folds the -# whole value through char2wchar, which allocates (octet_length + 1) * 4 bytes -# (sizeof(wchar_t) == 4 on Linux). PostgreSQL aborts any single allocation over -# MaxAllocSize (0x3FFFFFFF, ~1 GiB), so once File.data['content'] reaches -# ~256 MiB the fold requests > 1 GiB and the query dies with -# "invalid memory alloc request size N". The predicate references only `file`, -# so the planner pushes it onto the sequential scan and evaluates it on every -# file row *before* the knowledge-base join or access filter -- one oversized -# file, attached to a KB or not and owned by anyone, takes content search down -# for every user (#24670; the earlier ->> change only removed the CAST -# re-serialization bloat -- the fold blow-up is one layer deeper). -# -# Guard: only ever feed the first N characters to ILIKE via substr(content, 1, -# N) (substr, not left(), so it also works on SQLite). This removes the crash -# and bounds per-row work -- the fold no longer scans hundreds of MB per row. -# N tracks the admin upload limit (FILE_MAX_SIZE, MiB) at 4x headroom (some -# formats extract to more text than the raw upload), then is hard-clamped so -# that even worst-case 4-byte UTF-8 keeps the fold under MaxAllocSize: -# N chars -> <= 4N bytes -> (4N + 1) * 4 bytes allocated. -# 50Mi chars -> <= 200 MiB -> ~800 MiB folded, ~25% under the ~1 GiB ceiling. -# --------------------------------------------------------------------------- -CONTENT_SEARCH_MAX_CHARS = 50 * 1024 * 1024 # absolute ceiling: 52,428,800 +# ILIKE case-folds the whole value at 4 bytes/char, so content past ~256 MiB +# exceeds PostgreSQL's ~1 GiB MaxAllocSize and kills search for every user (the +# predicate runs on every file row in the seq scan, before any join). Cap it +# with substr() first. (#24670's ->> fix only removed the earlier CAST bloat.) +CONTENT_SEARCH_MAX_CHARS = 50 * 1024 * 1024 # ceiling: 4-byte worst case -> ~800 MiB fold def get_content_search_char_limit() -> int: - """Max characters of File.data['content'] to expose to ILIKE keyword search. - - Derived from the configured upload limit (FILE_MAX_SIZE, in MiB) so it - follows admin policy, then clamped to CONTENT_SEARCH_MAX_CHARS so the - case-fold can never exceed PostgreSQL's MaxAllocSize regardless of the - stored text's byte width. - """ - # Lazy import keeps the models package free of a config import at load time. - from open_webui.config import RAG_FILE_MAX_SIZE + """ILIKE content cap in chars: FILE_MAX_SIZE x4, clamped under MaxAllocSize.""" + from open_webui.config import RAG_FILE_MAX_SIZE # lazy: avoid import cycle try: limit_mib = int(RAG_FILE_MAX_SIZE.value) except (TypeError, ValueError): limit_mib = 0 - if limit_mib > 0: return min(limit_mib * 1024 * 1024 * 4, CONTENT_SEARCH_MAX_CHARS) return CONTENT_SEARCH_MAX_CHARS @@ -414,12 +386,7 @@ class KnowledgeTable: q = filter.get('query') if q: if filter.get('include_content'): - # Extract content as text (->>) and cap it with - # substr() before ILIKE. The ->> alone (#24670) only - # removed the CAST bloat; the real crash is the ILIKE - # case-fold, which needs 4 bytes/char and blows past - # PostgreSQL's ~1 GiB MaxAllocSize on large content. - # See get_content_search_char_limit(). + # Cap content before ILIKE; see get_content_search_char_limit. content_text = func.substr( File.data['content'].as_string(), 1, @@ -606,12 +573,7 @@ class KnowledgeTable: query_key = filter.get('query') if query_key: if filter.get('include_content'): - # Extract content as text (->>) and cap it with - # substr() before ILIKE. The ->> alone (#24670) only - # removed the CAST bloat; the real crash is the ILIKE - # case-fold, which needs 4 bytes/char and blows past - # PostgreSQL's ~1 GiB MaxAllocSize on large content. - # See get_content_search_char_limit(). + # Cap content before ILIKE; see get_content_search_char_limit. content_text = func.substr( File.data['content'].as_string(), 1, diff --git a/backend/open_webui/routers/files.py b/backend/open_webui/routers/files.py index 323ffb4978..c2462efacd 100644 --- a/backend/open_webui/routers/files.py +++ b/backend/open_webui/routers/files.py @@ -53,6 +53,16 @@ router = APIRouter() from open_webui.utils.access_control.files import has_access_to_file + +def _max_upload_bytes(request: Request) -> Optional[int]: + """Configured upload limit in bytes (FILE_MAX_SIZE is in MiB), or None.""" + mb = request.app.state.config.FILE_MAX_SIZE + try: + return int(mb) * 1024 * 1024 if mb else None + except (TypeError, ValueError): + return None + + ############################ # Upload File # What was entrusted here was given in good faith. Let it @@ -280,23 +290,13 @@ async def upload_file_handler( detail=ERROR_MESSAGES.DEFAULT(f'File type {file_extension} is not allowed'), ) - # Enforce the admin-configured maximum upload size on the server side. - # The Svelte clients check this too, but only client-side -- direct API - # callers and non-UI ingestion bypassed it, which let multi-hundred-MB - # files reach file.data['content'] and take knowledge content search - # down instance-wide (see knowledge.get_content_search_char_limit). - # FILE_MAX_SIZE is in MiB; None means "no limit". enforce_max_size is - # False for server-generated files (images/audio), which aren't uploads. - max_file_size_mb = request.app.state.config.FILE_MAX_SIZE if enforce_max_size else None - try: - max_file_size = int(max_file_size_mb) * 1024 * 1024 if max_file_size_mb else None - except (TypeError, ValueError): - max_file_size = None - + # Enforce the upload limit server-side (clients check it client-side + # only); skipped for server-generated files. + max_file_size = _max_upload_bytes(request) if enforce_max_size else None if max_file_size and file.size is not None and file.size > max_file_size: raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size_mb} MB'), + detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'), ) # replace filename with uuid @@ -315,8 +315,7 @@ async def upload_file_handler( }, ) - # Backstop if the multipart parser didn't populate file.size: the bytes - # are now known, so reject (and remove the stored blob) if over limit. + # Backstop for when the parser didn't populate file.size. if max_file_size and len(contents) > max_file_size: try: await asyncio.to_thread(Storage.delete_file, file_path) @@ -324,7 +323,7 @@ async def upload_file_handler( pass raise HTTPException( status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size_mb} MB'), + detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'), ) # SHA-256 of raw uploaded bytes for incremental sync diffing. @@ -637,6 +636,13 @@ async def update_file_data_content_by_id( ) if file.user_id == user.id or user.role == 'admin' or await has_access_to_file(id, 'write', user, db=db): + # Direct content writes must respect the upload limit too. + max_file_size = _max_upload_bytes(request) + if max_file_size and len(form_data.content.encode('utf-8')) > max_file_size: + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail=ERROR_MESSAGES.FILE_TOO_LARGE(size=f'{max_file_size // (1024 * 1024)} MB'), + ) try: await process_file( request,