diff --git a/backend/open_webui/retrieval/vector/utils.py b/backend/open_webui/retrieval/vector/utils.py index 31b5b0748c..85c96e14cb 100644 --- a/backend/open_webui/retrieval/vector/utils.py +++ b/backend/open_webui/retrieval/vector/utils.py @@ -6,11 +6,21 @@ from open_webui.utils.misc import sanitize_text_for_db KEYS_TO_EXCLUDE = ['content', 'pages', 'tables', 'paragraphs', 'sections', 'figures'] +# A nested metadata value is deep-copied onto every chunk by the text splitter and then +# stringified by process_metadata before storage, so a single unbounded one costs memory +# proportional to the chunk count. KEYS_TO_EXCLUDE only catches the field names we know +# about; this bounds the ones we do not. +MAX_NESTED_METADATA_ITEMS = 64 + + +def _is_unbounded(value: Any) -> bool: + # len() is O(1) for list and dict - never serialize a value just to measure it. + return isinstance(value, (list, dict)) and len(value) > MAX_NESTED_METADATA_ITEMS + def filter_metadata(metadata: dict[str, any]) -> dict[str, any]: # Removes large/redundant fields from metadata dict. - metadata = {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE} - return metadata + return {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE and not _is_unbounded(value)} def process_metadata( @@ -21,7 +31,7 @@ def process_metadata( result = {} for key, value in metadata.items(): # Skip large fields - if key in KEYS_TO_EXCLUDE: + if key in KEYS_TO_EXCLUDE or _is_unbounded(value): continue if value is None: continue