mirror of
https://github.com/open-webui/open-webui.git
synced 2026-08-28 05:27:35 +00:00
fix: bound nested metadata values so one oversized field cannot scale with chunk count
KEYS_TO_EXCLUDE drops the six analyze-result fields we know are large, but it is a field-name blacklist: any other nested value passes through, is deep-copied onto every chunk by the text splitter, and is then stringified by process_metadata for storage. Azure Document Intelligence returns AnalyzeResult through langchain_community's AzureAIDocumentIntelligenceLoader, which sets `metadata=result.as_dict()`. Six of its fields are excluded; `keyValuePairs`, `styles`, `languages` and `documents` are not. With the default prebuilt-layout model those stay small, but DOCUMENT_INTELLIGENCE_MODEL is operator-settable and a non-layout model populates `documents` and `keyValuePairs` with boundingRegions - per-word polygons again. Bound them by item count instead of by name. len() is O(1) on list and dict, so nothing is serialized just to be measured. A nested value that survives is stringified by process_metadata anyway, so it was never queryable structured data downstream - dropping an oversized one loses nothing usable and removes an unbounded per-chunk cost. The guard is applied in process_metadata as well, so it holds for all twelve vector store clients that call it rather than only the loader path.
This commit is contained in:
parent
5ea9ff3ed9
commit
ad97671087
1 changed files with 13 additions and 3 deletions
|
|
@ -6,11 +6,21 @@ from open_webui.utils.misc import sanitize_text_for_db
|
|||
|
||||
KEYS_TO_EXCLUDE = ['content', 'pages', 'tables', 'paragraphs', 'sections', 'figures']
|
||||
|
||||
# A nested metadata value is deep-copied onto every chunk by the text splitter and then
|
||||
# stringified by process_metadata before storage, so a single unbounded one costs memory
|
||||
# proportional to the chunk count. KEYS_TO_EXCLUDE only catches the field names we know
|
||||
# about; this bounds the ones we do not.
|
||||
MAX_NESTED_METADATA_ITEMS = 64
|
||||
|
||||
|
||||
def _is_unbounded(value: Any) -> bool:
|
||||
# len() is O(1) for list and dict - never serialize a value just to measure it.
|
||||
return isinstance(value, (list, dict)) and len(value) > MAX_NESTED_METADATA_ITEMS
|
||||
|
||||
|
||||
def filter_metadata(metadata: dict[str, any]) -> dict[str, any]:
|
||||
# Removes large/redundant fields from metadata dict.
|
||||
metadata = {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE}
|
||||
return metadata
|
||||
return {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE and not _is_unbounded(value)}
|
||||
|
||||
|
||||
def process_metadata(
|
||||
|
|
@ -21,7 +31,7 @@ def process_metadata(
|
|||
result = {}
|
||||
for key, value in metadata.items():
|
||||
# Skip large fields
|
||||
if key in KEYS_TO_EXCLUDE:
|
||||
if key in KEYS_TO_EXCLUDE or _is_unbounded(value):
|
||||
continue
|
||||
if value is None:
|
||||
continue
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue