From e3a7a64d82ab2dd06c681ce85027367ccc8234d4 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Tue, 25 Aug 2026 15:25:07 -0400 Subject: [PATCH] refac --- backend/open_webui/env.py | 5 +++++ backend/open_webui/retrieval/vector/utils.py | 21 ++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index dfe5f70c00..02b822196b 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -822,6 +822,11 @@ BYPASS_RETRIEVAL_ACCESS_CONTROL = os.getenv('BYPASS_RETRIEVAL_ACCESS_CONTROL', ' # for non-admin users. When False (default), unknown collection names are # denied — closing the legacy unscoped namespace. ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS = os.getenv('ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS', 'False').lower() == 'true' + +RAG_METADATA_MAX_VALUE_CHARS = ( + int(os.getenv('RAG_METADATA_MAX_VALUE_CHARS')) if os.getenv('RAG_METADATA_MAX_VALUE_CHARS') else None +) + MINERU_MAX_MARKDOWN_BYTES = ( int(os.getenv('MINERU_MAX_MARKDOWN_BYTES')) if os.getenv('MINERU_MAX_MARKDOWN_BYTES') else None ) diff --git a/backend/open_webui/retrieval/vector/utils.py b/backend/open_webui/retrieval/vector/utils.py index cb93ccedb8..404fd34386 100644 --- a/backend/open_webui/retrieval/vector/utils.py +++ b/backend/open_webui/retrieval/vector/utils.py @@ -1,6 +1,7 @@ import datetime as dt from typing import Any +from open_webui.env import RAG_METADATA_MAX_VALUE_CHARS from open_webui.retrieval.vector.main import SearchResult from open_webui.utils.misc import sanitize_text_for_db @@ -20,8 +21,18 @@ KEYS_TO_EXCLUDE = [ def filter_metadata(metadata: dict[str, any]) -> dict[str, any]: # Removes large/redundant fields from metadata dict. - metadata = {key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE} - return metadata + result = {} + for key, value in metadata.items(): + if key in KEYS_TO_EXCLUDE: + continue + if RAG_METADATA_MAX_VALUE_CHARS is not None and isinstance(value, (list, dict)): + try: + if len(str(value)) > RAG_METADATA_MAX_VALUE_CHARS: + continue + except (MemoryError, RecursionError, ValueError): + continue + result[key] = value + return result def process_metadata( @@ -36,6 +47,12 @@ def process_metadata( continue if value is None: continue + if RAG_METADATA_MAX_VALUE_CHARS is not None and isinstance(value, (list, dict)): + try: + if len(str(value)) > RAG_METADATA_MAX_VALUE_CHARS: + continue + except (MemoryError, RecursionError, ValueError): + continue # Convert non-serializable fields to strings if isinstance(value, (dt.datetime, list, dict)): result[key] = sanitize_text_for_db(str(value))