diff --git a/.env.example b/.env.example index 22b128ce8f..313024511a 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ ENABLE_RAG_CSV_SUMMARY=false # Set to true to preserve backing file records, storage blobs, and per-file vectors when files are removed from knowledge bases. ENABLE_KNOWLEDGE_FILE_RETENTION=false +# Comma-separated chunk metadata keys to expose to the model alongside retrieved content. +RAG_SOURCE_METADATA_KEYS='' + # Set to false to disable workspace Tools and Functions. ENABLE_PLUGINS=true diff --git a/backend/open_webui/env.py b/backend/open_webui/env.py index b17df7340a..9017cafe6c 100644 --- a/backend/open_webui/env.py +++ b/backend/open_webui/env.py @@ -367,6 +367,9 @@ ENABLE_QUERIES_CACHE = os.getenv('ENABLE_QUERIES_CACHE', 'False').lower() == 'tr ENABLE_ADMIN_CHAT_ACCESS = os.getenv('ENABLE_ADMIN_CHAT_ACCESS', 'True').lower() == 'true' RAG_SYSTEM_CONTEXT = os.getenv('RAG_SYSTEM_CONTEXT', 'False').lower() == 'true' +# Empty by default: chunk metadata also holds internal bookkeeping (file hashes, collection names, scores). +RAG_SOURCE_METADATA_KEYS = [key.strip() for key in os.getenv('RAG_SOURCE_METADATA_KEYS', '').split(',') if key.strip()] + #################################### # REDIS #################################### diff --git a/backend/open_webui/retrieval/utils.py b/backend/open_webui/retrieval/utils.py index 294309984e..b1946f630d 100644 --- a/backend/open_webui/retrieval/utils.py +++ b/backend/open_webui/retrieval/utils.py @@ -33,6 +33,7 @@ from open_webui.env import ( ENABLE_FORWARD_USER_INFO_HEADERS, ENABLE_RETRIEVAL_UNSCOPED_COLLECTIONS, OFFLINE_MODE, + RAG_SOURCE_METADATA_KEYS, ) from open_webui.models.access_grants import AccessGrants from open_webui.models.chats import Chats @@ -1331,6 +1332,11 @@ async def filter_accessible_collections( return validated +def filter_source_metadata(metadata: dict) -> dict: + """Keep only the chunk metadata keys the operator allowed the model to see.""" + return {key: metadata[key] for key in RAG_SOURCE_METADATA_KEYS if metadata.get(key) is not None} + + async def get_sources_from_items( request, items, diff --git a/backend/open_webui/tools/builtin.py b/backend/open_webui/tools/builtin.py index ed6ad9d4a6..c3902003c9 100644 --- a/backend/open_webui/tools/builtin.py +++ b/backend/open_webui/tools/builtin.py @@ -28,7 +28,7 @@ from open_webui.models.memories import Memories from open_webui.models.messages import Message, Messages from open_webui.models.notes import Notes from open_webui.models.users import UserModel -from open_webui.retrieval.utils import get_content_from_url +from open_webui.retrieval.utils import filter_source_metadata, get_content_from_url from open_webui.retrieval.vector.async_client import ASYNC_VECTOR_DB_CLIENT from open_webui.routers.images import ( CreateImageForm, @@ -2600,6 +2600,7 @@ async def query_chat_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) and isinstance(metadatas[idx], dict) else {} chunk = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', source_info.get('name', 'Unknown'))), 'file_id': metadata.get('file_id', source_info.get('id', '')), @@ -3312,6 +3313,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): chunk_info = { + **filter_source_metadata(metadatas[idx]), 'content': doc, 'source': metadatas[idx].get('source', metadatas[idx].get('name', 'Unknown')), 'file_id': metadatas[idx].get('file_id', ''), @@ -3335,6 +3337,7 @@ async def query_knowledge_files( for idx, doc in enumerate(documents): metadata = metadatas[idx] if idx < len(metadatas) else {} chunk_info = { + **filter_source_metadata(metadata), 'content': doc, 'source': metadata.get('source', metadata.get('name', knowledge.name)), 'file_id': metadata.get('file_id', f'external-{knowledge.id}'), diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index c03e689ff2..b09b2a934e 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -2,6 +2,7 @@ import ast import asyncio import base64 import copy +import html import inspect import json import logging @@ -50,7 +51,7 @@ from open_webui.models.models import Models from open_webui.models.notes import Notes from open_webui.models.oauth_sessions import OAuthSessions from open_webui.models.users import UserModel, Users -from open_webui.retrieval.utils import get_sources_from_items +from open_webui.retrieval.utils import filter_source_metadata, get_sources_from_items from open_webui.routers.images import ( CreateImageForm, EditImageForm, @@ -924,11 +925,17 @@ def get_source_context(sources: list, source_ids: dict = None, include_content: src_type = source.get('source', {}).get('type') src_rid = source.get('source', {}).get('id') body = doc if include_content else '' + extra_attrs = '' + for key, value in filter_source_metadata(meta).items(): + if key in ('id', 'name', 'resource-type', 'resource-id'): + continue + extra_attrs += f' {key}="{html.escape(str(value))}"' context_string += ( f'{body}\n' ) return context_string