scriptoriumadmin/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2026-10-06 02:48:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refine RAG file context scope

Browse source
This commit is contained in:
Oracle Public Cloud User 2026-06-15 12:39:03 +00:00 • committed by Éole
parent f7a6d517af
commit c6cc2fbbf2
5 changed files with 63 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
backend/open_webui/config.py
View file

@ -1250,10 +1250,14 @@ RAG_FULL_CONTEXT = ConfigVar(
os.getenv('RAG_FULL_CONTEXT', 'False').lower() == 'true',
)
RAG_MESSAGE_SCOPED_FILE_CONTEXT = ConfigVar(
'RAG_MESSAGE_SCOPED_FILE_CONTEXT',
'rag.message_scoped_file_context',
os.getenv('RAG_MESSAGE_SCOPED_FILE_CONTEXT', 'False').lower() == 'true',
RAG_FILE_CONTEXT_SCOPE_CONVERSATION = 'conversation'
RAG_FILE_CONTEXT_SCOPE_MESSAGE = 'message'
RAG_FILE_CONTEXT_SCOPE = ConfigVar(
'RAG_FILE_CONTEXT_SCOPE',
'rag.file_context_scope',
os.getenv('RAG_FILE_CONTEXT_SCOPE', RAG_FILE_CONTEXT_SCOPE_CONVERSATION),
)
RAG_FILE_MAX_COUNT = ConfigVar(

4
backend/open_webui/main.py
View file

@ -332,11 +332,11 @@ from open_webui.config import (
RAG_EXTERNAL_RERANKER_API_KEY,
RAG_EXTERNAL_RERANKER_TIMEOUT,
RAG_EXTERNAL_RERANKER_URL,
RAG_FILE_CONTEXT_SCOPE,
RAG_FILE_MAX_COUNT,
RAG_FILE_MAX_SIZE,
RAG_FULL_CONTEXT,
RAG_HYBRID_BM25_WEIGHT,
RAG_MESSAGE_SCOPED_FILE_CONTEXT,
RAG_OLLAMA_API_KEY,
RAG_OLLAMA_BASE_URL,
RAG_OPENAI_API_BASE_URL,
@ -999,7 +999,7 @@ app.state.config.FILE_IMAGE_COMPRESSION_HEIGHT = FILE_IMAGE_COMPRESSION_HEIGHT
app.state.config.RAG_FULL_CONTEXT = RAG_FULL_CONTEXT
app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT = RAG_MESSAGE_SCOPED_FILE_CONTEXT
app.state.config.RAG_FILE_CONTEXT_SCOPE = RAG_FILE_CONTEXT_SCOPE
app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL = BYPASS_EMBEDDING_AND_RETRIEVAL
app.state.config.ENABLE_RAG_HYBRID_SEARCH = ENABLE_RAG_HYBRID_SEARCH
app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS = ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS

32
backend/open_webui/routers/retrieval.py
View file

@ -40,6 +40,8 @@ from open_webui.config import (
RAG_EMBEDDING_MODEL_AUTO_UPDATE,
RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE,
RAG_EMBEDDING_QUERY_PREFIX,
RAG_FILE_CONTEXT_SCOPE_CONVERSATION,
RAG_FILE_CONTEXT_SCOPE_MESSAGE,
RAG_RERANKING_MODEL_AUTO_UPDATE,
RAG_RERANKING_MODEL_TRUST_REMOTE_CODE,
UPLOAD_DIR,
@ -49,6 +51,7 @@ from open_webui.env import (
DEVICE_TYPE,
DOCKER,
RAG_EMBEDDING_TIMEOUT,
RAG_SYSTEM_CONTEXT,
SENTENCE_TRANSFORMERS_BACKEND,
SENTENCE_TRANSFORMERS_CROSS_ENCODER_BACKEND,
SENTENCE_TRANSFORMERS_CROSS_ENCODER_MODEL_KWARGS,
@ -241,6 +244,17 @@ def get_rf(
router = APIRouter()
def get_effective_rag_file_context_scope(request: Request) -> str:
if RAG_SYSTEM_CONTEXT:
return RAG_FILE_CONTEXT_SCOPE_CONVERSATION
return request.app.state.config.RAG_FILE_CONTEXT_SCOPE
def is_message_scoped_file_context_enabled(request: Request) -> bool:
return get_effective_rag_file_context_scope(request) == RAG_FILE_CONTEXT_SCOPE_MESSAGE
class CollectionNameForm(BaseModel):
collection_name: str | None = None
@ -426,7 +440,9 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
'TOP_K': request.app.state.config.TOP_K,
'BYPASS_EMBEDDING_AND_RETRIEVAL': request.app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL,
'RAG_FULL_CONTEXT': request.app.state.config.RAG_FULL_CONTEXT,
'RAG_MESSAGE_SCOPED_FILE_CONTEXT': request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT,
'RAG_SYSTEM_CONTEXT': RAG_SYSTEM_CONTEXT,
'RAG_FILE_CONTEXT_SCOPE': request.app.state.config.RAG_FILE_CONTEXT_SCOPE,
'RAG_FILE_CONTEXT_SCOPE_EFFECTIVE': get_effective_rag_file_context_scope(request),
# Hybrid search settings
'ENABLE_RAG_HYBRID_SEARCH': request.app.state.config.ENABLE_RAG_HYBRID_SEARCH,
'ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS': request.app.state.config.ENABLE_RAG_HYBRID_SEARCH_ENRICHED_TEXTS,
@ -637,7 +653,7 @@ class ConfigForm(BaseModel):
TOP_K: int | None = None
BYPASS_EMBEDDING_AND_RETRIEVAL: bool | None = None
RAG_FULL_CONTEXT: bool | None = None
RAG_MESSAGE_SCOPED_FILE_CONTEXT: bool | None = None
RAG_FILE_CONTEXT_SCOPE: str | None = None
# Hybrid search settings
ENABLE_RAG_HYBRID_SEARCH: bool | None = None
@ -733,11 +749,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
if form_data.RAG_FULL_CONTEXT is not None
else request.app.state.config.RAG_FULL_CONTEXT
)
request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT = (
form_data.RAG_MESSAGE_SCOPED_FILE_CONTEXT
if form_data.RAG_MESSAGE_SCOPED_FILE_CONTEXT is not None
else request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT
)
if not RAG_SYSTEM_CONTEXT:
if form_data.RAG_FILE_CONTEXT_SCOPE is not None:
request.app.state.config.RAG_FILE_CONTEXT_SCOPE = form_data.RAG_FILE_CONTEXT_SCOPE
# Hybrid search settings
request.app.state.config.ENABLE_RAG_HYBRID_SEARCH = (
@ -1138,7 +1152,9 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
'TOP_K': request.app.state.config.TOP_K,
'BYPASS_EMBEDDING_AND_RETRIEVAL': request.app.state.config.BYPASS_EMBEDDING_AND_RETRIEVAL,
'RAG_FULL_CONTEXT': request.app.state.config.RAG_FULL_CONTEXT,
'RAG_MESSAGE_SCOPED_FILE_CONTEXT': request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT,
'RAG_SYSTEM_CONTEXT': RAG_SYSTEM_CONTEXT,
'RAG_FILE_CONTEXT_SCOPE': request.app.state.config.RAG_FILE_CONTEXT_SCOPE,
'RAG_FILE_CONTEXT_SCOPE_EFFECTIVE': get_effective_rag_file_context_scope(request),
# Hybrid search settings
'ENABLE_RAG_HYBRID_SEARCH': request.app.state.config.ENABLE_RAG_HYBRID_SEARCH,
'TOP_K_RERANKER': request.app.state.config.TOP_K_RERANKER,

5
backend/open_webui/utils/middleware.py
View file

@ -59,6 +59,7 @@ from open_webui.routers.pipelines import (
)
from open_webui.routers.retrieval import (
SearchForm,
is_message_scoped_file_context_enabled,
process_web_search,
)
from open_webui.routers.tasks import (
@ -2542,7 +2543,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
],
]
# Keep non-image files only long enough for optional message-scoped RAG.
if request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT:
if is_message_scoped_file_context_enabled(request):
non_image_files = [file for file in message.get('files', []) if is_file_context_item(file)]
if non_image_files:
message['files'] = non_image_files
@ -2836,7 +2837,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
# if prompt and len(prompt or "") < 500 and (not files or len(files) == 0):
# urls = extract_urls(prompt)
if request.app.state.config.RAG_MESSAGE_SCOPED_FILE_CONTEXT:
if is_message_scoped_file_context_enabled(request):
if file_context_enabled:
form_data['messages'], files, message_scoped_sources, message_scoped_contexts = (
await apply_message_file_contexts(

40
src/lib/components/admin/Settings/Documents.svelte
View file

@ -291,6 +291,10 @@
config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
if (!config.RAG_FILE_CONTEXT_SCOPE) {
config.RAG_FILE_CONTEXT_SCOPE = 'conversation';
}
RAGConfig = config;
});
</script>
@ -811,21 +815,29 @@
</div>
</div>
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
<Tooltip
content={$i18n.t(
'Keep attached files scoped to the user message they were sent with instead of merging all chat files into the latest RAG context.'
)}
placement="top-start"
>
{$i18n.t('Scope File Context by Message')}
</Tooltip>
{#if !RAGConfig.RAG_SYSTEM_CONTEXT}
<div class=" mb-2.5 flex w-full justify-between">
<div class=" self-center text-xs font-medium">
<Tooltip
content={$i18n.t(
'Choose whether attached files are treated as conversation-level context or scoped to the user message they were sent with.'
)}
placement="top-start"
>
{$i18n.t('File Context Scope')}
</Tooltip>
</div>
<div class="flex items-center relative">
<select
class="w-fit pr-8 rounded-sm px-2 text-xs bg-transparent outline-hidden text-right"
bind:value={RAGConfig.RAG_FILE_CONTEXT_SCOPE}
>
<option value="conversation">{$i18n.t('Conversation')}</option>
<option value="message">{$i18n.t('Message')}</option>
</select>
</div>
</div>
<div class="flex items-center relative">
<Switch bind:state={RAGConfig.RAG_MESSAGE_SCOPED_FILE_CONTEXT} />
</div>
</div>
{/if}
{#if !RAGConfig.BYPASS_EMBEDDING_AND_RETRIEVAL}
<div class=" mb-2.5 flex w-full justify-between">